PREHOOK: query: CREATE TABLE columnarserde_create_shortcut(a array, b array, c map, d int, e string) STORED AS RCFILE PREHOOK: type: CREATETABLE POSTHOOK: query: CREATE TABLE columnarserde_create_shortcut(a array, b array, c map, d int, e string) STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: default@columnarserde_create_shortcut PREHOOK: query: EXPLAIN FROM src_thrift INSERT OVERWRITE TABLE columnarserde_create_shortcut SELECT src_thrift.lint, src_thrift.lstring, src_thrift.mstringstring, src_thrift.aint, src_thrift.astring DISTRIBUTE BY 1 PREHOOK: type: QUERY POSTHOOK: query: EXPLAIN FROM src_thrift INSERT OVERWRITE TABLE columnarserde_create_shortcut SELECT src_thrift.lint, src_thrift.lstring, src_thrift.mstringstring, src_thrift.aint, src_thrift.astring DISTRIBUTE BY 1 POSTHOOK: type: QUERY ABSTRACT SYNTAX TREE: (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src_thrift))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME columnarserde_create_shortcut))) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL src_thrift) lint)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src_thrift) lstring)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src_thrift) mstringstring)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src_thrift) aint)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL src_thrift) astring))) (TOK_DISTRIBUTEBY 1))) STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 depends on stages: Stage-1 Stage-2 depends on stages: Stage-0 STAGE PLANS: Stage: Stage-1 Map Reduce Alias -> Map Operator Tree: src_thrift TableScan alias: src_thrift Select Operator expressions: expr: lint type: array expr: lstring type: array expr: mstringstring type: map expr: aint type: int expr: astring type: string outputColumnNames: _col0, _col1, _col2, _col3, _col4 Reduce Output Operator sort order: Map-reduce partition columns: expr: 1 type: int tag: -1 value expressions: expr: _col0 type: array expr: _col1 type: array expr: _col2 type: map expr: _col3 type: int expr: _col4 type: string Reduce Operator Tree: Extract File Output Operator compressed: false GlobalTableId: 1 table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.columnarserde_create_shortcut Stage: Stage-0 Move Operator tables: replace: true table: input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe name: default.columnarserde_create_shortcut Stage: Stage-2 Stats-Aggr Operator PREHOOK: query: FROM src_thrift INSERT OVERWRITE TABLE columnarserde_create_shortcut SELECT src_thrift.lint, src_thrift.lstring, src_thrift.mstringstring, src_thrift.aint, src_thrift.astring DISTRIBUTE BY 1 PREHOOK: type: QUERY PREHOOK: Input: default@src_thrift PREHOOK: Output: default@columnarserde_create_shortcut POSTHOOK: query: FROM src_thrift INSERT OVERWRITE TABLE columnarserde_create_shortcut SELECT src_thrift.lint, src_thrift.lstring, src_thrift.mstringstring, src_thrift.aint, src_thrift.astring DISTRIBUTE BY 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src_thrift POSTHOOK: Output: default@columnarserde_create_shortcut POSTHOOK: Lineage: columnarserde_create_shortcut.a SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.b SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lstring, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.c SIMPLE [(src_thrift)src_thrift.FieldSchema(name:mstringstring, type:map, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.d SIMPLE [(src_thrift)src_thrift.FieldSchema(name:aint, type:int, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.e SIMPLE [(src_thrift)src_thrift.FieldSchema(name:astring, type:string, comment:from deserializer), ] PREHOOK: query: SELECT columnarserde_create_shortcut.* FROM columnarserde_create_shortcut CLUSTER BY 1 PREHOOK: type: QUERY PREHOOK: Input: default@columnarserde_create_shortcut #### A masked pattern was here #### POSTHOOK: query: SELECT columnarserde_create_shortcut.* FROM columnarserde_create_shortcut CLUSTER BY 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@columnarserde_create_shortcut #### A masked pattern was here #### POSTHOOK: Lineage: columnarserde_create_shortcut.a SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.b SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lstring, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.c SIMPLE [(src_thrift)src_thrift.FieldSchema(name:mstringstring, type:map, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.d SIMPLE [(src_thrift)src_thrift.FieldSchema(name:aint, type:int, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.e SIMPLE [(src_thrift)src_thrift.FieldSchema(name:astring, type:string, comment:from deserializer), ] [0,0,0] ["0","0","0"] {"key_0":"value_0"} 1712634731 record_0 [1,2,3] ["10","100","1000"] {"key_1":"value_1"} 465985200 record_1 [2,4,6] ["20","200","2000"] {"key_2":"value_2"} -751827638 record_2 [3,6,9] ["30","300","3000"] {"key_3":"value_3"} 477111222 record_3 [4,8,12] ["40","400","4000"] {"key_4":"value_4"} -734328909 record_4 [5,10,15] ["50","500","5000"] {"key_5":"value_5"} -1952710710 record_5 [6,12,18] ["60","600","6000"] {"key_6":"value_6"} 1244525190 record_6 [7,14,21] ["70","700","7000"] {"key_7":"value_7"} -1461153973 record_7 [8,16,24] ["80","800","8000"] {"key_8":"value_8"} 1638581578 record_8 [9,18,27] ["90","900","9000"] {"key_9":"value_9"} 336964413 record_9 null null null 0 NULL PREHOOK: query: SELECT columnarserde_create_shortcut.a[0], columnarserde_create_shortcut.b[0], columnarserde_create_shortcut.c['key2'], columnarserde_create_shortcut.d, columnarserde_create_shortcut.e FROM columnarserde_create_shortcut CLUSTER BY 1 PREHOOK: type: QUERY PREHOOK: Input: default@columnarserde_create_shortcut #### A masked pattern was here #### POSTHOOK: query: SELECT columnarserde_create_shortcut.a[0], columnarserde_create_shortcut.b[0], columnarserde_create_shortcut.c['key2'], columnarserde_create_shortcut.d, columnarserde_create_shortcut.e FROM columnarserde_create_shortcut CLUSTER BY 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@columnarserde_create_shortcut #### A masked pattern was here #### POSTHOOK: Lineage: columnarserde_create_shortcut.a SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.b SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lstring, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.c SIMPLE [(src_thrift)src_thrift.FieldSchema(name:mstringstring, type:map, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.d SIMPLE [(src_thrift)src_thrift.FieldSchema(name:aint, type:int, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.e SIMPLE [(src_thrift)src_thrift.FieldSchema(name:astring, type:string, comment:from deserializer), ] 0 0 NULL 1712634731 record_0 1 10 NULL 465985200 record_1 2 20 NULL -751827638 record_2 3 30 NULL 477111222 record_3 4 40 NULL -734328909 record_4 5 50 NULL -1952710710 record_5 6 60 NULL 1244525190 record_6 7 70 NULL -1461153973 record_7 8 80 NULL 1638581578 record_8 9 90 NULL 336964413 record_9 NULL NULL NULL 0 NULL PREHOOK: query: CREATE table columnShortcutTable (key STRING, value STRING) STORED AS RCFILE PREHOOK: type: CREATETABLE POSTHOOK: query: CREATE table columnShortcutTable (key STRING, value STRING) STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: default@columnShortcutTable POSTHOOK: Lineage: columnarserde_create_shortcut.a SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.b SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lstring, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.c SIMPLE [(src_thrift)src_thrift.FieldSchema(name:mstringstring, type:map, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.d SIMPLE [(src_thrift)src_thrift.FieldSchema(name:aint, type:int, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.e SIMPLE [(src_thrift)src_thrift.FieldSchema(name:astring, type:string, comment:from deserializer), ] PREHOOK: query: FROM src INSERT OVERWRITE TABLE columnShortcutTable SELECT src.key, src.value LIMIT 10 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@columnshortcuttable POSTHOOK: query: FROM src INSERT OVERWRITE TABLE columnShortcutTable SELECT src.key, src.value LIMIT 10 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@columnshortcuttable POSTHOOK: Lineage: columnarserde_create_shortcut.a SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.b SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lstring, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.c SIMPLE [(src_thrift)src_thrift.FieldSchema(name:mstringstring, type:map, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.d SIMPLE [(src_thrift)src_thrift.FieldSchema(name:aint, type:int, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.e SIMPLE [(src_thrift)src_thrift.FieldSchema(name:astring, type:string, comment:from deserializer), ] POSTHOOK: Lineage: columnshortcuttable.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: columnshortcuttable.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: describe columnShortcutTable PREHOOK: type: DESCTABLE POSTHOOK: query: describe columnShortcutTable POSTHOOK: type: DESCTABLE POSTHOOK: Lineage: columnarserde_create_shortcut.a SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.b SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lstring, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.c SIMPLE [(src_thrift)src_thrift.FieldSchema(name:mstringstring, type:map, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.d SIMPLE [(src_thrift)src_thrift.FieldSchema(name:aint, type:int, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.e SIMPLE [(src_thrift)src_thrift.FieldSchema(name:astring, type:string, comment:from deserializer), ] POSTHOOK: Lineage: columnshortcuttable.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: columnshortcuttable.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] key string value string PREHOOK: query: SELECT columnShortcutTable.* FROM columnShortcutTable ORDER BY key ASC, value ASC PREHOOK: type: QUERY PREHOOK: Input: default@columnshortcuttable #### A masked pattern was here #### POSTHOOK: query: SELECT columnShortcutTable.* FROM columnShortcutTable ORDER BY key ASC, value ASC POSTHOOK: type: QUERY POSTHOOK: Input: default@columnshortcuttable #### A masked pattern was here #### POSTHOOK: Lineage: columnarserde_create_shortcut.a SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.b SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lstring, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.c SIMPLE [(src_thrift)src_thrift.FieldSchema(name:mstringstring, type:map, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.d SIMPLE [(src_thrift)src_thrift.FieldSchema(name:aint, type:int, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.e SIMPLE [(src_thrift)src_thrift.FieldSchema(name:astring, type:string, comment:from deserializer), ] POSTHOOK: Lineage: columnshortcuttable.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: columnshortcuttable.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] 165 val_165 238 val_238 255 val_255 27 val_27 278 val_278 311 val_311 409 val_409 484 val_484 86 val_86 98 val_98 PREHOOK: query: ALTER TABLE columnShortcutTable ADD COLUMNS (c string) PREHOOK: type: ALTERTABLE_ADDCOLS PREHOOK: Input: default@columnshortcuttable PREHOOK: Output: default@columnshortcuttable POSTHOOK: query: ALTER TABLE columnShortcutTable ADD COLUMNS (c string) POSTHOOK: type: ALTERTABLE_ADDCOLS POSTHOOK: Input: default@columnshortcuttable POSTHOOK: Output: default@columnshortcuttable POSTHOOK: Lineage: columnarserde_create_shortcut.a SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.b SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lstring, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.c SIMPLE [(src_thrift)src_thrift.FieldSchema(name:mstringstring, type:map, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.d SIMPLE [(src_thrift)src_thrift.FieldSchema(name:aint, type:int, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.e SIMPLE [(src_thrift)src_thrift.FieldSchema(name:astring, type:string, comment:from deserializer), ] POSTHOOK: Lineage: columnshortcuttable.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: columnshortcuttable.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: SELECT columnShortcutTable.* FROM columnShortcutTable ORDER BY key ASC, value ASC PREHOOK: type: QUERY PREHOOK: Input: default@columnshortcuttable #### A masked pattern was here #### POSTHOOK: query: SELECT columnShortcutTable.* FROM columnShortcutTable ORDER BY key ASC, value ASC POSTHOOK: type: QUERY POSTHOOK: Input: default@columnshortcuttable #### A masked pattern was here #### POSTHOOK: Lineage: columnarserde_create_shortcut.a SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.b SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lstring, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.c SIMPLE [(src_thrift)src_thrift.FieldSchema(name:mstringstring, type:map, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.d SIMPLE [(src_thrift)src_thrift.FieldSchema(name:aint, type:int, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.e SIMPLE [(src_thrift)src_thrift.FieldSchema(name:astring, type:string, comment:from deserializer), ] POSTHOOK: Lineage: columnshortcuttable.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: columnshortcuttable.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] 165 val_165 NULL 238 val_238 NULL 255 val_255 NULL 27 val_27 NULL 278 val_278 NULL 311 val_311 NULL 409 val_409 NULL 484 val_484 NULL 86 val_86 NULL 98 val_98 NULL PREHOOK: query: ALTER TABLE columnShortcutTable REPLACE COLUMNS (key int) PREHOOK: type: ALTERTABLE_REPLACECOLS PREHOOK: Input: default@columnshortcuttable PREHOOK: Output: default@columnshortcuttable POSTHOOK: query: ALTER TABLE columnShortcutTable REPLACE COLUMNS (key int) POSTHOOK: type: ALTERTABLE_REPLACECOLS POSTHOOK: Input: default@columnshortcuttable POSTHOOK: Output: default@columnshortcuttable POSTHOOK: Lineage: columnarserde_create_shortcut.a SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.b SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lstring, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.c SIMPLE [(src_thrift)src_thrift.FieldSchema(name:mstringstring, type:map, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.d SIMPLE [(src_thrift)src_thrift.FieldSchema(name:aint, type:int, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.e SIMPLE [(src_thrift)src_thrift.FieldSchema(name:astring, type:string, comment:from deserializer), ] POSTHOOK: Lineage: columnshortcuttable.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: columnshortcuttable.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: SELECT columnShortcutTable.* FROM columnShortcutTable ORDER BY key ASC PREHOOK: type: QUERY PREHOOK: Input: default@columnshortcuttable #### A masked pattern was here #### POSTHOOK: query: SELECT columnShortcutTable.* FROM columnShortcutTable ORDER BY key ASC POSTHOOK: type: QUERY POSTHOOK: Input: default@columnshortcuttable #### A masked pattern was here #### POSTHOOK: Lineage: columnarserde_create_shortcut.a SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lint, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.b SIMPLE [(src_thrift)src_thrift.FieldSchema(name:lstring, type:array, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.c SIMPLE [(src_thrift)src_thrift.FieldSchema(name:mstringstring, type:map, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.d SIMPLE [(src_thrift)src_thrift.FieldSchema(name:aint, type:int, comment:from deserializer), ] POSTHOOK: Lineage: columnarserde_create_shortcut.e SIMPLE [(src_thrift)src_thrift.FieldSchema(name:astring, type:string, comment:from deserializer), ] POSTHOOK: Lineage: columnshortcuttable.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: columnshortcuttable.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] 27 86 98 165 238 255 278 311 409 484