PREHOOK: query: create table json_t (key string, jstring string) PREHOOK: type: CREATETABLE POSTHOOK: query: create table json_t (key string, jstring string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: default@json_t PREHOOK: query: insert overwrite table json_t select * from ( select '1', '{"f1": "value1", "f2": "value2", "f3": 3, "f5": 5.23}' from src limit 1 union all select '2', '{"f1": "value12", "f3": "value3", "f2": 2, "f4": 4.01}' from src limit 1 union all select '3', '{"f1": "value13", "f4": "value44", "f3": "value33", "f2": 2, "f5": 5.01}' from src limit 1 union all select '4', cast(null as string) from src limit 1 union all select '5', '{"f1": "", "f5": null}' from src limit 1 union all select '6', '[invalid JSON string]' from src limit 1 ) s PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@json_t POSTHOOK: query: insert overwrite table json_t select * from ( select '1', '{"f1": "value1", "f2": "value2", "f3": 3, "f5": 5.23}' from src limit 1 union all select '2', '{"f1": "value12", "f3": "value3", "f2": 2, "f4": 4.01}' from src limit 1 union all select '3', '{"f1": "value13", "f4": "value44", "f3": "value33", "f2": 2, "f5": 5.01}' from src limit 1 union all select '4', cast(null as string) from src limit 1 union all select '5', '{"f1": "", "f5": null}' from src limit 1 union all select '6', '[invalid JSON string]' from src limit 1 ) s POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@json_t POSTHOOK: Lineage: json_t.jstring EXPRESSION [] POSTHOOK: Lineage: json_t.key EXPRESSION [] PREHOOK: query: explain select a.key, b.* from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key PREHOOK: type: QUERY POSTHOOK: query: explain select a.key, b.* from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key POSTHOOK: type: QUERY POSTHOOK: Lineage: json_t.jstring EXPRESSION [] POSTHOOK: Lineage: json_t.key EXPRESSION [] ABSTRACT SYNTAX TREE: (TOK_QUERY (TOK_FROM (TOK_LATERAL_VIEW (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION json_tuple (. (TOK_TABLE_OR_COL a) jstring) 'f1' 'f2' 'f3' 'f4' 'f5') f1 f2 f3 f4 f5 (TOK_TABALIAS b))) (TOK_TABREF (TOK_TABNAME json_t) a))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key)) (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME b)))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL a) key))))) STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 is a root stage STAGE PLANS: Stage: Stage-1 Map Reduce Alias -> Map Operator Tree: a TableScan alias: a Lateral View Forward Select Operator expressions: expr: key type: string outputColumnNames: key Lateral View Join Operator outputColumnNames: _col0, _col4, _col5, _col6, _col7, _col8 Select Operator expressions: expr: _col0 type: string expr: _col4 type: string expr: _col5 type: string expr: _col6 type: string expr: _col7 type: string expr: _col8 type: string outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Reduce Output Operator key expressions: expr: _col0 type: string sort order: + tag: -1 value expressions: expr: _col0 type: string expr: _col1 type: string expr: _col2 type: string expr: _col3 type: string expr: _col4 type: string expr: _col5 type: string Select Operator expressions: expr: jstring type: string expr: 'f1' type: string expr: 'f2' type: string expr: 'f3' type: string expr: 'f4' type: string expr: 'f5' type: string outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 UDTF Operator function name: json_tuple Lateral View Join Operator outputColumnNames: _col0, _col4, _col5, _col6, _col7, _col8 Select Operator expressions: expr: _col0 type: string expr: _col4 type: string expr: _col5 type: string expr: _col6 type: string expr: _col7 type: string expr: _col8 type: string outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 Reduce Output Operator key expressions: expr: _col0 type: string sort order: + tag: -1 value expressions: expr: _col0 type: string expr: _col1 type: string expr: _col2 type: string expr: _col3 type: string expr: _col4 type: string expr: _col5 type: string Reduce Operator Tree: Extract File Output Operator compressed: false GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat Stage: Stage-0 Fetch Operator limit: -1 PREHOOK: query: select a.key, b.* from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key PREHOOK: type: QUERY PREHOOK: Input: default@json_t #### A masked pattern was here #### POSTHOOK: query: select a.key, b.* from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key POSTHOOK: type: QUERY POSTHOOK: Input: default@json_t #### A masked pattern was here #### POSTHOOK: Lineage: json_t.jstring EXPRESSION [] POSTHOOK: Lineage: json_t.key EXPRESSION [] 1 value1 value2 3 NULL 5.23 2 value12 2 value3 4.01 NULL 3 value13 2 value33 value44 5.01 4 NULL NULL NULL NULL NULL 5 NULL NULL NULL NULL 6 NULL NULL NULL NULL NULL PREHOOK: query: explain select json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') as (f1, f2, f3, f4, f5) from json_t a order by f1, f2, f3 PREHOOK: type: QUERY POSTHOOK: query: explain select json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') as (f1, f2, f3, f4, f5) from json_t a order by f1, f2, f3 POSTHOOK: type: QUERY POSTHOOK: Lineage: json_t.jstring EXPRESSION [] POSTHOOK: Lineage: json_t.key EXPRESSION [] ABSTRACT SYNTAX TREE: (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME json_t) a)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION json_tuple (. (TOK_TABLE_OR_COL a) jstring) 'f1' 'f2' 'f3' 'f4' 'f5') f1 f2 f3 f4 f5)) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL f1)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL f2)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL f3))))) STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 is a root stage STAGE PLANS: Stage: Stage-1 Map Reduce Alias -> Map Operator Tree: a TableScan alias: a Select Operator expressions: expr: jstring type: string expr: 'f1' type: string expr: 'f2' type: string expr: 'f3' type: string expr: 'f4' type: string expr: 'f5' type: string outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 UDTF Operator function name: json_tuple Reduce Output Operator key expressions: expr: c0 type: string expr: c1 type: string expr: c2 type: string sort order: +++ tag: -1 value expressions: expr: c0 type: string expr: c1 type: string expr: c2 type: string expr: c3 type: string expr: c4 type: string Reduce Operator Tree: Extract File Output Operator compressed: false GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat Stage: Stage-0 Fetch Operator limit: -1 PREHOOK: query: select json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') as (f1, f2, f3, f4, f5) from json_t a order by f1, f2, f3 PREHOOK: type: QUERY PREHOOK: Input: default@json_t #### A masked pattern was here #### POSTHOOK: query: select json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') as (f1, f2, f3, f4, f5) from json_t a order by f1, f2, f3 POSTHOOK: type: QUERY POSTHOOK: Input: default@json_t #### A masked pattern was here #### POSTHOOK: Lineage: json_t.jstring EXPRESSION [] POSTHOOK: Lineage: json_t.key EXPRESSION [] NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL value1 value2 3 NULL 5.23 value12 2 value3 4.01 NULL value13 2 value33 value44 5.01 PREHOOK: query: explain select a.key, b.f2, b.f5 from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key PREHOOK: type: QUERY POSTHOOK: query: explain select a.key, b.f2, b.f5 from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key POSTHOOK: type: QUERY POSTHOOK: Lineage: json_t.jstring EXPRESSION [] POSTHOOK: Lineage: json_t.key EXPRESSION [] ABSTRACT SYNTAX TREE: (TOK_QUERY (TOK_FROM (TOK_LATERAL_VIEW (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION json_tuple (. (TOK_TABLE_OR_COL a) jstring) 'f1' 'f2' 'f3' 'f4' 'f5') f1 f2 f3 f4 f5 (TOK_TABALIAS b))) (TOK_TABREF (TOK_TABNAME json_t) a))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) f2)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) f5))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL a) key))))) STAGE DEPENDENCIES: Stage-1 is a root stage Stage-0 is a root stage STAGE PLANS: Stage: Stage-1 Map Reduce Alias -> Map Operator Tree: a TableScan alias: a Lateral View Forward Select Operator expressions: expr: key type: string outputColumnNames: key Lateral View Join Operator outputColumnNames: _col0, _col4, _col5, _col6, _col7, _col8 Select Operator expressions: expr: _col0 type: string expr: _col5 type: string expr: _col8 type: string outputColumnNames: _col0, _col1, _col2 Reduce Output Operator key expressions: expr: _col0 type: string sort order: + tag: -1 value expressions: expr: _col0 type: string expr: _col1 type: string expr: _col2 type: string Select Operator expressions: expr: jstring type: string expr: 'f1' type: string expr: 'f2' type: string expr: 'f3' type: string expr: 'f4' type: string expr: 'f5' type: string outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 UDTF Operator function name: json_tuple Lateral View Join Operator outputColumnNames: _col0, _col4, _col5, _col6, _col7, _col8 Select Operator expressions: expr: _col0 type: string expr: _col5 type: string expr: _col8 type: string outputColumnNames: _col0, _col1, _col2 Reduce Output Operator key expressions: expr: _col0 type: string sort order: + tag: -1 value expressions: expr: _col0 type: string expr: _col1 type: string expr: _col2 type: string Reduce Operator Tree: Extract File Output Operator compressed: false GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat Stage: Stage-0 Fetch Operator limit: -1 PREHOOK: query: select a.key, b.f2, b.f5 from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key PREHOOK: type: QUERY PREHOOK: Input: default@json_t #### A masked pattern was here #### POSTHOOK: query: select a.key, b.f2, b.f5 from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 order by a.key POSTHOOK: type: QUERY POSTHOOK: Input: default@json_t #### A masked pattern was here #### POSTHOOK: Lineage: json_t.jstring EXPRESSION [] POSTHOOK: Lineage: json_t.key EXPRESSION [] 1 value2 5.23 2 2 NULL 3 2 5.01 4 NULL NULL 5 NULL NULL 6 NULL NULL PREHOOK: query: explain select f2, count(*) from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 where f1 is not null group by f2 order by f2 PREHOOK: type: QUERY POSTHOOK: query: explain select f2, count(*) from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 where f1 is not null group by f2 order by f2 POSTHOOK: type: QUERY POSTHOOK: Lineage: json_t.jstring EXPRESSION [] POSTHOOK: Lineage: json_t.key EXPRESSION [] ABSTRACT SYNTAX TREE: (TOK_QUERY (TOK_FROM (TOK_LATERAL_VIEW (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION json_tuple (. (TOK_TABLE_OR_COL a) jstring) 'f1' 'f2' 'f3' 'f4' 'f5') f1 f2 f3 f4 f5 (TOK_TABALIAS b))) (TOK_TABREF (TOK_TABNAME json_t) a))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL f2)) (TOK_SELEXPR (TOK_FUNCTIONSTAR count))) (TOK_WHERE (TOK_FUNCTION TOK_ISNOTNULL (TOK_TABLE_OR_COL f1))) (TOK_GROUPBY (TOK_TABLE_OR_COL f2)) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL f2))))) STAGE DEPENDENCIES: Stage-1 is a root stage Stage-2 depends on stages: Stage-1 Stage-0 is a root stage STAGE PLANS: Stage: Stage-1 Map Reduce Alias -> Map Operator Tree: a TableScan alias: a Lateral View Forward Select Operator Lateral View Join Operator outputColumnNames: _col4, _col5, _col6, _col7, _col8 Filter Operator predicate: expr: _col4 is not null type: boolean Select Operator expressions: expr: _col5 type: string outputColumnNames: _col5 Group By Operator aggregations: expr: count() bucketGroup: false keys: expr: _col5 type: string mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: expr: _col0 type: string sort order: + Map-reduce partition columns: expr: _col0 type: string tag: -1 value expressions: expr: _col1 type: bigint Select Operator expressions: expr: jstring type: string expr: 'f1' type: string expr: 'f2' type: string expr: 'f3' type: string expr: 'f4' type: string expr: 'f5' type: string outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5 UDTF Operator function name: json_tuple Lateral View Join Operator outputColumnNames: _col4, _col5, _col6, _col7, _col8 Filter Operator predicate: expr: _col4 is not null type: boolean Select Operator expressions: expr: _col5 type: string outputColumnNames: _col5 Group By Operator aggregations: expr: count() bucketGroup: false keys: expr: _col5 type: string mode: hash outputColumnNames: _col0, _col1 Reduce Output Operator key expressions: expr: _col0 type: string sort order: + Map-reduce partition columns: expr: _col0 type: string tag: -1 value expressions: expr: _col1 type: bigint Reduce Operator Tree: Group By Operator aggregations: expr: count(VALUE._col0) bucketGroup: false keys: expr: KEY._col0 type: string mode: mergepartial outputColumnNames: _col0, _col1 Select Operator expressions: expr: _col0 type: string expr: _col1 type: bigint outputColumnNames: _col0, _col1 File Output Operator compressed: false GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat Stage: Stage-2 Map Reduce Alias -> Map Operator Tree: #### A masked pattern was here #### Reduce Output Operator key expressions: expr: _col0 type: string sort order: + tag: -1 value expressions: expr: _col0 type: string expr: _col1 type: bigint Reduce Operator Tree: Extract File Output Operator compressed: false GlobalTableId: 0 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat Stage: Stage-0 Fetch Operator limit: -1 PREHOOK: query: select f2, count(*) from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 where f1 is not null group by f2 order by f2 PREHOOK: type: QUERY PREHOOK: Input: default@json_t #### A masked pattern was here #### POSTHOOK: query: select f2, count(*) from json_t a lateral view json_tuple(a.jstring, 'f1', 'f2', 'f3', 'f4', 'f5') b as f1, f2, f3, f4, f5 where f1 is not null group by f2 order by f2 POSTHOOK: type: QUERY POSTHOOK: Input: default@json_t #### A masked pattern was here #### POSTHOOK: Lineage: json_t.jstring EXPRESSION [] POSTHOOK: Lineage: json_t.key EXPRESSION [] NULL 1 2 2 value2 1 PREHOOK: query: -- Verify that json_tuple can handle new lines in JSON values CREATE TABLE dest1(c1 STRING) STORED AS RCFILE PREHOOK: type: CREATETABLE POSTHOOK: query: -- Verify that json_tuple can handle new lines in JSON values CREATE TABLE dest1(c1 STRING) STORED AS RCFILE POSTHOOK: type: CREATETABLE POSTHOOK: Output: default@dest1 POSTHOOK: Lineage: json_t.jstring EXPRESSION [] POSTHOOK: Lineage: json_t.key EXPRESSION [] PREHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT '{"a":"b\nc"}' FROM src LIMIT 1 PREHOOK: type: QUERY PREHOOK: Input: default@src PREHOOK: Output: default@dest1 POSTHOOK: query: INSERT OVERWRITE TABLE dest1 SELECT '{"a":"b\nc"}' FROM src LIMIT 1 POSTHOOK: type: QUERY POSTHOOK: Input: default@src POSTHOOK: Output: default@dest1 POSTHOOK: Lineage: dest1.c1 SIMPLE [] POSTHOOK: Lineage: json_t.jstring EXPRESSION [] POSTHOOK: Lineage: json_t.key EXPRESSION [] PREHOOK: query: SELECT * FROM dest1 PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### POSTHOOK: query: SELECT * FROM dest1 POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1 #### A masked pattern was here #### POSTHOOK: Lineage: dest1.c1 SIMPLE [] POSTHOOK: Lineage: json_t.jstring EXPRESSION [] POSTHOOK: Lineage: json_t.key EXPRESSION [] {"a":"b c"} PREHOOK: query: SELECT json FROM dest1 a LATERAL VIEW json_tuple(c1, 'a') b AS json PREHOOK: type: QUERY PREHOOK: Input: default@dest1 #### A masked pattern was here #### POSTHOOK: query: SELECT json FROM dest1 a LATERAL VIEW json_tuple(c1, 'a') b AS json POSTHOOK: type: QUERY POSTHOOK: Input: default@dest1 #### A masked pattern was here #### POSTHOOK: Lineage: dest1.c1 SIMPLE [] POSTHOOK: Lineage: json_t.jstring EXPRESSION [] POSTHOOK: Lineage: json_t.key EXPRESSION [] b c