PREHOOK: query: -- scanning partitioned data create table tmptable(key string, value string, hr string, ds string) PREHOOK: type: CREATETABLE POSTHOOK: query: -- scanning partitioned data create table tmptable(key string, value string, hr string, ds string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: default@tmptable PREHOOK: query: explain extended insert overwrite table tmptable select a.* from srcpart a where rand(1) < 0.1 and a.ds = '2008-04-08' PREHOOK: type: QUERY POSTHOOK: query: explain extended insert overwrite table tmptable select a.* from srcpart a where rand(1) < 0.1 and a.ds = '2008-04-08' POSTHOOK: type: QUERY ABSTRACT SYNTAX TREE: (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart) a)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME tmptable))) (TOK_SELECT (TOK_SELEXPR (TOK_ALLCOLREF (TOK_TABNAME a)))) (TOK_WHERE (and (< (TOK_FUNCTION rand 1) 0.1) (= (. (TOK_TABLE_OR_COL a) ds) '2008-04-08'))))) STAGE DEPENDENCIES: Stage-1 is a root stage Stage-5 depends on stages: Stage-1 , consists of Stage-4, Stage-3 Stage-4 Stage-0 depends on stages: Stage-4, Stage-3 Stage-2 depends on stages: Stage-0 Stage-3 STAGE PLANS: Stage: Stage-1 Map Reduce Alias -> Map Operator Tree: a TableScan alias: a GatherStats: false Filter Operator isSamplingPred: false predicate: expr: (rand(1) < 0.1) type: boolean Select Operator expressions: expr: key type: string expr: value type: string expr: ds type: string expr: hr type: string outputColumnNames: _col0, _col1, _col2, _col3 File Output Operator compressed: false GlobalTableId: 1 directory: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_16-59-18_964_3618004060055722151/-ext-10002 NumFilesPerFileSink: 1 Stats Publishing Key Prefix: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_16-59-18_964_3618004060055722151/-ext-10000/ table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 columns key,value,hr,ds columns.types string:string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat location pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/tmptable name default.tmptable serialization.ddl struct tmptable { string key, string value, string hr, string ds} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe transient_lastDdlTime 1297385958 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable TotalFiles: 1 GatherStats: true MultiFileSpray: false Needs Tagging: false Path -> Alias: pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 [a] pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 [a] Path -> Partition: pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 Partition base file name: hr=11 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat partition values: ds 2008-04-08 hr 11 properties: bucket_count -1 columns key,value columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat location pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11 name default.srcpart partition_columns ds/hr serialization.ddl struct srcpart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe transient_lastDdlTime 1297378962 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 columns key,value columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat location pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/srcpart name default.srcpart partition_columns ds/hr serialization.ddl struct srcpart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe transient_lastDdlTime 1297378962 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.srcpart name: default.srcpart pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 Partition base file name: hr=12 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat partition values: ds 2008-04-08 hr 12 properties: bucket_count -1 columns key,value columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat location pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=12 name default.srcpart partition_columns ds/hr serialization.ddl struct srcpart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe transient_lastDdlTime 1297378962 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 columns key,value columns.types string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat location pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/srcpart name default.srcpart partition_columns ds/hr serialization.ddl struct srcpart { string key, string value} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe transient_lastDdlTime 1297378962 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.srcpart name: default.srcpart Stage: Stage-5 Conditional Operator Stage: Stage-4 Move Operator files: hdfs directory: true source: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_16-59-18_964_3618004060055722151/-ext-10002 destination: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_16-59-18_964_3618004060055722151/-ext-10000 Stage: Stage-0 Move Operator tables: replace: true source: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_16-59-18_964_3618004060055722151/-ext-10000 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 columns key,value,hr,ds columns.types string:string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat location pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/tmptable name default.tmptable serialization.ddl struct tmptable { string key, string value, string hr, string ds} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe transient_lastDdlTime 1297385958 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable tmp directory: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_16-59-18_964_3618004060055722151/-ext-10001 Stage: Stage-2 Stats-Aggr Operator Stats Aggregation Key Prefix: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_16-59-18_964_3618004060055722151/-ext-10000/ Stage: Stage-3 Map Reduce Alias -> Map Operator Tree: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_16-59-18_964_3618004060055722151/-ext-10002 File Output Operator compressed: false GlobalTableId: 0 directory: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_16-59-18_964_3618004060055722151/-ext-10000 NumFilesPerFileSink: 1 table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 columns key,value,hr,ds columns.types string:string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat location pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/tmptable name default.tmptable serialization.ddl struct tmptable { string key, string value, string hr, string ds} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe transient_lastDdlTime 1297385958 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable TotalFiles: 1 GatherStats: false MultiFileSpray: false Needs Tagging: false Path -> Alias: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_16-59-18_964_3618004060055722151/-ext-10002 [pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_16-59-18_964_3618004060055722151/-ext-10002] Path -> Partition: pfile:/data/users/sdong/www/open-source-hive1/build/ql/scratchdir/hive_2011-02-10_16-59-18_964_3618004060055722151/-ext-10002 Partition base file name: -ext-10002 input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 columns key,value,hr,ds columns.types string:string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat location pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/tmptable name default.tmptable serialization.ddl struct tmptable { string key, string value, string hr, string ds} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe transient_lastDdlTime 1297385958 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat properties: bucket_count -1 columns key,value,hr,ds columns.types string:string:string:string file.inputformat org.apache.hadoop.mapred.TextInputFormat file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat location pfile:/data/users/sdong/www/open-source-hive1/build/ql/test/data/warehouse/tmptable name default.tmptable serialization.ddl struct tmptable { string key, string value, string hr, string ds} serialization.format 1 serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe transient_lastDdlTime 1297385958 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name: default.tmptable name: default.tmptable PREHOOK: query: insert overwrite table tmptable select a.* from srcpart a where rand(1) < 0.1 and a.ds = '2008-04-08' PREHOOK: type: QUERY PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 PREHOOK: Output: default@tmptable POSTHOOK: query: insert overwrite table tmptable select a.* from srcpart a where rand(1) < 0.1 and a.ds = '2008-04-08' POSTHOOK: type: QUERY POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11 POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12 POSTHOOK: Output: default@tmptable POSTHOOK: Lineage: tmptable.ds SIMPLE [(srcpart)a.FieldSchema(name:hr, type:string, comment:null), ] POSTHOOK: Lineage: tmptable.hr SIMPLE [(srcpart)a.FieldSchema(name:ds, type:string, comment:null), ] POSTHOOK: Lineage: tmptable.key SIMPLE [(srcpart)a.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tmptable.value SIMPLE [(srcpart)a.FieldSchema(name:value, type:string, comment:default), ] PREHOOK: query: select * from tmptable x sort by x.key,x.value,x.ds,x.hr PREHOOK: type: QUERY PREHOOK: Input: default@tmptable PREHOOK: Output: file:/tmp/sdong/hive_2011-02-10_16-59-23_923_7364727689222600083/-mr-10000 POSTHOOK: query: select * from tmptable x sort by x.key,x.value,x.ds,x.hr POSTHOOK: type: QUERY POSTHOOK: Input: default@tmptable POSTHOOK: Output: file:/tmp/sdong/hive_2011-02-10_16-59-23_923_7364727689222600083/-mr-10000 POSTHOOK: Lineage: tmptable.ds SIMPLE [(srcpart)a.FieldSchema(name:hr, type:string, comment:null), ] POSTHOOK: Lineage: tmptable.hr SIMPLE [(srcpart)a.FieldSchema(name:ds, type:string, comment:null), ] POSTHOOK: Lineage: tmptable.key SIMPLE [(srcpart)a.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: tmptable.value SIMPLE [(srcpart)a.FieldSchema(name:value, type:string, comment:default), ] 103 val_103 2008-04-08 11 118 val_118 2008-04-08 12 119 val_119 2008-04-08 12 119 val_119 2008-04-08 12 126 val_126 2008-04-08 12 131 val_131 2008-04-08 12 133 val_133 2008-04-08 11 134 val_134 2008-04-08 11 138 val_138 2008-04-08 11 143 val_143 2008-04-08 12 152 val_152 2008-04-08 11 153 val_153 2008-04-08 11 162 val_162 2008-04-08 12 169 val_169 2008-04-08 11 170 val_170 2008-04-08 11 175 val_175 2008-04-08 12 176 val_176 2008-04-08 11 176 val_176 2008-04-08 11 18 val_18 2008-04-08 11 18 val_18 2008-04-08 12 191 val_191 2008-04-08 12 197 val_197 2008-04-08 11 199 val_199 2008-04-08 11 200 val_200 2008-04-08 12 201 val_201 2008-04-08 12 202 val_202 2008-04-08 11 203 val_203 2008-04-08 11 209 val_209 2008-04-08 11 214 val_214 2008-04-08 12 217 val_217 2008-04-08 11 218 val_218 2008-04-08 12 221 val_221 2008-04-08 11 223 val_223 2008-04-08 12 224 val_224 2008-04-08 11 229 val_229 2008-04-08 12 230 val_230 2008-04-08 12 233 val_233 2008-04-08 11 233 val_233 2008-04-08 12 237 val_237 2008-04-08 12 238 val_238 2008-04-08 12 256 val_256 2008-04-08 12 26 val_26 2008-04-08 11 265 val_265 2008-04-08 12 273 val_273 2008-04-08 12 277 val_277 2008-04-08 11 277 val_277 2008-04-08 12 280 val_280 2008-04-08 12 286 val_286 2008-04-08 12 288 val_288 2008-04-08 11 298 val_298 2008-04-08 11 309 val_309 2008-04-08 11 309 val_309 2008-04-08 12 310 val_310 2008-04-08 11 317 val_317 2008-04-08 11 322 val_322 2008-04-08 12 323 val_323 2008-04-08 12 325 val_325 2008-04-08 12 331 val_331 2008-04-08 11 332 val_332 2008-04-08 12 336 val_336 2008-04-08 11 336 val_336 2008-04-08 12 339 val_339 2008-04-08 12 341 val_341 2008-04-08 12 342 val_342 2008-04-08 12 348 val_348 2008-04-08 11 348 val_348 2008-04-08 12 35 val_35 2008-04-08 12 364 val_364 2008-04-08 12 37 val_37 2008-04-08 11 378 val_378 2008-04-08 11 384 val_384 2008-04-08 11 389 val_389 2008-04-08 11 400 val_400 2008-04-08 11 403 val_403 2008-04-08 12 407 val_407 2008-04-08 12 409 val_409 2008-04-08 11 417 val_417 2008-04-08 12 42 val_42 2008-04-08 11 424 val_424 2008-04-08 12 429 val_429 2008-04-08 11 429 val_429 2008-04-08 12 430 val_430 2008-04-08 12 431 val_431 2008-04-08 11 432 val_432 2008-04-08 12 44 val_44 2008-04-08 11 453 val_453 2008-04-08 11 454 val_454 2008-04-08 11 457 val_457 2008-04-08 11 457 val_457 2008-04-08 12 458 val_458 2008-04-08 11 466 val_466 2008-04-08 12 467 val_467 2008-04-08 11 469 val_469 2008-04-08 11 469 val_469 2008-04-08 11 469 val_469 2008-04-08 11 47 val_47 2008-04-08 12 470 val_470 2008-04-08 12 489 val_489 2008-04-08 11 491 val_491 2008-04-08 11 496 val_496 2008-04-08 12 498 val_498 2008-04-08 11 498 val_498 2008-04-08 12 51 val_51 2008-04-08 11 58 val_58 2008-04-08 12 70 val_70 2008-04-08 11 72 val_72 2008-04-08 12 74 val_74 2008-04-08 11 77 val_77 2008-04-08 11 77 val_77 2008-04-08 12 78 val_78 2008-04-08 11 82 val_82 2008-04-08 12 87 val_87 2008-04-08 12 90 val_90 2008-04-08 12 97 val_97 2008-04-08 12 97 val_97 2008-04-08 12 98 val_98 2008-04-08 12