############################################################################ # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. # The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. $cfg = { 'driver' => 'Pig', 'nummachines' => 5, # 'noverify' => 'true', # 1 Test that a nested foreach gives instant feedback: after user issue the foreach statement in Grunt # 2 Test that a nested foreach gives instant feedback: as part of execution: when we run this foreach statement, we will dump the schema for the nested alias # 3 Describe single Alias resulting from a nested Foreach # 4 Describe multiple Alias resulting from a nested Foreach # 5 Describe resulting from a nested Foreach that contains a positional parameter # 6 Describe for child Alias resulting from a nested Foreach where the child alias had multiple assignments The expected behavior is that the last assignment will destermine the result of the describe statement. # 7 Describe for an Alias resulting from a nested Foreach where the projection for the nested alias is empty # 8 Describe within a foreach statement # 9 Describe for alias with complex data types # 10 Describe that uses references an alias from an AS clause 'groups' => [ { 'name' => 'Grunt', 'tests' => [ { # TEST Simple Test using ls 'num' => 100, 'pig' => q\ ls :INPATH: sh ls sh echo hello sh ls -l :PIGHARNESS: sh echo hello2 \, ,'expected_out_regex' => "hdfs:.*singlefile" ,'expected_out_regex' => "hello" ,'expected_out_regex' => "hello2" ,'expected_out_regex' => "libexec" ,'rc' => 0 },{ # TEST Extensive Test using ls 'num' => 1, 'pig' => q\ sh ls -a :PIGHARNESS:/in/sql/address.txt sh ls -l :PIGHARNESS:/in/sql/dates.txt sh ls :PIGHARNESS: sh bash -c 'ls :PIGHARNESS:/in/sql/student*' sh bash -c 'ls -l :PIGHARNESS:/in/sql/c*.txt' sh ls /bin ls \, ,'expected_out_regex' => "address.txt" ,'expected_out_regex' => "dates.txt" ,'expected_out_regex' => "libexec" ,'expected_out_regex' => "-r.-r.-r--" ,'expected_out_regex' => "student" ,'expected_out_regex' => "customer" ,'expected_out_regex' => "rmdir" ,'rc' => 0 },{ # TEST pwd 'num' => 2, 'pig' => q\ pwd sh pwd \, ,'expected_out_regex' => "hdfs:" ,'rc' => 0 },{ # TEST for cat 'num' => 3, 'pig' => q\ sh cat :PIGHARNESS:/in/sql/numbers.txt \, ,'expected_out_regex' => "5000:5000:2:906:5060:5060L:5000.2f:5000.2" ,'rc' => 0 },{ # TEST for pipe using cat | wc #BROKEN should be fixed with nightly - pipe fixed 'num' => 4, 'pig' => q\ sh bash -c 'cat :PIGHARNESS:/in/sql/numbers.txt | wc' \, ,'expected_out_regex' => "5000.*5000.*199455" ,'rc' => 0 # TEST for cd },{ 'num' => 5, 'pig' => q\ sh bash -c cd :LOCALINPATH: \, ,'rc' => 0 # TEST for touch },{ 'num' => 6, 'pig' => q\ sh mkdir -p :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_6.test sh touch :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_6.test/newfile.txt sh ls -l :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_6.test/newfile.txt \, ,'expected_out_regex' => "Grunt_6.test/newfile.txt" ,'rc' => 0 },{ # TEST for rm 'num' => 7, 'pig' => q\ sh mkdir -p :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_7.test sh touch :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_7.test/newfile.txt sh rm -fv :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_7.test/newfile.txt \, ,'expected_out_regex' => "removed.*Grunt_7.test/newfile.txt" ,'rc' => 0 },{ # TEST for mkdir 'num' => 8, 'pig' => q\ sh mkdir -p :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_8.test/newdir sh touch :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_8.test/newdir/newfile.txt sh ls -l :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_8.test/newdir/newfile.txt \, ,'expected_out_regex' => "-r.-r.*newfile.txt" ,'rc' => 0 },{ # TEST for mkdir # TEST for chmod 775 file 'num' => 9, 'pig' => q\ sh mkdir -p :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_9.test sh touch :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_9.test/newfile.txt sh chmod -v 755 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_9.test/newfile.txt \, ,'expected_out_regex' => "changed to 0755" ,'rc' => 0 },{ 'num' => 10, 'pig' => q\ sh mkdir -vp :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_10.test/newdir sh rm -Rfv :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_10.test/newdir \, ,'expected_out_regex' => "mkdir: created directory.*newdir" ,'expected_out_regex' => "removed directory.*newdir" ,'rc' => 0 },{ # TEST for diff at directory level 'num' => 11, 'pig' => q\ A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double); B = limit A 5; C = limit A 10; store B into ':OUTPATH:/B.out'; store C into ':OUTPATH:/C.out'; copyToLocal :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_11.tmp sh bash -c 'diff :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_11.tmp/B.out/part-r-00000 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_11.tmp/C.out/part-r-00000' \, ,'expected_out_regex' => "1000.*9.*1.*904.*5018.*5018.*904.501.*904.501" ,'rc' => 0 },{ # TEST for diff two file where one is empty file 'num' => 12, 'pig' => q\ A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double); B = limit A 10; Empty = filter A by id > 99999; store B into ':OUTPATH:/B.out'; store Empty into ':OUTPATH:/Empty.out'; copyToLocal :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_12.test sh bash -c 'diff :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_12.test/B.out/part-r-00000 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_12.test/Empty.out/part-m-00000' \, ,'expected_out_regex'=>"1000.*10.*2.*900.*5051.*5051.*10.2.*10.2" ,'rc' => 0 },{ # TEST form chmod +x file 'num' => 13, 'pig' => q\ A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double); B = limit A 10; store B into ':OUTPATH:'; copyToLocal :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_13.test sh bash -c 'chmod -v -x :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_13.test/part-r-00000' \, ,'expected_out_regex' => "changed.*to.*0666.*rw-rw-rw-" ,'rc' => 0 },{ # TEST form chmod +775 file 'num' => 14, 'pig' => q\ A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double); B = limit A 10; store B into ':OUTPATH:'; copyToLocal :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_14.test sh bash -c 'chmod -v 775 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_14.test/part-r-00000' \, ,'expected_out_regex' => "changed.*to.*0775.*rwxrwxr-x" ,'rc' => 0 },{ # TEST for sort # TEST command "tail" # TEST multple unix commands in a single line #BROKEN NEEED UPDATE SHOULD WORK ON NIGHTLY 'num' => 15, 'pig' => q\ A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double); B = limit A 10; store B into ':OUTPATH:'; copyToLocal :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_15.test sh bash -c 'sort :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_15.test/part-r-00000 > :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_15.test/out.sorted' sh bash -c 'tail -n 2 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_15.test/out.sorted' \, ,'expected_out_regex' => "1000.*9.*1.*904.*5018.*5018.*904.501.*904.501" ,'rc' => 0 },{ # TEST for sort # TEST command "head" #BROKEN NEEED UPDATE SHOULD WORK ON NIGHTLY 'num' => 16, 'pig' => q\ A = load ':INPATH:/types/numbers.txt' using PigStorage(':') as (intnum1000: int,id: int,intnum5: int,intnum100: int,intnum: int,longnum: long,floatnum: float,doublenum: double); B = limit A 10; store B into ':OUTPATH:'; copyToLocal :OUTPATH: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_16.test sh bash -c 'sort :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_16.test/part-r-00000 > :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_16.test/out.sorted' sh bash -c 'head -n 2 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_16.test/out.sorted' \, ,'expected_out_regex' => "1000.*1.*3.*90.*5084.*5084.*8.3.*8.3" ,'rc' => 0 },{ 'num' => 17, 'pig' => q\ sh whoami \, ,'expected_out_regex' => "hadoopqa" ,'rc' => 0 },{ # TEST for comment # TEST for wc # TEST for wc with pipe # COMMENTS NOT SUPPORTED # sh #THIS IS A COMMENT #sh cat :INPATH:/types/numbers.txt | wc 'num' => 18, 'pig' => q\ copyToLocal :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_18.tmp/numbers.txt copyToLocal :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_18.tmp/numbers2.txt sh wc :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_18.tmp/numbers.txt sh wc -l :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_18.tmp/numbers2.txt \, ,'expected_out_regex' => "5000.*5000.*199455.*numbers.txt" ,'expected_out_regex' => "5000.*numbers2.txt" ,'rc' => 0 },{ # TEST for grep # TEST foringle quotes #BROKEN NEEED UPDATE SHOULD WORK ON NIGHTLY #May want t rewrite this so that we use pig to generate results to compare against 'num' => 19, 'pig' => q# copyToLocal :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_19.tmp/numbers.txt sh grep 503 :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_19.tmp/numbers.txt sh grep :506*: :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_19.tmp/numbers.txt sh bash -c 'grep \':501:\' :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_19.tmp/numbers.txt' #, ,'expected_out_regex' => "1000:501:5:90:5019:5019L:8.5f:8.5" ,'expected_out_regex' => "1000:503:3:90:5064:5064L:0.506490f:0.506490" ,'expected_out_regex' => "5000:4972:5:90:5066:5066L:5.9f:5.9" ,'rc' => 0 },{ # TEST for uniq 'num' => 20, 'pig' => q\ copyToLocal :INPATH:/singlefile/studenttab10k :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_20.tmp/studenttab10k sh uniq :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_20.tmp/studenttab10k \, ,'expected_out_regex' => "gabriella.*brown.*60.*1.71" ,'rc' => 0 # TEST for find # TEST for wildcard #sh find :INPATH: -name studenttab10k },{ 'num' => 21, 'pig' => q\ sh mkdir :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_21.tmp/singlefile copyToLocal :INPATH:/singlefile/studenttab10k :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_21.tmp/singlefile/studenttab10k sh find :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_21.tmp -name student* \, ,'expected_out_regex' => "studenttab10k" ,'rc' => 0 },{ # TEST for df 'num' => 22, 'pig' => q\ sh df \, ,'expected_out_regex' => "Filesystem.*Used.*Available" ,'rc' => 0 },{ # TEST for du 'num' => 23, 'pig' => q\ sh du :PIGHARNESS:/in \, ,'expected_out_regex' => "sql" ,'rc' => 0 },{ # TEST for comment - not supported #sh #IGNORE THIS # TEST for echo 'num' => 24, 'pig' => q\ sh echo This little piggy went to market sh echo This little piggy stayed home sh echo This little piggy had roast beef sh echo This little piggy had none sh echo This little piggy went wee wee wee, all the way home \, ,'expected_out_regex' => "market" ,'expected_out_regex' => "stayed" ,'expected_out_regex' => "roast" ,'expected_out_regex' => "none" ,'expected_out_regex' => "all the way home" ,'rc' => 0 },{ # TEST for pipe #BROKEN NEEED UPDATE SHOULD WORK ON NIGHTLY 'num' => 25, 'pig' => q\ sh bash -c 'echo This little piggy went to market | wc' sh bash -c 'echo This little piggy stayed home | wc' sh bash -c 'echo This little piggy had roast beef | wc' sh bash -c 'echo This little piggy had none | wc' sh bash -c 'echo This little piggy went wee wee wee, all the way home | wc' \, ,'expected_out_regex' => "1.*6.*33" ,'expected_out_regex' => "1.*5.*30" ,'expected_out_regex' => "1.*5.*27" ,'rc' => 0 },{ # TEST for environmet variable 'num' => 26, 'pig' => q\ sh bash -c 'echo $PIG_HARNESS_ROOT' sh bash -c 'ls $PIG_HARNESS_ROOT' \, ,'expected_out_regex' => "libexec" ,'rc' => 0 # TEST read file you don't have read rights for },{ 'num' => 27, 'pig' => q\ sh touch /bin/bad \, ,'expected_err_regex' => "Permission denied" ,'rc' => 0 },{ 'num' => 28, 'pig' => q\ sh ls -l doesnotexist1 sh ls -l doesnotexist2/bad sh ls -l :OUTPATH:/doesnotexist3.txt \, ,'expected_err_regex' => "doesnotexist1.*No such file or directory" ,'expected_err_regex' => "doesnotexist2/bad.*No such file or directory" ,'expected_err_regex' => "doesnotexist3.txt.*No such file or directory" ,'rc' => 0 },{ # TEST run a shell script # BUG-3990418 May result in "../../../../..: Permission denied" Error if the top # level directory does not have read permissions, this will not be fixed 'num' => 29, 'pig' => " sh $ENV{PIG_HARNESS_ROOT}/bin/run_fb.sh -h ", ,'expected_out_regex' => "Usage.*run_fb.sh" ,'rc' => 0 },{ # TEST run awk script #BROKEN - removing this awk script for now # 'num' => 30, # 'pig' => q\ #sh mkdir :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_30.tmp/types #copyToLocal :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_30.tmp/numbers.txt #sh bash -c 'awk '{ print }' :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_30.tmp/numbers.txt' #\, # ,'expected_err_regex' => "5000:5000:2:906:5060:5060L:5000.2f:5000.2" # # ,'rc' => 0 # },{ # TEST run sed script 'num' => 31, 'pig' => q\ sh mkdir :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_31.tmp/types copyToLocal :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_31.tmp/types/numbers.txt sh bash -c 'sed -e "1d" :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_31.tmp/types/numbers.txt' \, ,'expected_err_regex' => "5000:5000:2:906:5060:5060L:5000.2f:5000.2" ,'rc' => 0 },{ #TEST run perl script 'num' => 32, 'pig' => q\ sh mkdir :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_32.tmp/types copyToLocal :INPATH:/types/numbers.txt :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_32.tmp/types/numbers.txt sh bash -c 'perl -p -e "s/:/+/g;" :LOCALTESTPATH:/hadoopqa.:RUNID:/Grunt_32.tmp/types/numbers.txt' \, },{ #TEST run perl script #BROKEN Causes a 2998 Unhandled Exception, check to see if fixed in nightly 'num' => 33, 'pig' => q\ sh bash -c ls -l //grid/2/dev/pigqa/in/types; ls -l /grid/2/dev/pigqa/in/glob; ls -l //grid/2/dev/pigqa/in/types; ls -l /grid/2/dev/pigqa/in/glob; ls \, #sh bash -c 'ls -l //grid/2/dev/pigqa/in/types; ls -l /grid/2/dev/pigqa/in/glob; ls -l //grid/2/dev/pigqa/in/types; ls -l /grid/2/dev/pigqa/in/glob; ls' ,'expected_err_regex' => "address.txt" ,'rc' => 0 #IGNORE THIS TEST ISN'T READY # },{ # 'num' => 33, # 'pig' => q\ #sh echo "export FOO=MyEnvVariable" > :OUTPATH:/env.txt #sh source :OUTPATH:/env.txt #sh source :OUTPATH:/env.txt; echo $FOO #\, # ,'expected_out_regex' => "E: {age: bytearray}" # ,'rc' => 0 } #ADD #wildcards #Add sort with tab delimeter ] }, ], }, ;