# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. ############################################################################### # curl command tests for templeton # # #use Yahoo::Miners::Test::PigSetup; #PigSetup::setup(); #my $me = `whoami`; #chomp $me; $cfg = { 'driver' => 'Curl', 'groups' => [ ##============================================================================================================= { 'name' => 'TestKillJob', 'tests' => [ { 'num' => 1, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming', 'post_options' => ['user.name=:UNAME:','input=:INPDIR_HDFS:/nums.txt', 'input=:INPDIR_HDFS:/nums.txt', 'output=:OUTDIR:/mycounts', 'mapper=sleep 100', 'reducer=wc'], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'KILLED', # 'check_call_back' => 1, #TODO - enable call back check after fix 'kill_job_timeout' => 10, }, ] }, ##============================================================================================================= { 'name' => 'TestJobs', 'tests' => [ { 'num' => 1, #using non-existent jobid; checks http status code 'method' => 'GET', 'url' => ':TEMPLETON_URL:/templeton/v1/jobs/job_1399496111138_0011?user.name=:UNAME:', #results 'status_code' => 400, }, { 'num' => 2, #using non-existent jobid; checks http status code 'method' => 'DELETE', 'url' => ':TEMPLETON_URL:/templeton/v1/jobs/job_1399496111138_0011?user.name=:UNAME:', #results 'status_code' => 400, }, ] }, ##============================================================================================================= { 'name' => 'TestMapReduce', 'tests' => [ { 'num' => 1, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/jar', 'post_options' => ['user.name=:UNAME:','arg=:INPDIR_HDFS:/nums.txt', 'arg= :OUTDIR:/wc.txt', 'jar=:INPDIR_HDFS:/hexamples.jar', 'class=wordcount', 'define=MYPROP1=VALUE1', 'define=MYPROP2=VALUE2',], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_percent_complete' => 'map 100% reduce 100%', 'check_job_exit_value' => 0, 'check_call_back' => 1, }, { # with log enabled 'num' => 2, 'ignore23' => 'Log collector does not work with Hadoop 2', 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/jar', 'post_options' => ['user.name=:UNAME:','arg=:INPDIR_HDFS:/nums.txt', 'arg= :OUTDIR:/wc.txt', 'jar=:INPDIR_HDFS:/hexamples.jar', 'class=wordcount', 'statusdir=:OUTDIR:/status', 'enablelog=true' ], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_logs' => { 'job_num' => '1' }, 'check_job_exit_value' => 0, 'check_call_back' => 1, }, ] }, ##============================================================================================================= { 'name' => 'TestPig', 'tests' => [ { #test syntax error 'num' => 1, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/pig', 'post_options' => ['user.name=:UNAME:','execute=asdf', ], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_exit_value' => 8, 'check_call_back' => 1, }, { #valid syntax, hdfs operation through pig 'num' => 2, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/pig', 'post_options' => ['user.name=:UNAME:','execute=fs -ls :INPDIR_HDFS:;', ], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_exit_value' => 0, 'check_call_back' => 1, }, { #syntax check - valid syntax 'num' => 3, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/pig', 'post_options' => ['user.name=:UNAME:','arg=-check', 'file=:INPDIR_HDFS:/loadstore.pig', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', ], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_exit_value' => 0, 'check_call_back' => 1, }, { #syntax check cmd - valid syntax 'num' => 4, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/pig', 'post_options' => ['user.name=:UNAME:', 'arg=-d', 'arg=INFO' , 'execute=fs -ls :INPDIR_HDFS: ', ], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_exit_value' => 0, 'check_call_back' => 1, }, { #a simple load store script 'num' => 5, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/pig', 'post_options' => ['user.name=:UNAME:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/loadstore.pig', ], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_percent_complete' => '100% complete', 'check_job_exit_value' => 0, 'check_call_back' => 1, }, { #pig query registering jar 'num' => 6, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/pig', 'post_options' => ['user.name=:UNAME:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/jarregistered.pig', 'files=:INPDIR_HDFS:/piggybank.jar' ], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_exit_value' => 0, 'check_call_back' => 1, }, { #macro 'num' => 7, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/pig', 'post_options' => ['user.name=:UNAME:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/rowcount_withmacro.pig', 'files=:INPDIR_HDFS:/rowcountmacro.pig' ], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_exit_value' => 0, 'check_call_back' => 1, }, { #no file to be copied, should result in launcher job error 'num' => 8, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/pig', 'post_options' => ['user.name=:UNAME:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/no_such_file.pig', 'files=:INPDIR_HDFS:/rowcountmacro.pig' ], 'json_field_substr_match' => { 'error' => 'does not exist'}, #results 'status_code' => 400, }, { #Auto add quote around args 'ignore' => 'MS9 feature, will reenable later', 'num' => 9, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/pig', 'post_options' => ['user.name=:UNAME:','arg=-check', 'file=:INPDIR_HDFS:/loadstore.pig', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', ], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_exit_value' => 0, 'check_call_back' => 1, }, { #a simple load store script with log enabled 'num' => 10, 'ignore23' => 'Log collector does not work with Hadoop 2', 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/pig', 'post_options' => ['user.name=:UNAME:', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p', 'arg=OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/loadstore.pig', 'statusdir=:OUTDIR:/status', 'enablelog=true'], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_logs' => { 'job_num' => '1' }, 'check_job_exit_value' => 0, 'check_call_back' => 1, }, { #note: this test will fail unless Hive is installed in the default location Pig expects it in #HIVE-5547 will address this limitation 'num' => 11, 'setup' => [ { 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/ddl', 'status_code' => 200, 'post_options' => ['user.name=:UNAME:','exec=drop table if exists hcattest_pig; create table hcattest_pig(i int, j int) STORED AS textfile;'], 'json_field_substr_match' => {'stderr' => 'OK'} } ], 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/pig', 'post_options' => ['user.name=:UNAME:', 'arg=-useHCatalog', 'arg=-p', 'arg=INPDIR=:INPDIR_HDFS:', 'arg=-p', 'arg= OUTDIR=:OUTDIR:', 'file=:INPDIR_HDFS:/hcatloadstore.pig'], 'json_field_substr_match' => { 'id' => '\d+'}, 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_exit_value' => 0, 'check_call_back' => 1, }, #test 11 #TODO jython test ] }, ##============================================================================================================= { 'name' => 'TestHive', 'tests' => [ { #test syntax error 'num' => 1, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/hive', 'post_options' => ['user.name=:UNAME:','execute=asdf', ], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_exit_value' => 64, }, { #test show tables 'num' => 2, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/hive', 'post_options' => ['user.name=:UNAME:','execute=show tables', ], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_exit_value' => 0, 'check_call_back' => 1, }, { #test show tables 'num' => 3, 'ignore23' => 'Log collector does not work with Hadoop 2', 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/hive', 'post_options' => ['user.name=:UNAME:','execute=drop table if exists mynums;', ], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_exit_value' => 0, 'check_call_back' => 1, }, { #test show tables 'num' => 4, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/hive', 'post_options' => ['user.name=:UNAME:','execute=create external table mynums(a int, b int) location \':INPDIR_HDFS:/numstable/\';', ], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_exit_value' => 0, 'check_call_back' => 1, }, { #test describe 'num' => 5, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/hive', 'post_options' => ['user.name=:UNAME:','execute=describe formatted mynums', ], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_exit_value' => 0, 'check_call_back' => 1, }, { #test select * 'num' => 6, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/hive', 'post_options' => ['user.name=:UNAME:','execute=select * from mynums', ], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_exit_value' => 0, 'check_call_back' => 1, }, { #test select a,b 'num' => 7,#seems to be the same as test 6 except for percent_complete check 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/hive', 'post_options' => ['user.name=:UNAME:','execute=select count(*) from mynums', ], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_percent_complete' => 'map 100% reduce 100%|100% complete', 'check_job_exit_value' => 0, 'check_call_back' => 1, }, { #test udfs : select a,rand(b) 'num' => 8, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/hive', 'post_options' => ['user.name=:UNAME:','execute=select a,rand(b) from mynums', ], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_exit_value' => 0, }, { # test to use "add jar" command in hive script, the jar must be shipped to # templeton controller job using "files". Test #9 is a positive test case # for this case when jar is shipped, #10 is a negative test case 'num' => 9, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/hive', 'post_options' => ['user.name=:UNAME:','execute=add jar piggybank.jar', 'files=:INPDIR_HDFS:/piggybank.jar',], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_exit_value' => 0, 'check_call_back' => 1, }, { # negative test case for "add jar" when jar is not shipped to templeton # controller job using "files" 'num' => 10, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/hive', 'post_options' => ['user.name=:UNAME:','execute=add jar piggybank.jar',], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_exit_value' => 1, 'check_call_back' => 1, }, { #test add jar 'num' => 11, 'ignore23' => 'Log collector does not work with Hadoop 2', 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/hive', 'post_options' => ['user.name=:UNAME:','execute=add jar piggybank.jar', 'files=:INPDIR_HDFS:/piggybank.jar',], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_exit_value' => 0, 'check_call_back' => 1, }, { #test add jar when the jar is not shipped 'num' => 12, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/hive', 'post_options' => ['user.name=:UNAME:','execute=add jar piggybank.jar',], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_exit_value' => 1, 'check_call_back' => 1, }, { #enable logs 'num' => 13, 'ignore23' => 'Log collector does not work with Hadoop 2', 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/hive', 'post_options' => ['user.name=:UNAME:','execute=select a,b from mynums', 'statusdir=:OUTDIR:/status', 'enablelog=true'], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_logs' => { 'job_num' => '1' }, 'check_job_exit_value' => 0, 'check_call_back' => 1, }, ] }, ##============================================================================================================= { 'name' => 'TestSqoop', 'tests' => [ { # Test Sqoop Export 'num' => 1, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/sqoop?user.name=:UNAME:', 'post_options' => ['libdir=/apps/templeton/jdbc', 'command=export --connect :DB_CONNECTION_STRING: --username :DB_USER_NAME: --password :DB_PASSWORD: --export-dir :INPDIR_HDFS:/sqoop --table person','statusdir=TestSqoop_:TNUM:' ], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_percent_complete' => 'map 100% reduce 0%', 'check_job_exit_value' => 0, 'check_call_back' => 1, }, { # Test Sqoop Import and option file 'num' => 2, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/sqoop?user.name=:UNAME:', 'post_options' => ['libdir=/apps/templeton/jdbc', 'files=:INPDIR_HDFS:/sqoopcommand.txt','command=import --connect :DB_CONNECTION_STRING: --username :DB_USER_NAME: --password :DB_PASSWORD: --options-file sqoopcommand.txt','statusdir=TestSqoop_:TNUM:' ], 'json_field_substr_match' => { 'id' => '\d+'}, #results 'status_code' => 200, 'check_job_created' => 1, 'check_job_complete' => 'SUCCESS', 'check_job_percent_complete' => 'map 100% reduce 0%', 'check_job_exit_value' => 0, 'check_call_back' => 1, }, { # Test Sqoop using incomplete command 'num' => 3, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/sqoop?user.name=:UNAME:', 'post_options' => ['files=:INPDIR_HDFS:/sqoopcommand.txt','statusdir=TestSqoop_:TNUM:' ], 'json_field_substr_match' => { 'error' => 'Must define Sqoop command or a optionsfile contains Sqoop command to run Sqoop job.'}, #results 'status_code' => 400, }, { # Test Sqoop using incorrect command 'num' => 4, 'method' => 'POST', 'url' => ':TEMPLETON_URL:/templeton/v1/sqoop?user.name=:UNAME:', 'post_options' => ['optionsfile=:INPDIR_HDFS:/sqoopcommand.txt','command=import --connect :DB_CONNECTION_STRING: --username :DB_USER_NAME: --password :DB_PASSWORD: --options-file sqoopcommand.txt','statusdir=TestSqoop_:TNUM:' ], 'json_field_substr_match' => { 'error' => 'Cannot set command and optionsfile at the same time.'}, #results 'status_code' => 400, }, ] }, ] }, ;