<%! /** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ %> <%@ page contentType="text/html; charset=UTF-8" import="javax.servlet.http.*" import="java.io.*" import="java.util.*" import="org.apache.hadoop.http.HtmlQuoting" import="org.apache.hadoop.mapred.*" import="org.apache.hadoop.fs.*" import="org.apache.hadoop.util.*" import="java.text.SimpleDateFormat" import="org.apache.hadoop.mapred.JobHistory.*" %> <%! private static SimpleDateFormat dateFormat = new SimpleDateFormat("d/MM HH:mm:ss") ; %> <%! private static final long serialVersionUID = 1L; %> <% String logFile = request.getParameter("logFile"); if (logFile == null) { out.println("Missing job!!"); return; } String encodedLogFileName = JobHistory.JobInfo.encodeJobHistoryFilePath(logFile); String jobid = JSPUtil.getJobID(new Path(encodedLogFileName).getName()); String numTasks = request.getParameter("numTasks"); int showTasks = 10 ; if (numTasks != null) { showTasks = Integer.parseInt(numTasks); } FileSystem fs = (FileSystem) application.getAttribute("fileSys"); JobConf jobConf = (JobConf) application.getAttribute("jobConf"); ACLsManager aclsManager = (ACLsManager) application.getAttribute("aclManager"); JobHistory.JobInfo job = JSPUtil.checkAccessAndGetJobInfo(request, response, jobConf, aclsManager, fs, new Path(logFile)); if (job == null) { return; }%>

Hadoop Job <%=jobid %>

User : <%=HtmlQuoting.quoteHtmlChars(job.get(Keys.USER)) %>
JobName : <%=HtmlQuoting.quoteHtmlChars(job.get(Keys.JOBNAME)) %>
JobConf : <%=job.get(Keys.JOBCONF) %>
Submitted At : <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLong(Keys.SUBMIT_TIME), 0 ) %>
Launched At : <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLong(Keys.LAUNCH_TIME), job.getLong(Keys.SUBMIT_TIME)) %>
Finished At : <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLong(Keys.FINISH_TIME), job.getLong(Keys.LAUNCH_TIME)) %>
Status : <%= ((job.get(Keys.JOB_STATUS) == null)?"Incomplete" :job.get(Keys.JOB_STATUS)) %>

<% if (!Values.SUCCESS.name().equals(job.get(Keys.JOB_STATUS))) { out.print("

No Analysis available as job did not finish

"); return; } Map tasks = job.getAllTasks(); int finishedMaps = job.getInt(Keys.FINISHED_MAPS) ; int finishedReduces = job.getInt(Keys.FINISHED_REDUCES) ; JobHistory.Task [] mapTasks = new JobHistory.Task[finishedMaps]; JobHistory.Task [] reduceTasks = new JobHistory.Task[finishedReduces]; int mapIndex = 0 , reduceIndex=0; long avgMapTime = 0; long avgReduceTime = 0; long avgShuffleTime = 0; for (JobHistory.Task task : tasks.values()) { Map attempts = task.getTaskAttempts(); for (JobHistory.TaskAttempt attempt : attempts.values()) { if (attempt.get(Keys.TASK_STATUS).equals(Values.SUCCESS.name())) { long avgFinishTime = (attempt.getLong(Keys.FINISH_TIME) - attempt.getLong(Keys.START_TIME)); if (Values.MAP.name().equals(task.get(Keys.TASK_TYPE))) { mapTasks[mapIndex++] = attempt ; avgMapTime += avgFinishTime; } else if (Values.REDUCE.name().equals(task.get(Keys.TASK_TYPE))) { reduceTasks[reduceIndex++] = attempt; avgShuffleTime += (attempt.getLong(Keys.SHUFFLE_FINISHED) - attempt.getLong(Keys.START_TIME)); avgReduceTime += (attempt.getLong(Keys.FINISH_TIME) - attempt.getLong(Keys.SHUFFLE_FINISHED)); } break; } } } if (finishedMaps > 0) { avgMapTime /= finishedMaps; } if (finishedReduces > 0) { avgReduceTime /= finishedReduces; avgShuffleTime /= finishedReduces; } Comparator cMap = new Comparator(){ public int compare(JobHistory.Task t1, JobHistory.Task t2){ long l1 = t1.getLong(Keys.FINISH_TIME) - t1.getLong(Keys.START_TIME); long l2 = t2.getLong(Keys.FINISH_TIME) - t2.getLong(Keys.START_TIME); return (l2 cShuffle = new Comparator(){ public int compare(JobHistory.Task t1, JobHistory.Task t2){ long l1 = t1.getLong(Keys.SHUFFLE_FINISHED) - t1.getLong(Keys.START_TIME); long l2 = t2.getLong(Keys.SHUFFLE_FINISHED) - t2.getLong(Keys.START_TIME); return (l2 cFinishMapRed = new Comparator() { public int compare(JobHistory.Task t1, JobHistory.Task t2){ long l1 = t1.getLong(Keys.FINISH_TIME); long l2 = t2.getLong(Keys.FINISH_TIME); return (l2 0) { Arrays.sort(mapTasks, cMap); JobHistory.Task minMap = mapTasks[mapTasks.length-1] ; %>

Time taken by best performing Map task <%=minMap.get(Keys.TASKID) %> : <%=StringUtils.formatTimeDiff(minMap.getLong(Keys.FINISH_TIME), minMap.getLong(Keys.START_TIME) ) %>

Average time taken by Map tasks: <%=StringUtils.formatTimeDiff(avgMapTime, 0) %>

Worse performing map tasks

<% for (int i=0;i <% } %>

Task Id	Time taken
<%=mapTasks[i].get(Keys.TASKID) %>	<%=StringUtils.formatTimeDiff(mapTasks[i].getLong(Keys.FINISH_TIME), mapTasks[i].getLong(Keys.START_TIME)) %>

<% Arrays.sort(mapTasks, cFinishMapRed); JobHistory.Task lastMap = mapTasks[0] ; %>

The last Map task <%=lastMap.get(Keys.TASKID) %> finished at (relative to the Job launch time): <%=StringUtils.getFormattedTimeWithDiff(dateFormat, lastMap.getLong(Keys.FINISH_TIME), job.getLong(Keys.LAUNCH_TIME) ) %>

<% }//end if(mapTasks.length > 0) if (reduceTasks.length <= 0) return; Arrays.sort(reduceTasks, cShuffle); JobHistory.Task minShuffle = reduceTasks[reduceTasks.length-1] ; %>

Time taken by best performing shufflejobId <%=minShuffle.get(Keys.TASKID)%> : <%=StringUtils.formatTimeDiff(minShuffle.getLong(Keys.SHUFFLE_FINISHED), minShuffle.getLong(Keys.START_TIME) ) %>

Average time taken by Shuffle: <%=StringUtils.formatTimeDiff(avgShuffleTime, 0) %>

Worse performing Shuffle(s)

<% for (int i=0;i <% } %>

Task Id	Time taken
<%=reduceTasks[i].get(Keys.TASKID) %>	<%= StringUtils.formatTimeDiff( reduceTasks[i].getLong(Keys.SHUFFLE_FINISHED), reduceTasks[i].getLong(Keys.START_TIME)) %>

<% Comparator cFinishShuffle = new Comparator() { public int compare(JobHistory.Task t1, JobHistory.Task t2){ long l1 = t1.getLong(Keys.SHUFFLE_FINISHED); long l2 = t2.getLong(Keys.SHUFFLE_FINISHED); return (l2

The last Shuffle <%=lastShuffle.get(Keys.TASKID)%> finished at (relative to the Job launch time): <%=StringUtils.getFormattedTimeWithDiff(dateFormat, lastShuffle.getLong(Keys.SHUFFLE_FINISHED), job.getLong(Keys.LAUNCH_TIME) ) %>

<% Comparator cReduce = new Comparator(){ public int compare(JobHistory.Task t1, JobHistory.Task t2){ long l1 = t1.getLong(Keys.FINISH_TIME) - t1.getLong(Keys.SHUFFLE_FINISHED); long l2 = t2.getLong(Keys.FINISH_TIME) - t2.getLong(Keys.SHUFFLE_FINISHED); return (l2

Time taken by best performing Reduce task : <%=minReduce.get(Keys.TASKID) %> : <%=StringUtils.formatTimeDiff(minReduce.getLong(Keys.FINISH_TIME), minReduce.getLong(Keys.SHUFFLE_FINISHED) ) %>

Average time taken by Reduce tasks: <%=StringUtils.formatTimeDiff(avgReduceTime, 0) %>

Worse performing reduce tasks

<% for (int i=0;i <% } %>

Task Id	Time taken
<%=reduceTasks[i].get(Keys.TASKID) %>	<%=StringUtils.formatTimeDiff( reduceTasks[i].getLong(Keys.FINISH_TIME), reduceTasks[i].getLong(Keys.SHUFFLE_FINISHED)) %>

<% Arrays.sort(reduceTasks, cFinishMapRed); JobHistory.Task lastReduce = reduceTasks[0] ; %>

The last Reduce task <%=lastReduce.get(Keys.TASKID)%> finished at (relative to the Job launch time): <%=StringUtils.getFormattedTimeWithDiff(dateFormat, lastReduce.getLong(Keys.FINISH_TIME), job.getLong(Keys.LAUNCH_TIME) ) %>