Hadoop Job <%=jobid%> on History Viewer

<%! /** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ %> <%@ page contentType="text/html; charset=UTF-8" import="javax.servlet.http.*" import="java.io.*" import="java.util.*" import="org.apache.hadoop.fs.*" import="org.apache.hadoop.http.HtmlQuoting" import="org.apache.hadoop.mapred.*" import="org.apache.hadoop.util.*" import="java.text.*" import="org.apache.hadoop.mapred.JobHistory.*" import="java.security.PrivilegedExceptionAction" import="org.apache.hadoop.security.AccessControlException" import="org.apache.hadoop.mapreduce.JobACL" import="org.apache.hadoop.security.authorize.AccessControlList" %> <%! private static final long serialVersionUID = 1L; %> <%! static SimpleDateFormat dateFormat = new SimpleDateFormat("d-MMM-yyyy HH:mm:ss") ; %> <% String logFile = request.getParameter("logFile"); String jobid = JSPUtil.getJobID(new Path(logFile).getName()); FileSystem fs = (FileSystem) application.getAttribute("fileSys"); JobConf jobConf = (JobConf) application.getAttribute("jobConf"); ACLsManager aclsManager = (ACLsManager) application.getAttribute("aclManager"); JobHistory.JobInfo job = JSPUtil.checkAccessAndGetJobInfo(request, response, jobConf, aclsManager, fs, new Path(logFile)); if (job == null) { return; } String encodedLogFileName = JobHistory.JobInfo.encodeJobHistoryFilePath(logFile); %> Hadoop Job <%=jobid%> on History Viewer

Hadoop Job <%=jobid %> on History Viewer

User: <%=HtmlQuoting.quoteHtmlChars(job.get(Keys.USER)) %>
JobName: <%=HtmlQuoting.quoteHtmlChars(job.get(Keys.JOBNAME)) %>
JobConf: <%=job.get(Keys.JOBCONF) %>
<% Map jobAcls = job.getJobACLs(); JSPUtil.printJobACLs(jobConf, jobAcls, out); %> Submitted At: <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLong(Keys.SUBMIT_TIME), 0 ) %>
Launched At: <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLong(Keys.LAUNCH_TIME), job.getLong(Keys.SUBMIT_TIME)) %>
Finished At: <%=StringUtils.getFormattedTimeWithDiff(dateFormat, job.getLong(Keys.FINISH_TIME), job.getLong(Keys.LAUNCH_TIME)) %>
Status: <%= ((job.get(Keys.JOB_STATUS) == "")?"Incomplete" :job.get(Keys.JOB_STATUS)) %>
Failure Info: <%= ((job.get(Keys.FAIL_REASON) == null)?"NA" : job.get(Keys.FAIL_REASON)) %>
<% Map tasks = job.getAllTasks(); int totalMaps = 0 ; int totalReduces = 0; int totalCleanups = 0; int totalSetups = 0; int numFailedMaps = 0; int numKilledMaps = 0; int numFailedReduces = 0 ; int numKilledReduces = 0; int numFinishedCleanups = 0; int numFailedCleanups = 0; int numKilledCleanups = 0; int numFinishedSetups = 0; int numFailedSetups = 0; int numKilledSetups = 0; long mapStarted = 0 ; long mapFinished = 0 ; long reduceStarted = 0 ; long reduceFinished = 0; long cleanupStarted = 0; long cleanupFinished = 0; long setupStarted = 0; long setupFinished = 0; Map allHosts = new TreeMap(); for (JobHistory.Task task : tasks.values()) { Map attempts = task.getTaskAttempts(); allHosts.put(task.get(Keys.HOSTNAME), ""); for (TaskAttempt attempt : attempts.values()) { long startTime = attempt.getLong(Keys.START_TIME) ; long finishTime = attempt.getLong(Keys.FINISH_TIME) ; if (Values.MAP.name().equals(task.get(Keys.TASK_TYPE))){ if (mapStarted==0 || mapStarted > startTime ) { mapStarted = startTime; } if (mapFinished < finishTime ) { mapFinished = finishTime ; } totalMaps++; if (Values.FAILED.name().equals(attempt.get(Keys.TASK_STATUS))) { numFailedMaps++; } else if (Values.KILLED.name().equals(attempt.get(Keys.TASK_STATUS))) { numKilledMaps++; } } else if (Values.REDUCE.name().equals(task.get(Keys.TASK_TYPE))) { if (reduceStarted==0||reduceStarted > startTime) { reduceStarted = startTime ; } if (reduceFinished < finishTime) { reduceFinished = finishTime; } totalReduces++; if (Values.FAILED.name().equals(attempt.get(Keys.TASK_STATUS))) { numFailedReduces++; } else if (Values.KILLED.name().equals(attempt.get(Keys.TASK_STATUS))) { numKilledReduces++; } } else if (Values.CLEANUP.name().equals(task.get(Keys.TASK_TYPE))) { if (cleanupStarted==0||cleanupStarted > startTime) { cleanupStarted = startTime ; } if (cleanupFinished < finishTime) { cleanupFinished = finishTime; } totalCleanups++; if (Values.SUCCESS.name().equals(attempt.get(Keys.TASK_STATUS))) { numFinishedCleanups++; } else if (Values.FAILED.name().equals(attempt.get(Keys.TASK_STATUS))) { numFailedCleanups++; } else if (Values.KILLED.name().equals(attempt.get(Keys.TASK_STATUS))) { numKilledCleanups++; } } else if (Values.SETUP.name().equals(task.get(Keys.TASK_TYPE))) { if (setupStarted==0||setupStarted > startTime) { setupStarted = startTime ; } if (setupFinished < finishTime) { setupFinished = finishTime; } totalSetups++; if (Values.SUCCESS.name().equals(attempt.get(Keys.TASK_STATUS))) { numFinishedSetups++; } else if (Values.FAILED.name().equals(attempt.get(Keys.TASK_STATUS))) { numFailedSetups++; } else if (Values.KILLED.name().equals(attempt.get(Keys.TASK_STATUS))) { numKilledSetups++; } } } } %> Analyse This Job

Kind	Total Tasks(successful+failed+killed)	Successful tasks	Failed tasks	Killed tasks	Start Time	Finish Time
Setup	<%=totalSetups%>	<%=numFinishedSetups%>	<%=numFailedSetups%>	<%=numKilledSetups%>	<%=StringUtils.getFormattedTimeWithDiff(dateFormat, setupStarted, 0) %>	<%=StringUtils.getFormattedTimeWithDiff(dateFormat, setupFinished, setupStarted) %>
Map	<%=totalMaps %>	<%=job.getInt(Keys.FINISHED_MAPS) %>	<%=numFailedMaps %>	<%=numKilledMaps %>	<%=StringUtils.getFormattedTimeWithDiff(dateFormat, mapStarted, 0) %>	<%=StringUtils.getFormattedTimeWithDiff(dateFormat, mapFinished, mapStarted) %>
Reduce	<%=totalReduces%>	<%=job.getInt(Keys.FINISHED_REDUCES)%>	<%=numFailedReduces%>	<%=numKilledReduces%>	<%=StringUtils.getFormattedTimeWithDiff(dateFormat, reduceStarted, 0) %>	<%=StringUtils.getFormattedTimeWithDiff(dateFormat, reduceFinished, reduceStarted) %>
Cleanup	<%=totalCleanups%>	<%=numFinishedCleanups%>	<%=numFailedCleanups%>	<%=numKilledCleanups%>	<%=StringUtils.getFormattedTimeWithDiff(dateFormat, cleanupStarted, 0) %>	<%=StringUtils.getFormattedTimeWithDiff(dateFormat, cleanupFinished, cleanupStarted) %>

<% Counters totalCounters = Counters.fromEscapedCompactString(job.get(Keys.COUNTERS)); Counters mapCounters = Counters.fromEscapedCompactString(job.get(Keys.MAP_COUNTERS)); Counters reduceCounters = Counters.fromEscapedCompactString(job.get(Keys.REDUCE_COUNTERS)); if (totalCounters != null) { for (String groupName : totalCounters.getGroupNames()) { Counters.Group totalGroup = totalCounters.getGroup(groupName); Counters.Group mapGroup = mapCounters.getGroup(groupName); Counters.Group reduceGroup = reduceCounters.getGroup(groupName); Format decimal = new DecimalFormat(); boolean isFirst = true; Iterator ctrItr = totalGroup.iterator(); while(ctrItr.hasNext()) { Counters.Counter counter = ctrItr.next(); String name = counter.getDisplayName(); String mapValue = decimal.format(mapGroup.getCounter(name)); String reduceValue = decimal.format(reduceGroup.getCounter(name)); String totalValue = decimal.format(counter.getCounter()); %> <% if (isFirst) { isFirst = false; %> <% } %> <% } } } %>

	Counter	Map	Reduce	Total
<%=HtmlQuoting.quoteHtmlChars(totalGroup.getDisplayName())%>	<%=HtmlQuoting.quoteHtmlChars(counter.getDisplayName())%>	<%=mapValue%>	<%=reduceValue%>	<%=totalValue%>

<% DefaultJobHistoryParser.FailedOnNodesFilter filter = new DefaultJobHistoryParser.FailedOnNodesFilter(); JobHistory.parseHistoryFromFS(logFile, filter, fs); Map> badNodes = filter.getValues(); if (badNodes.size() > 0) { %>

Failed tasks attempts by nodes

<% for (Map.Entry> entry : badNodes.entrySet()) { String node = entry.getKey(); Set failedTasks = entry.getValue(); %> <% } } %>

Hostname	Failed Tasks
<%=node %>	<% for (String t : failedTasks) { %> <%=t %>, <% } %>

<% DefaultJobHistoryParser.KilledOnNodesFilter killedFilter = new DefaultJobHistoryParser.KilledOnNodesFilter(); JobHistory.parseHistoryFromFS(logFile, filter, fs); badNodes = killedFilter.getValues(); if (badNodes.size() > 0) { %>

Killed tasks attempts by nodes

<% for (Map.Entry> entry : badNodes.entrySet()) { String node = entry.getKey(); Set killedTasks = entry.getValue(); %> <% } } %>

Hostname	Killed Tasks
<%=node %>	<% for (String t : killedTasks) { %> <%=t %>, <% } %>