<%! /** * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ %> <%@ page import="javax.servlet.*" import="javax.servlet.http.*" import="java.io.*" import="java.text.*" import="java.util.*" import="java.text.DecimalFormat" import="org.apache.hadoop.http.HtmlQuoting" import="org.apache.hadoop.mapred.*" import="org.apache.hadoop.mapreduce.*" import="org.apache.hadoop.util.*" import="org.codehaus.jackson.map.ObjectMapper" %> <%!static SimpleDateFormat dateFormat = new SimpleDateFormat( "d-MMM-yyyy HH:mm:ss"); %> <%! private static final long serialVersionUID = 1L; %> <%! private static DecimalFormat percentFormat = new DecimalFormat("##0.00"); public void generateSummaryTable(JspWriter out, ClusterMetrics metrics, JobTracker tracker) throws IOException { String tasksPerNode = metrics.getTaskTrackerCount() > 0 ? percentFormat.format(((double)(metrics.getMapSlotCapacity() + metrics.getReduceSlotCapacity())) / metrics.getTaskTrackerCount()): "-"; out.print("\n"+ "" + "" + "" + "" + "" + "" + "" + "" + "" + "\n"); out.print("
Running Map TasksRunning Reduce TasksTotal SubmissionsNodesOccupied Map SlotsOccupied Reduce SlotsReserved Map SlotsReserved Reduce SlotsMap Task CapacityReduce Task CapacityAvg. Tasks/NodeBlacklisted NodesGraylisted NodesExcluded Nodes
" + metrics.getRunningMaps() + "" + metrics.getRunningReduces() + "" + metrics.getTotalJobSubmissions() + "" + metrics.getTaskTrackerCount() + "" + metrics.getOccupiedMapSlots() + "" + metrics.getOccupiedReduceSlots() + "" + metrics.getReservedMapSlots() + "" + metrics.getReservedReduceSlots() + "" + metrics.getMapSlotCapacity() + "" + metrics.getReduceSlotCapacity() + "" + tasksPerNode + "" + metrics.getBlackListedTaskTrackerCount() + "" + "" + metrics.getGrayListedTaskTrackerCount() + "" + "" + metrics.getDecommissionedTaskTrackerCount() + "" + "
\n"); out.print("
"); if (tracker.hasRestarted()) { out.print(""); if (tracker.hasRecovered()) { out.print("The JobTracker got restarted and recovered back in " ); out.print(StringUtils.formatTime(tracker.getRecoveryDuration())); } else { out.print("The JobTracker got restarted and is still recovering"); } out.print(""); } } %> <%! public static class ErrorResponse { private final long errorCode; private final String errorDescription; // Constructor ErrorResponse(long ec, String ed) { errorCode = ec; errorDescription = ed; } // Getters public long getErrorCode() { return errorCode; } public String getErrorDescription() { return errorDescription; } } public static class JobTrackerResponse { public static class JobTrackerMetaInfo { private final String jobTrackerName; private final String status; private final String startTimestamp; private final String version; private final String compilationInfo; private final String identifier; private final String safeModeStatus; private final boolean hasRestarted; private final boolean hasRecovered; private final long recoveryDurationSecs; // Constructor JobTrackerMetaInfo(JobTracker jt) { jobTrackerName = StringUtils.simpleHostname(jt.getJobTrackerMachine()); status = jt.getClusterStatus().getJobTrackerState().toString(); startTimestamp = dateFormat.format(new Date(jt.getStartTime())); version = VersionInfo.getVersion() + ", revision " + VersionInfo.getRevision(); compilationInfo = VersionInfo.getDate() + " by " + VersionInfo.getUser(); identifier = jt.getTrackerIdentifier(); safeModeStatus = jt.getSafeModeText(); hasRestarted = jt.hasRestarted(); hasRecovered = jt.hasRecovered(); if (hasRestarted && hasRecovered) { recoveryDurationSecs = jt.getRecoveryDuration() / 1000; } else { recoveryDurationSecs = 0; } } // Getters public String getJobTrackerName() { return jobTrackerName; } public String getStatus() { return status; } public String getStartTimestamp() { return startTimestamp; } public String getVersion() { return version; } public String getCompilationInfo() { return compilationInfo; } public String getIdentifier() { return identifier; } public String getSafeModeStatus() { return safeModeStatus; } public boolean getHasRestarted() { return hasRestarted; } public boolean getHasRecovered() { return hasRecovered; } public long getRecoveryDurationSecs() { return recoveryDurationSecs; } } public static class JobTrackerClusterSummary { private final long usedHeapMemoryBytes; private final long totalHeapMemoryBytes; private final long numTotalTaskTrackers; private final long numBlackListedTaskTrackers; private final long numGrayListedTaskTrackers; private final long numDecommissionedTaskTrackers; private final long runningMapTasks; private final long runningReduceTasks; private final long totalJobSubmissions; private final long occupiedMapSlots; private final long occupiedReduceSlots; private final long reservedMapSlots; private final long reservedReduceSlots; private final long mapTaskCapacity; private final long reduceTaskCapacity; private final float avgTasksPerTaskTracker; // Constructor JobTrackerClusterSummary(JobTracker jt) { usedHeapMemoryBytes = Runtime.getRuntime().totalMemory(); totalHeapMemoryBytes = Runtime.getRuntime().maxMemory(); ClusterMetrics metrics = jt.getClusterMetrics(); numTotalTaskTrackers = metrics.getTaskTrackerCount(); numBlackListedTaskTrackers = metrics.getBlackListedTaskTrackerCount(); numGrayListedTaskTrackers = metrics.getGrayListedTaskTrackerCount(); numDecommissionedTaskTrackers = metrics.getDecommissionedTaskTrackerCount(); runningMapTasks = metrics.getRunningMaps(); runningReduceTasks = metrics.getRunningReduces(); totalJobSubmissions = metrics.getTotalJobSubmissions(); occupiedMapSlots = metrics.getOccupiedMapSlots(); occupiedReduceSlots = metrics.getOccupiedReduceSlots(); reservedMapSlots = metrics.getReservedMapSlots(); reservedReduceSlots = metrics.getReservedReduceSlots(); mapTaskCapacity = metrics.getMapSlotCapacity(); reduceTaskCapacity = metrics.getReduceSlotCapacity(); avgTasksPerTaskTracker = (numTotalTaskTrackers > 0) ? (float)(((double)(mapTaskCapacity + reduceTaskCapacity)) / numTotalTaskTrackers) : 0; } // Getters public long getUsedHeapMemoryBytes() { return usedHeapMemoryBytes; } public long getTotalHeapMemoryBytes() { return totalHeapMemoryBytes; } public long getNumTotalTaskTrackers() { return numTotalTaskTrackers; } public long getNumBlackListedTaskTrackers() { return numBlackListedTaskTrackers; } public long getNumGrayListedTaskTrackers() { return numGrayListedTaskTrackers; } public long getNumDecommissionedTaskTrackers() { return numDecommissionedTaskTrackers; } public long getRunningMapTasks() { return runningMapTasks; } public long getRunningReduceTasks() { return runningReduceTasks; } public long getTotalJobSubmissions() { return totalJobSubmissions; } public long getOccupiedMapSlots() { return occupiedMapSlots; } public long getOccupiedReduceSlots() { return occupiedReduceSlots; } public long getReservedMapSlots() { return reservedMapSlots; } public long getReservedReduceSlots() { return reservedReduceSlots; } public long getMapTaskCapacity() { return mapTaskCapacity; } public long getReduceTaskCapacity() { return reduceTaskCapacity; } public float getAvgTasksPerTaskTracker(){ return avgTasksPerTaskTracker; } } public static class JobSummaryInfo { public static class JobTaskStats { private final int numCompleted; private final int numTotal; private final float completionPercentage; // Constructor JobTaskStats(int nc, int nt, float cp) { numCompleted = nc; numTotal = nt; completionPercentage = cp; } // Getters public int getNumCompleted() { return numCompleted; } public int getNumTotal() { return numTotal; } public float getCompletionPercentage() { return completionPercentage; } } private final String jobId; private final String jobName; private final String userName; private final String jobPriority; private final JobTaskStats mapStats; private final JobTaskStats reduceStats; private final String jobSchedulingInfo; // Constructor JobSummaryInfo(JobInProgress jip) { JobProfile jobProfile = jip.getProfile(); jobId = jobProfile.getJobID().toString(); jobName = jobProfile.getJobName(); userName = jobProfile.getUser(); jobPriority = jip.getPriority().toString(); JobStatus jobStatus = jip.getStatus(); mapStats = new JobTaskStats(jip.finishedMaps(), jip.desiredMaps(), jobStatus.mapProgress() * 100.0f); reduceStats = new JobTaskStats(jip.finishedReduces(), jip.desiredReduces(), jobStatus.reduceProgress() * 100.0f); jobSchedulingInfo = jip.getStatus().getSchedulingInfo(); } // Getters public String getJobId() { return jobId; } public String getJobName() { return jobName; } public String getUserName() { return userName; } public String getJobPriority() { return jobPriority; } public JobTaskStats getMapStats() { return mapStats; } public JobTaskStats getReduceStats() { return reduceStats; } public String getJobSchedulingInfo() { return jobSchedulingInfo; } } private final JobTrackerMetaInfo metaInfo; private final JobTrackerClusterSummary clusterSummary; private final Collection runningJobsSummaryInfo; private final Collection completedJobsSummaryInfo; private final Collection failedJobsSummaryInfo; private void populateJobsSummaryInfo (Collection jips, Collection jsis) { for (JobInProgress jip : jips) { jsis.add(new JobSummaryInfo(jip)); } } // Constructor JobTrackerResponse(JobTracker jt) { metaInfo = new JobTrackerMetaInfo(jt); clusterSummary = new JobTrackerClusterSummary(jt); Collection runningJobs = jt.runningJobs(); runningJobsSummaryInfo = (runningJobs.size() > 0) ? new ArrayList() : null; populateJobsSummaryInfo(runningJobs, runningJobsSummaryInfo); Collection completedJobs = jt.completedJobs(); completedJobsSummaryInfo = (completedJobs.size() > 0) ? new ArrayList() : null; populateJobsSummaryInfo(completedJobs, completedJobsSummaryInfo); Collection failedJobs = jt.failedJobs(); failedJobsSummaryInfo = (failedJobs.size() > 0) ? new ArrayList() : null; populateJobsSummaryInfo(failedJobs, failedJobsSummaryInfo); } // Getters public JobTrackerMetaInfo getMetaInfo() { return metaInfo; } public JobTrackerClusterSummary getClusterSummary() { return clusterSummary; } public Collection getRunningJobsSummaryInfo() { return runningJobsSummaryInfo; } public Collection getCompletedJobsSummaryInfo() { return completedJobsSummaryInfo; } public Collection getFailedJobsSummaryInfo() { return failedJobsSummaryInfo; } } %> <% String response_format = request.getParameter("format"); if (response_format != null) { /* Eventually, the HTML output should also be driven off of these *Response * objects. * * Someday. */ JobTrackerResponse theJobTrackerResponse = null; ErrorResponse theErrorResponse = null; JobTracker tracker = (JobTracker) application.getAttribute("job.tracker"); theJobTrackerResponse = new JobTrackerResponse(tracker); /* ------------ Response generation begins here ------------ */ /* For now, "json" is the only supported format. * * As more formats are supported, this should become a cascading * if-elsif-else block. */ if ("json".equals(response_format)) { response.setContentType("application/json"); ObjectMapper responseObjectMapper = new ObjectMapper(); /* A lack of an error response implies we have a meaningful * application response? Why not! */ out.println(responseObjectMapper.writeValueAsString ((theErrorResponse == null) ? theJobTrackerResponse : theErrorResponse)); } else { response.setStatus(HttpServletResponse.SC_NOT_IMPLEMENTED); } } else { %> <% // Spit out HTML only in the absence of the "format" query parameter. response.setContentType("text/html; charset=UTF-8"); JobTracker tracker = (JobTracker) application.getAttribute("job.tracker"); ClusterStatus status = tracker.getClusterStatus(); ClusterMetrics metrics = tracker.getClusterMetrics(); String trackerName = StringUtils.simpleHostname(tracker.getJobTrackerMachine()); JobQueueInfo[] queues = tracker.getQueues(); Vector runningJobs = tracker.runningJobs(); Vector completedJobs = tracker.completedJobs(); Vector failedJobs = tracker.failedJobs(); %> <%= trackerName %> Hadoop Map/Reduce Administration <% JSPUtil.processButtons(request, response, tracker); %>

<%= trackerName %> Hadoop Map/Reduce Administration

State: <%= status.getJobTrackerState() %>
Started: <%= new Date(tracker.getStartTime())%>
Version: <%= VersionInfo.getVersion()%>, r<%= VersionInfo.getRevision()%>
Compiled: <%= VersionInfo.getDate()%> by <%= VersionInfo.getUser()%>
Identifier: <%= tracker.getTrackerIdentifier()%>
SafeMode: <%= tracker.getSafeModeText()%>

Cluster Summary (Heap Size is <%= StringUtils.byteDesc(Runtime.getRuntime().totalMemory()) %>/<%= StringUtils.byteDesc(Runtime.getRuntime().maxMemory()) %>)

<% generateSummaryTable(out, metrics, tracker); %>

Scheduling Information

<% for(JobQueueInfo queue: queues) { String queueName = queue.getQueueName(); String state = queue.getQueueState(); String schedulingInformation = queue.getSchedulingInfo(); if(schedulingInformation == null || schedulingInformation.trim().equals("")) { schedulingInformation = "NA"; } %> <% } %>
Queue Name State Scheduling Information
<%=queueName%> <%=state%> <%=HtmlQuoting.quoteHtmlChars(schedulingInformation).replaceAll("\n","
") %>
Filter (Jobid, Priority, User, Name)
Example: 'user:smith 3200' will filter by 'smith' only in the user field and '3200' in all fields

Running Jobs

<%=JSPUtil.generateJobTable("Running", runningJobs, 30, 0, tracker.conf)%>
<% if (completedJobs.size() > 0) { out.print("

Completed Jobs

"); out.print(JSPUtil.generateJobTable("Completed", completedJobs, 0, runningJobs.size(), tracker.conf)); out.print("
"); } %> <% if (failedJobs.size() > 0) { out.print("

Failed Jobs

"); out.print(JSPUtil.generateJobTable("Failed", failedJobs, 0, (runningJobs.size()+completedJobs.size()), tracker.conf)); out.print("
"); } %>

Retired Jobs

<%=JSPUtil.generateRetiredJobTable(tracker, (runningJobs.size()+completedJobs.size()+failedJobs.size()))%>

Local Logs

Log directory, Job Tracker History <% out.println(ServletUtil.htmlFooter()); %> <% } // if (response_format != null) %>