1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25 package org.apache.hadoop.chukwa.inputtools.log4j;
26
27
28 import java.io.File;
29 import java.io.FilenameFilter;
30 import java.io.IOException;
31 import java.text.SimpleDateFormat;
32 import java.util.ArrayList;
33 import java.util.Calendar;
34 import java.util.Collections;
35 import java.util.Date;
36 import java.util.GregorianCalendar;
37 import java.util.List;
38 import java.util.Locale;
39 import java.util.TimeZone;
40 import java.util.regex.Pattern;
41
42 import org.apache.hadoop.chukwa.datacollection.controller.ChukwaAgentController;
43 import org.apache.hadoop.chukwa.datacollection.controller.ClientFinalizer;
44 import org.apache.hadoop.chukwa.util.AdaptorNamingUtils;
45 import org.apache.hadoop.chukwa.util.RecordConstants;
46 import org.apache.log4j.FileAppender;
47 import org.apache.log4j.Layout;
48 import org.apache.log4j.Logger;
49 import org.apache.log4j.helpers.LogLog;
50 import org.apache.log4j.spi.LoggingEvent;
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157 public class ChukwaDailyRollingFileAppender extends FileAppender {
158
159 static Logger log = Logger.getLogger(ChukwaDailyRollingFileAppender.class);
160
161
162 static final int TOP_OF_TROUBLE = -1;
163 static final int TOP_OF_MINUTE = 0;
164 static final int TOP_OF_HOUR = 1;
165 static final int HALF_DAY = 2;
166 static final int TOP_OF_DAY = 3;
167 static final int TOP_OF_WEEK = 4;
168 static final int TOP_OF_MONTH = 5;
169
170 static final String adaptorType = ChukwaAgentController.CharFileTailUTF8NewLineEscaped;
171
172 static final Object lock = new Object();
173 static String lastRotation = "";
174
175
176
177
178
179 private String datePattern = "'.'yyyy-MM-dd";
180
181
182
183
184
185
186
187
188
189 private String scheduledFilename;
190
191
192
193
194 private long nextCheck = System.currentTimeMillis() - 1;
195
196
197
198
199 private String cleanUpRegex = null;
200
201
202
203
204 private int maxBackupIndex = 10;
205
206 private ClientFinalizer clientFinalizer = null;
207
208 boolean hasBeenActivated = false;
209 Date now = new Date();
210
211 SimpleDateFormat sdf;
212
213 RollingCalendar rc = new RollingCalendar();
214
215 int checkPeriod = TOP_OF_TROUBLE;
216
217 ChukwaAgentController chukwaClient;
218 boolean chukwaClientIsNull = true;
219 static final Object chukwaLock = new Object();
220
221 String chukwaClientHostname;
222 int chukwaClientPortNum;
223 long chukwaClientConnectNumRetry;
224 long chukwaClientConnectRetryInterval;
225
226 String recordType;
227
228
229 static final TimeZone gmtTimeZone = TimeZone.getTimeZone("GMT");
230
231
232
233
234 public ChukwaDailyRollingFileAppender() throws IOException {
235 super();
236 }
237
238
239
240
241
242
243
244 public ChukwaDailyRollingFileAppender(Layout layout, String filename,
245 String datePattern) throws IOException {
246 super(layout, filename, true);
247 System.out
248 .println("Daily Rolling File Appender successfully registered file with agent: "
249 + filename);
250 this.datePattern = datePattern;
251 }
252
253
254
255
256
257 public void setDatePattern(String pattern) {
258 datePattern = pattern;
259 }
260
261
262 public String getDatePattern() {
263 return datePattern;
264 }
265
266 public String getRecordType() {
267 if (recordType != null)
268 return recordType;
269 else
270 return "unknown";
271 }
272
273 public void setRecordType(String recordType) {
274 this.recordType = recordType;
275 }
276
277 public void activateOptions() {
278
279
280 if (!hasBeenActivated)
281 { return;}
282
283 super.activateOptions();
284 if (datePattern != null && fileName != null) {
285 now.setTime(System.currentTimeMillis());
286 sdf = new SimpleDateFormat(datePattern);
287 int type = computeCheckPeriod();
288 printPeriodicity(type);
289 rc.setType(type);
290 File file = new File(fileName);
291 scheduledFilename = fileName + sdf.format(new Date(file.lastModified()));
292
293 } else {
294 LogLog
295 .error("Either File or DatePattern options are not set for appender ["
296 + name + "].");
297 }
298 }
299
300 void printPeriodicity(int type) {
301 switch (type) {
302 case TOP_OF_MINUTE:
303 LogLog.debug("Appender [" + name + "] to be rolled every minute.");
304 break;
305 case TOP_OF_HOUR:
306 LogLog
307 .debug("Appender [" + name + "] to be rolled on top of every hour.");
308 break;
309 case HALF_DAY:
310 LogLog.debug("Appender [" + name
311 + "] to be rolled at midday and midnight.");
312 break;
313 case TOP_OF_DAY:
314 LogLog.debug("Appender [" + name + "] to be rolled at midnight.");
315 break;
316 case TOP_OF_WEEK:
317 LogLog.debug("Appender [" + name + "] to be rolled at start of week.");
318 break;
319 case TOP_OF_MONTH:
320 LogLog.debug("Appender [" + name
321 + "] to be rolled at start of every month.");
322 break;
323 default:
324 LogLog.warn("Unknown periodicity for appender [" + name + "].");
325 }
326 }
327
328
329
330
331
332
333
334
335
336
337 int computeCheckPeriod() {
338 RollingCalendar rollingCalendar = new RollingCalendar(gmtTimeZone,
339 Locale.ENGLISH);
340
341 Date epoch = new Date(0);
342 if (datePattern != null) {
343 for (int i = TOP_OF_MINUTE; i <= TOP_OF_MONTH; i++) {
344 SimpleDateFormat simpleDateFormat = new SimpleDateFormat(datePattern);
345 simpleDateFormat.setTimeZone(gmtTimeZone);
346
347 String r0 = simpleDateFormat.format(epoch);
348 rollingCalendar.setType(i);
349 Date next = new Date(rollingCalendar.getNextCheckMillis(epoch));
350 String r1 = simpleDateFormat.format(next);
351
352 if (r0 != null && r1 != null && !r0.equals(r1)) {
353 return i;
354 }
355 }
356 }
357 return TOP_OF_TROUBLE;
358 }
359
360
361
362
363 void rollOver() throws IOException {
364
365
366 if (datePattern == null) {
367 errorHandler.error("Missing DatePattern option in rollOver().");
368 return;
369 }
370
371 String datedFilename = fileName + sdf.format(now);
372
373
374
375 if (scheduledFilename.equals(datedFilename)) {
376 return;
377 }
378
379
380 this.closeFile();
381
382 File target = new File(scheduledFilename);
383 if (target.exists()) {
384 target.delete();
385 }
386
387 File file = new File(fileName);
388
389 boolean result = file.renameTo(target);
390 if (result) {
391 LogLog.debug(fileName + " -> " + scheduledFilename);
392 } else {
393 LogLog.error("Failed to rename [" + fileName + "] to ["
394 + scheduledFilename + "].");
395 }
396
397 try {
398
399
400 this.setFile(fileName, false, this.bufferedIO, this.bufferSize);
401 } catch (IOException e) {
402 errorHandler.error("setFile(" + fileName + ", false) call failed.");
403 }
404 scheduledFilename = datedFilename;
405 cleanUp();
406 }
407
408 public String getCleanUpRegex() {
409 return cleanUpRegex;
410 }
411
412 public void setCleanUpRegex(String cleanUpRegex) {
413 this.cleanUpRegex = cleanUpRegex;
414 }
415
416 public int getMaxBackupIndex() {
417 return maxBackupIndex;
418 }
419
420 public void setMaxBackupIndex(int maxBackupIndex) {
421 this.maxBackupIndex = maxBackupIndex;
422 }
423
424 protected synchronized void cleanUp() {
425 String regex = "";
426 try {
427 File actualFile = new File(fileName);
428
429 String directoryName = actualFile.getParent();
430 String actualFileName = actualFile.getName();
431 File dirList = new File(directoryName);
432
433 if (cleanUpRegex == null || !cleanUpRegex.contains("$fileName")) {
434 LogLog
435 .error("cleanUpRegex == null || !cleanUpRegex.contains(\"$fileName\")");
436 return;
437 }
438 regex = cleanUpRegex.replace("$fileName", actualFileName);
439 String[] dirFiles = dirList.list(new LogFilter(actualFileName, regex));
440
441 List<String> files = new ArrayList<String>();
442 for (String file : dirFiles) {
443 files.add(file);
444 }
445 Collections.sort(files);
446
447 while (files.size() > maxBackupIndex) {
448 String file = files.remove(0);
449 File f = new File(directoryName + "/" + file);
450 f.delete();
451 LogLog.debug("Removing: " + file);
452 }
453 } catch (Exception e) {
454 errorHandler
455 .error("cleanUp(" + fileName + "," + regex + ") call failed.");
456 }
457 }
458
459 private class LogFilter implements FilenameFilter {
460 private Pattern p = null;
461 private String logFile = null;
462
463 public LogFilter(String logFile, String regex) {
464 this.logFile = logFile;
465 p = Pattern.compile(regex);
466 }
467
468 @Override
469 public boolean accept(File dir, String name) {
470
471 if (name.intern() == this.logFile.intern()) {
472 return false;
473 }
474
475 if (!name.startsWith(logFile)) {
476 return false;
477 }
478 return p.matcher(name).find();
479 }
480 }
481
482
483
484
485 @Override
486 protected boolean checkEntryConditions() {
487 if (!hasBeenActivated) {
488 synchronized(chukwaLock) {
489 if (!hasBeenActivated) {
490 hasBeenActivated = true;
491 activateOptions();
492 }
493 }
494 }
495 return super.checkEntryConditions();
496 }
497
498
499
500
501
502
503
504
505 protected void subAppend(LoggingEvent event) {
506 try {
507
508
509
510
511
512
513
514 if (chukwaClientIsNull) {
515 synchronized (chukwaLock) {
516
517 String log4jFileName = getFile();
518 String recordType = getRecordType();
519
520 long currentLength = 0L;
521 try {
522 File fooLog = new File(log4jFileName);
523 log4jFileName = fooLog.getAbsolutePath();
524 currentLength = fooLog.length();
525 } catch (Throwable e) {
526 log.warn("Exception while trying to get current file size for " + log4jFileName);
527 currentLength = 0L;
528 }
529
530 if (chukwaClient == null) {
531 if (getChukwaClientHostname() != null
532 && getChukwaClientPortNum() != 0) {
533 chukwaClient = new ChukwaAgentController(
534 getChukwaClientHostname(), getChukwaClientPortNum());
535 log.debug("setup adaptor with hostname "
536 + getChukwaClientHostname() + " and portnum "
537 + getChukwaClientPortNum());
538 } else {
539 chukwaClient = new ChukwaAgentController();
540 log
541 .debug("setup adaptor with no args, which means it used its defaults");
542 }
543
544 chukwaClientIsNull = false;
545
546
547
548
549
550
551
552
553 long retryInterval = chukwaClientConnectRetryInterval;
554 if (retryInterval == 0) {
555 retryInterval = 1000 * 60 * 30;
556 }
557 long numRetries = chukwaClientConnectNumRetry;
558 if (numRetries == 0) {
559 numRetries = 48;
560 }
561
562 String name = AdaptorNamingUtils.synthesizeAdaptorID
563 (ChukwaAgentController.CharFileTailUTF8NewLineEscaped, recordType, log4jFileName);
564
565 String adaptorID = chukwaClient.addByName(name, ChukwaAgentController.CharFileTailUTF8NewLineEscaped,
566 recordType,currentLength + " " + log4jFileName, currentLength,
567 numRetries, retryInterval);
568
569
570 clientFinalizer = new ClientFinalizer(chukwaClient);
571 Runtime.getRuntime().addShutdownHook(clientFinalizer);
572
573 if (adaptorID != null) {
574 log.debug("Added file tailing adaptor to chukwa agent for file "
575 + log4jFileName + ", adaptorId:" + adaptorID
576 + " using this recordType :" + recordType
577 + ", starting at offset:" + currentLength);
578 } else {
579 log.debug("Chukwa adaptor not added, addFile(" + log4jFileName
580 + ") returned " + adaptorID
581 + ", current offset:" + currentLength);
582 }
583
584 }
585 }
586 }
587
588 long n = System.currentTimeMillis();
589 if (n >= nextCheck) {
590 now.setTime(n);
591 nextCheck = rc.getNextCheckMillis(now);
592 try {
593 rollOver();
594 } catch (IOException ioe) {
595 LogLog.error("rollOver() failed.", ioe);
596 }
597 }
598
599 boolean written = false;
600 if(layout.ignoresThrowable()) {
601 String[] s = event.getThrowableStrRep();
602 if (s != null) {
603 int len = s.length;
604 StringBuilder sb = new StringBuilder();
605 sb.append(this.layout.format(event));
606 for(int i = 0; i < len; i++) {
607 sb.append(s[i]).append("\n");
608 }
609
610 written = true;
611 this.qw.write(RecordConstants.escapeAllButLastRecordSeparator("\n",sb.toString()));
612 }
613 }
614
615 if (!written) {
616
617 this.qw.write(RecordConstants.escapeAllButLastRecordSeparator("\n",this.layout.format(event)));
618 }
619
620 if (this.immediateFlush) {
621 this.qw.flush();
622 }
623 } catch (Throwable e) {
624 System.err.println("Exception in ChukwaRollingAppender: "
625 + e.getMessage());
626 e.printStackTrace();
627 }
628
629 }
630
631 public String getChukwaClientHostname() {
632 return chukwaClientHostname;
633 }
634
635 public void setChukwaClientHostname(String chukwaClientHostname) {
636 this.chukwaClientHostname = chukwaClientHostname;
637 }
638
639 public int getChukwaClientPortNum() {
640 return chukwaClientPortNum;
641 }
642
643 public void setChukwaClientPortNum(int chukwaClientPortNum) {
644 this.chukwaClientPortNum = chukwaClientPortNum;
645 }
646
647 public void setChukwaClientConnectNumRetry(int i) {
648 this.chukwaClientConnectNumRetry = i;
649 }
650
651 public void setChukwaClientConnectRetryInterval(long i) {
652 this.chukwaClientConnectRetryInterval = i;
653 }
654
655 }
656
657
658
659
660
661
662
663 class RollingCalendar extends GregorianCalendar {
664
665
666
667
668 private static final long serialVersionUID = 2153481574198792767L;
669 int type = ChukwaDailyRollingFileAppender.TOP_OF_TROUBLE;
670
671 RollingCalendar() {
672 super();
673 }
674
675 RollingCalendar(TimeZone tz, Locale locale) {
676 super(tz, locale);
677 }
678
679 void setType(int type) {
680 this.type = type;
681 }
682
683 public long getNextCheckMillis(Date now) {
684 return getNextCheckDate(now).getTime();
685 }
686
687 public Date getNextCheckDate(Date now) {
688 this.setTime(now);
689
690 switch (type) {
691 case ChukwaDailyRollingFileAppender.TOP_OF_MINUTE:
692 this.set(Calendar.SECOND, 0);
693 this.set(Calendar.MILLISECOND, 0);
694 this.add(Calendar.MINUTE, 1);
695 break;
696 case ChukwaDailyRollingFileAppender.TOP_OF_HOUR:
697 this.set(Calendar.MINUTE, 0);
698 this.set(Calendar.SECOND, 0);
699 this.set(Calendar.MILLISECOND, 0);
700 this.add(Calendar.HOUR_OF_DAY, 1);
701 break;
702 case ChukwaDailyRollingFileAppender.HALF_DAY:
703 this.set(Calendar.MINUTE, 0);
704 this.set(Calendar.SECOND, 0);
705 this.set(Calendar.MILLISECOND, 0);
706 int hour = get(Calendar.HOUR_OF_DAY);
707 if (hour < 12) {
708 this.set(Calendar.HOUR_OF_DAY, 12);
709 } else {
710 this.set(Calendar.HOUR_OF_DAY, 0);
711 this.add(Calendar.DAY_OF_MONTH, 1);
712 }
713 break;
714 case ChukwaDailyRollingFileAppender.TOP_OF_DAY:
715 this.set(Calendar.HOUR_OF_DAY, 0);
716 this.set(Calendar.MINUTE, 0);
717 this.set(Calendar.SECOND, 0);
718 this.set(Calendar.MILLISECOND, 0);
719 this.add(Calendar.DATE, 1);
720 break;
721 case ChukwaDailyRollingFileAppender.TOP_OF_WEEK:
722 this.set(Calendar.DAY_OF_WEEK, getFirstDayOfWeek());
723 this.set(Calendar.HOUR_OF_DAY, 0);
724 this.set(Calendar.SECOND, 0);
725 this.set(Calendar.MILLISECOND, 0);
726 this.add(Calendar.WEEK_OF_YEAR, 1);
727 break;
728 case ChukwaDailyRollingFileAppender.TOP_OF_MONTH:
729 this.set(Calendar.DATE, 1);
730 this.set(Calendar.HOUR_OF_DAY, 0);
731 this.set(Calendar.SECOND, 0);
732 this.set(Calendar.MILLISECOND, 0);
733 this.add(Calendar.MONTH, 1);
734 break;
735 default:
736 throw new IllegalStateException("Unknown periodicity type.");
737 }
738 return getTime();
739 }
740 }