1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.logging.log4j.core.appender;
18
19 import java.io.File;
20 import java.io.IOException;
21 import java.io.OutputStream;
22 import java.io.RandomAccessFile;
23 import java.io.Serializable;
24 import java.lang.reflect.Method;
25 import java.nio.ByteOrder;
26 import java.nio.MappedByteBuffer;
27 import java.nio.channels.FileChannel;
28 import java.security.AccessController;
29 import java.security.PrivilegedActionException;
30 import java.security.PrivilegedExceptionAction;
31 import java.util.HashMap;
32 import java.util.Map;
33 import java.util.Objects;
34
35 import org.apache.logging.log4j.core.Layout;
36 import org.apache.logging.log4j.core.util.Closer;
37 import org.apache.logging.log4j.core.util.NullOutputStream;
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57 public class MemoryMappedFileManager extends OutputStreamManager {
58
59
60
61 static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
62 private static final int MAX_REMAP_COUNT = 10;
63 private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
64 private static final double NANOS_PER_MILLISEC = 1000.0 * 1000.0;
65
66 private final boolean isForce;
67 private final int regionLength;
68 private final String advertiseURI;
69 private final RandomAccessFile randomAccessFile;
70 private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<>();
71 private MappedByteBuffer mappedBuffer;
72 private long mappingOffset;
73
74 protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
75 final boolean force, final long position, final int regionLength, final String advertiseURI,
76 final Layout<? extends Serializable> layout, final boolean writeHeader) throws IOException {
77 super(os, fileName, layout, writeHeader);
78 this.isForce = force;
79 this.randomAccessFile = Objects.requireNonNull(file, "RandomAccessFile");
80 this.regionLength = regionLength;
81 this.advertiseURI = advertiseURI;
82 this.isEndOfBatch.set(Boolean.FALSE);
83 this.mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), position, regionLength);
84 this.mappingOffset = position;
85 }
86
87
88
89
90
91
92
93
94
95
96
97
98 public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
99 final boolean isForce, final int regionLength, final String advertiseURI,
100 final Layout<? extends Serializable> layout) {
101 return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength,
102 advertiseURI, layout), FACTORY);
103 }
104
105 public Boolean isEndOfBatch() {
106 return isEndOfBatch.get();
107 }
108
109 public void setEndOfBatch(final boolean endOfBatch) {
110 this.isEndOfBatch.set(Boolean.valueOf(endOfBatch));
111 }
112
113 @Override
114 protected synchronized void write(final byte[] bytes, int offset, int length) {
115 super.write(bytes, offset, length);
116
117 while (length > mappedBuffer.remaining()) {
118 final int chunk = mappedBuffer.remaining();
119 mappedBuffer.put(bytes, offset, chunk);
120 offset += chunk;
121 length -= chunk;
122 remap();
123 }
124 mappedBuffer.put(bytes, offset, length);
125
126
127
128 }
129
130 private synchronized void remap() {
131 final long offset = this.mappingOffset + mappedBuffer.position();
132 final int length = mappedBuffer.remaining() + regionLength;
133 try {
134 unsafeUnmap(mappedBuffer);
135 final long fileLength = randomAccessFile.length() + regionLength;
136 LOGGER.debug("{} {} extending {} by {} bytes to {}", getClass().getSimpleName(), getName(), getFileName(),
137 regionLength, fileLength);
138
139 final long startNanos = System.nanoTime();
140 randomAccessFile.setLength(fileLength);
141 final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
142 LOGGER.debug("{} {} extended {} OK in {} millis", getClass().getSimpleName(), getName(), getFileName(),
143 millis);
144
145 mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), offset, length);
146 mappingOffset = offset;
147 } catch (final Exception ex) {
148 logError("unable to remap", ex);
149 }
150 }
151
152 @Override
153 public synchronized void flush() {
154 mappedBuffer.force();
155 }
156
157 @Override
158 public synchronized void close() {
159 final long position = mappedBuffer.position();
160 final long length = mappingOffset + position;
161 try {
162 unsafeUnmap(mappedBuffer);
163 } catch (final Exception ex) {
164 logError("unable to unmap MappedBuffer", ex);
165 }
166 try {
167 LOGGER.debug("MMapAppender closing. Setting {} length to {} (offset {} + position {})", getFileName(),
168 length, mappingOffset, position);
169 randomAccessFile.setLength(length);
170 randomAccessFile.close();
171 } catch (final IOException ex) {
172 logError("unable to close MemoryMappedFile", ex);
173 }
174 }
175
176 public static MappedByteBuffer mmap(final FileChannel fileChannel, final String fileName, final long start,
177 final int size) throws IOException {
178 for (int i = 1;; i++) {
179 try {
180 LOGGER.debug("MMapAppender remapping {} start={}, size={}", fileName, start, size);
181
182 final long startNanos = System.nanoTime();
183 final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
184 map.order(ByteOrder.nativeOrder());
185
186 final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
187 LOGGER.debug("MMapAppender remapped {} OK in {} millis", fileName, millis);
188
189 return map;
190 } catch (final IOException e) {
191 if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) {
192 throw e;
193 }
194 LOGGER.debug("Remap attempt {}/{} failed. Retrying...", i, MAX_REMAP_COUNT, e);
195 if (i < MAX_REMAP_COUNT) {
196 Thread.yield();
197 } else {
198 try {
199 Thread.sleep(1);
200 } catch (final InterruptedException ignored) {
201 Thread.currentThread().interrupt();
202 throw e;
203 }
204 }
205 }
206 }
207 }
208
209 private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException {
210 LOGGER.debug("MMapAppender unmapping old buffer...");
211 final long startNanos = System.nanoTime();
212 AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {
213 @Override
214 public Object run() throws Exception {
215 final Method getCleanerMethod = mbb.getClass().getMethod("cleaner");
216 getCleanerMethod.setAccessible(true);
217 final Object cleaner = getCleanerMethod.invoke(mbb);
218 final Method cleanMethod = cleaner.getClass().getMethod("clean");
219 cleanMethod.invoke(cleaner);
220 return null;
221 }
222 });
223 final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
224 LOGGER.debug("MMapAppender unmapped buffer OK in {} millis", millis);
225 }
226
227
228
229
230
231
232 public String getFileName() {
233 return getName();
234 }
235
236
237
238
239
240
241 public int getRegionLength() {
242 return regionLength;
243 }
244
245
246
247
248
249
250
251 public boolean isImmediateFlush() {
252 return isForce;
253 }
254
255
256
257
258
259
260
261
262
263 @Override
264 public Map<String, String> getContentFormat() {
265 final Map<String, String> result = new HashMap<>(super.getContentFormat());
266 result.put("fileURI", advertiseURI);
267 return result;
268 }
269
270
271
272
273 private static class FactoryData {
274 private final boolean append;
275 private final boolean force;
276 private final int regionLength;
277 private final String advertiseURI;
278 private final Layout<? extends Serializable> layout;
279
280
281
282
283
284
285
286
287 public FactoryData(final boolean append, final boolean force, final int regionLength,
288 final String advertiseURI, final Layout<? extends Serializable> layout) {
289 this.append = append;
290 this.force = force;
291 this.regionLength = regionLength;
292 this.advertiseURI = advertiseURI;
293 this.layout = layout;
294 }
295 }
296
297
298
299
300 private static class MemoryMappedFileManagerFactory
301 implements ManagerFactory<MemoryMappedFileManager, FactoryData> {
302
303
304
305
306
307
308
309
310 @SuppressWarnings("resource")
311 @Override
312 public MemoryMappedFileManager createManager(final String name, final FactoryData data) {
313 final File file = new File(name);
314 final File parent = file.getParentFile();
315 if (null != parent && !parent.exists()) {
316 parent.mkdirs();
317 }
318 if (!data.append) {
319 file.delete();
320 }
321
322 final boolean writeHeader = !data.append || !file.exists();
323 final OutputStream os = NullOutputStream.NULL_OUTPUT_STREAM;
324 RandomAccessFile raf = null;
325 try {
326 raf = new RandomAccessFile(name, "rw");
327 final long position = (data.append) ? raf.length() : 0;
328 raf.setLength(position + data.regionLength);
329 return new MemoryMappedFileManager(raf, name, os, data.force, position, data.regionLength,
330 data.advertiseURI, data.layout, writeHeader);
331 } catch (final Exception ex) {
332 LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex, ex);
333 Closer.closeSilently(raf);
334 }
335 return null;
336 }
337 }
338 }