1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.logging.log4j.core.appender;
18
19 import java.io.File;
20 import java.io.IOException;
21 import java.io.OutputStream;
22 import java.io.RandomAccessFile;
23 import java.io.Serializable;
24 import java.lang.reflect.Method;
25 import java.nio.ByteBuffer;
26 import java.nio.ByteOrder;
27 import java.nio.MappedByteBuffer;
28 import java.nio.channels.FileChannel;
29 import java.security.AccessController;
30 import java.security.PrivilegedActionException;
31 import java.security.PrivilegedExceptionAction;
32 import java.util.HashMap;
33 import java.util.Map;
34 import java.util.Objects;
35
36 import org.apache.logging.log4j.core.Layout;
37 import org.apache.logging.log4j.core.util.Closer;
38 import org.apache.logging.log4j.core.util.NullOutputStream;
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58 public class MemoryMappedFileManager extends OutputStreamManager {
59
60
61
62 static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
63 private static final int MAX_REMAP_COUNT = 10;
64 private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
65 private static final double NANOS_PER_MILLISEC = 1000.0 * 1000.0;
66
67 private final boolean isForce;
68 private final int regionLength;
69 private final String advertiseURI;
70 private final RandomAccessFile randomAccessFile;
71 private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<>();
72 private MappedByteBuffer mappedBuffer;
73 private long mappingOffset;
74
75 protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
76 final boolean force, final long position, final int regionLength, final String advertiseURI,
77 final Layout<? extends Serializable> layout, final boolean writeHeader) throws IOException {
78 super(os, fileName, layout, writeHeader, ByteBuffer.wrap(new byte[0]));
79 this.isForce = force;
80 this.randomAccessFile = Objects.requireNonNull(file, "RandomAccessFile");
81 this.regionLength = regionLength;
82 this.advertiseURI = advertiseURI;
83 this.isEndOfBatch.set(Boolean.FALSE);
84 this.mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), position, regionLength);
85 this.byteBuffer = mappedBuffer;
86 this.mappingOffset = position;
87 }
88
89
90
91
92
93
94
95
96
97
98
99
100 public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
101 final boolean isForce, final int regionLength, final String advertiseURI,
102 final Layout<? extends Serializable> layout) {
103 return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength,
104 advertiseURI, layout), FACTORY);
105 }
106
107 public Boolean isEndOfBatch() {
108 return isEndOfBatch.get();
109 }
110
111 public void setEndOfBatch(final boolean endOfBatch) {
112 this.isEndOfBatch.set(Boolean.valueOf(endOfBatch));
113 }
114
115 @Override
116 protected synchronized void write(final byte[] bytes, int offset, int length, final boolean immediateFlush) {
117 while (length > mappedBuffer.remaining()) {
118 final int chunk = mappedBuffer.remaining();
119 mappedBuffer.put(bytes, offset, chunk);
120 offset += chunk;
121 length -= chunk;
122 remap();
123 }
124 mappedBuffer.put(bytes, offset, length);
125
126
127
128 }
129
130 private synchronized void remap() {
131 final long offset = this.mappingOffset + mappedBuffer.position();
132 final int length = mappedBuffer.remaining() + regionLength;
133 try {
134 unsafeUnmap(mappedBuffer);
135 final long fileLength = randomAccessFile.length() + regionLength;
136 LOGGER.debug("{} {} extending {} by {} bytes to {}", getClass().getSimpleName(), getName(), getFileName(),
137 regionLength, fileLength);
138
139 final long startNanos = System.nanoTime();
140 randomAccessFile.setLength(fileLength);
141 final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
142 LOGGER.debug("{} {} extended {} OK in {} millis", getClass().getSimpleName(), getName(), getFileName(),
143 millis);
144
145 mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), offset, length);
146 this.byteBuffer = mappedBuffer;
147 mappingOffset = offset;
148 } catch (final Exception ex) {
149 logError("Unable to remap", ex);
150 }
151 }
152
153 @Override
154 public synchronized void flush() {
155 mappedBuffer.force();
156 }
157
158 @Override
159 public synchronized boolean closeOutputStream() {
160 final long position = mappedBuffer.position();
161 final long length = mappingOffset + position;
162 try {
163 unsafeUnmap(mappedBuffer);
164 } catch (final Exception ex) {
165 logError("Unable to unmap MappedBuffer", ex);
166 }
167 try {
168 LOGGER.debug("MMapAppender closing. Setting {} length to {} (offset {} + position {})", getFileName(),
169 length, mappingOffset, position);
170 randomAccessFile.setLength(length);
171 randomAccessFile.close();
172 return true;
173 } catch (final IOException ex) {
174 logError("Unable to close MemoryMappedFile", ex);
175 return false;
176 }
177 }
178
179 public static MappedByteBuffer mmap(final FileChannel fileChannel, final String fileName, final long start,
180 final int size) throws IOException {
181 for (int i = 1;; i++) {
182 try {
183 LOGGER.debug("MMapAppender remapping {} start={}, size={}", fileName, start, size);
184
185 final long startNanos = System.nanoTime();
186 final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
187 map.order(ByteOrder.nativeOrder());
188
189 final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
190 LOGGER.debug("MMapAppender remapped {} OK in {} millis", fileName, millis);
191
192 return map;
193 } catch (final IOException e) {
194 if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) {
195 throw e;
196 }
197 LOGGER.debug("Remap attempt {}/{} failed. Retrying...", i, MAX_REMAP_COUNT, e);
198 if (i < MAX_REMAP_COUNT) {
199 Thread.yield();
200 } else {
201 try {
202 Thread.sleep(1);
203 } catch (final InterruptedException ignored) {
204 Thread.currentThread().interrupt();
205 throw e;
206 }
207 }
208 }
209 }
210 }
211
212 private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException {
213 LOGGER.debug("MMapAppender unmapping old buffer...");
214 final long startNanos = System.nanoTime();
215 AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {
216 @Override
217 public Object run() throws Exception {
218 final Method getCleanerMethod = mbb.getClass().getMethod("cleaner");
219 getCleanerMethod.setAccessible(true);
220 final Object cleaner = getCleanerMethod.invoke(mbb);
221 final Method cleanMethod = cleaner.getClass().getMethod("clean");
222 cleanMethod.invoke(cleaner);
223 return null;
224 }
225 });
226 final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
227 LOGGER.debug("MMapAppender unmapped buffer OK in {} millis", millis);
228 }
229
230
231
232
233
234
235 public String getFileName() {
236 return getName();
237 }
238
239
240
241
242
243
244 public int getRegionLength() {
245 return regionLength;
246 }
247
248
249
250
251
252
253
254 public boolean isImmediateFlush() {
255 return isForce;
256 }
257
258
259
260
261
262
263
264
265
266 @Override
267 public Map<String, String> getContentFormat() {
268 final Map<String, String> result = new HashMap<>(super.getContentFormat());
269 result.put("fileURI", advertiseURI);
270 return result;
271 }
272
273 @Override
274 protected void flushBuffer(final ByteBuffer buffer) {
275
276 }
277
278 @Override
279 public ByteBuffer getByteBuffer() {
280 return mappedBuffer;
281 }
282
283 @Override
284 public ByteBuffer drain(final ByteBuffer buf) {
285 remap();
286 return mappedBuffer;
287 }
288
289
290
291
292 private static class FactoryData {
293 private final boolean append;
294 private final boolean force;
295 private final int regionLength;
296 private final String advertiseURI;
297 private final Layout<? extends Serializable> layout;
298
299
300
301
302
303
304
305
306
307
308 public FactoryData(final boolean append, final boolean force, final int regionLength,
309 final String advertiseURI, final Layout<? extends Serializable> layout) {
310 this.append = append;
311 this.force = force;
312 this.regionLength = regionLength;
313 this.advertiseURI = advertiseURI;
314 this.layout = layout;
315 }
316 }
317
318
319
320
321 private static class MemoryMappedFileManagerFactory
322 implements ManagerFactory<MemoryMappedFileManager, FactoryData> {
323
324
325
326
327
328
329
330
331 @SuppressWarnings("resource")
332 @Override
333 public MemoryMappedFileManager createManager(final String name, final FactoryData data) {
334 final File file = new File(name);
335 final File parent = file.getParentFile();
336 if (null != parent && !parent.exists()) {
337 parent.mkdirs();
338 }
339 if (!data.append) {
340 file.delete();
341 }
342
343 final boolean writeHeader = !data.append || !file.exists();
344 final OutputStream os = NullOutputStream.getInstance();
345 RandomAccessFile raf = null;
346 try {
347 raf = new RandomAccessFile(name, "rw");
348 final long position = (data.append) ? raf.length() : 0;
349 raf.setLength(position + data.regionLength);
350 return new MemoryMappedFileManager(raf, name, os, data.force, position, data.regionLength,
351 data.advertiseURI, data.layout, writeHeader);
352 } catch (final Exception ex) {
353 LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex, ex);
354 Closer.closeSilently(raf);
355 }
356 return null;
357 }
358 }
359 }