1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.maven.plugins.shade;
20
21 import javax.inject.Named;
22 import javax.inject.Singleton;
23
24 import java.io.BufferedOutputStream;
25 import java.io.ByteArrayInputStream;
26 import java.io.File;
27 import java.io.FileInputStream;
28 import java.io.IOException;
29 import java.io.InputStream;
30 import java.io.InputStreamReader;
31 import java.io.OutputStreamWriter;
32 import java.io.PushbackInputStream;
33 import java.io.Writer;
34 import java.nio.charset.StandardCharsets;
35 import java.util.ArrayList;
36 import java.util.Arrays;
37 import java.util.Calendar;
38 import java.util.Collection;
39 import java.util.Collections;
40 import java.util.Enumeration;
41 import java.util.HashSet;
42 import java.util.Iterator;
43 import java.util.LinkedList;
44 import java.util.List;
45 import java.util.Objects;
46 import java.util.Set;
47 import java.util.concurrent.Callable;
48 import java.util.jar.JarEntry;
49 import java.util.jar.JarFile;
50 import java.util.jar.JarOutputStream;
51 import java.util.regex.Matcher;
52 import java.util.regex.Pattern;
53 import java.util.zip.CRC32;
54 import java.util.zip.ZipEntry;
55 import java.util.zip.ZipException;
56
57 import org.apache.commons.collections4.MultiValuedMap;
58 import org.apache.commons.collections4.multimap.HashSetValuedHashMap;
59 import org.apache.commons.compress.archivers.zip.ExtraFieldUtils;
60 import org.apache.commons.compress.archivers.zip.X5455_ExtendedTimestamp;
61 import org.apache.commons.compress.archivers.zip.ZipExtraField;
62 import org.apache.maven.plugin.MojoExecutionException;
63 import org.apache.maven.plugins.shade.filter.Filter;
64 import org.apache.maven.plugins.shade.relocation.Relocator;
65 import org.apache.maven.plugins.shade.resource.ManifestResourceTransformer;
66 import org.apache.maven.plugins.shade.resource.ReproducibleResourceTransformer;
67 import org.apache.maven.plugins.shade.resource.ResourceTransformer;
68 import org.codehaus.plexus.util.IOUtil;
69 import org.codehaus.plexus.util.io.CachingOutputStream;
70 import org.objectweb.asm.ClassReader;
71 import org.objectweb.asm.ClassVisitor;
72 import org.objectweb.asm.ClassWriter;
73 import org.objectweb.asm.commons.ClassRemapper;
74 import org.objectweb.asm.commons.Remapper;
75 import org.slf4j.Logger;
76 import org.slf4j.LoggerFactory;
77
78
79
80
81 @Singleton
82 @Named
83 public class DefaultShader implements Shader {
84 private static final int BUFFER_SIZE = 32 * 1024;
85
86 private final Logger logger;
87
88 public DefaultShader() {
89 this(LoggerFactory.getLogger(DefaultShader.class));
90 }
91
92 public DefaultShader(final Logger logger) {
93 this.logger = Objects.requireNonNull(logger);
94 }
95
96
97 private long getTime(ZipEntry entry) {
98 if (entry.getExtra() != null) {
99 try {
100 ZipExtraField[] fields =
101 ExtraFieldUtils.parse(entry.getExtra(), true, ExtraFieldUtils.UnparseableExtraField.SKIP);
102 for (ZipExtraField field : fields) {
103 if (X5455_ExtendedTimestamp.HEADER_ID.equals(field.getHeaderId())) {
104
105 Calendar cal = Calendar.getInstance();
106 cal.setTimeInMillis(entry.getTime());
107 return entry.getTime() - (cal.get(Calendar.ZONE_OFFSET) + cal.get(Calendar.DST_OFFSET));
108 }
109 }
110 } catch (ZipException ze) {
111
112 }
113 }
114 return entry.getTime();
115 }
116
117 public void shade(ShadeRequest shadeRequest) throws IOException, MojoExecutionException {
118 Set<String> resources = new HashSet<>();
119
120 ManifestResourceTransformer manifestTransformer = null;
121 List<ResourceTransformer> transformers = new ArrayList<>(shadeRequest.getResourceTransformers());
122 for (Iterator<ResourceTransformer> it = transformers.iterator(); it.hasNext(); ) {
123 ResourceTransformer transformer = it.next();
124 if (transformer instanceof ManifestResourceTransformer) {
125 manifestTransformer = (ManifestResourceTransformer) transformer;
126 it.remove();
127 }
128 }
129
130 final DefaultPackageMapper packageMapper = new DefaultPackageMapper(shadeRequest.getRelocators());
131
132
133 shadeRequest.getUberJar().getParentFile().mkdirs();
134
135 try (JarOutputStream out =
136 new JarOutputStream(new BufferedOutputStream(new CachingOutputStream(shadeRequest.getUberJar())))) {
137 goThroughAllJarEntriesForManifestTransformer(shadeRequest, resources, manifestTransformer, out);
138
139
140 MultiValuedMap<String, File> duplicates = new HashSetValuedHashMap<>(10000, 3);
141
142
143 shadeJars(shadeRequest, resources, transformers, out, duplicates, packageMapper);
144
145
146 MultiValuedMap<Collection<File>, String> overlapping = new HashSetValuedHashMap<>(20, 15);
147
148
149 for (String clazz : duplicates.keySet()) {
150 Collection<File> jarz = duplicates.get(clazz);
151 if (jarz.size() > 1) {
152 overlapping.put(jarz, clazz);
153 }
154 }
155
156
157 logSummaryOfDuplicates(overlapping);
158
159 if (overlapping.keySet().size() > 0) {
160 showOverlappingWarning();
161 }
162
163 for (ResourceTransformer transformer : transformers) {
164 if (transformer.hasTransformedResource()) {
165 transformer.modifyOutputStream(out);
166 }
167 }
168 }
169
170 for (Filter filter : shadeRequest.getFilters()) {
171 filter.finished();
172 }
173 }
174
175
176
177
178 private static class ZipHeaderPeekInputStream extends PushbackInputStream {
179
180 private static final byte[] ZIP_HEADER = new byte[] {0x50, 0x4b, 0x03, 0x04};
181
182 private static final int HEADER_LEN = 4;
183
184 protected ZipHeaderPeekInputStream(InputStream in) {
185 super(in, HEADER_LEN);
186 }
187
188 public boolean hasZipHeader() throws IOException {
189 final byte[] header = new byte[HEADER_LEN];
190 int len = super.read(header, 0, HEADER_LEN);
191 if (len != -1) {
192 super.unread(header, 0, len);
193 }
194 return Arrays.equals(header, ZIP_HEADER);
195 }
196 }
197
198
199
200
201 private static class CrcAndSize {
202
203 private final CRC32 crc = new CRC32();
204
205 private long size;
206
207 CrcAndSize(InputStream inputStream) throws IOException {
208 load(inputStream);
209 }
210
211 private void load(InputStream inputStream) throws IOException {
212 byte[] buffer = new byte[BUFFER_SIZE];
213 int bytesRead;
214 while ((bytesRead = inputStream.read(buffer)) != -1) {
215 this.crc.update(buffer, 0, bytesRead);
216 this.size += bytesRead;
217 }
218 }
219
220 public void setupStoredEntry(JarEntry entry) {
221 entry.setSize(this.size);
222 entry.setCompressedSize(this.size);
223 entry.setCrc(this.crc.getValue());
224 entry.setMethod(ZipEntry.STORED);
225 }
226 }
227
228 private void shadeJars(
229 ShadeRequest shadeRequest,
230 Set<String> resources,
231 List<ResourceTransformer> transformers,
232 JarOutputStream jos,
233 MultiValuedMap<String, File> duplicates,
234 DefaultPackageMapper packageMapper)
235 throws IOException {
236 for (File jar : shadeRequest.getJars()) {
237
238 logger.debug("Processing JAR " + jar);
239
240 List<Filter> jarFilters = getFilters(jar, shadeRequest.getFilters());
241 if (jar.isDirectory()) {
242 shadeDir(
243 shadeRequest,
244 resources,
245 transformers,
246 packageMapper,
247 jos,
248 duplicates,
249 jar,
250 jar,
251 "",
252 jarFilters);
253 } else {
254 shadeJar(shadeRequest, resources, transformers, packageMapper, jos, duplicates, jar, jarFilters);
255 }
256 }
257 }
258
259 private void shadeDir(
260 ShadeRequest shadeRequest,
261 Set<String> resources,
262 List<ResourceTransformer> transformers,
263 DefaultPackageMapper packageMapper,
264 JarOutputStream jos,
265 MultiValuedMap<String, File> duplicates,
266 File jar,
267 File current,
268 String prefix,
269 List<Filter> jarFilters)
270 throws IOException {
271 final File[] children = current.listFiles();
272 if (children == null) {
273 return;
274 }
275 for (final File file : children) {
276 final String name = prefix + file.getName();
277 if (file.isDirectory()) {
278 try {
279 shadeDir(
280 shadeRequest,
281 resources,
282 transformers,
283 packageMapper,
284 jos,
285 duplicates,
286 jar,
287 file,
288 prefix + file.getName() + '/',
289 jarFilters);
290 continue;
291 } catch (Exception e) {
292 throw new IOException(String.format("Problem shading JAR %s entry %s: %s", current, name, e), e);
293 }
294 }
295 if (isFiltered(jarFilters, name) || isExcludedEntry(name)) {
296 continue;
297 }
298
299 try {
300 shadeJarEntry(
301 shadeRequest,
302 resources,
303 transformers,
304 packageMapper,
305 jos,
306 duplicates,
307 jar,
308 new Callable<InputStream>() {
309 @Override
310 public InputStream call() throws Exception {
311 return new FileInputStream(file);
312 }
313 },
314 name,
315 file.lastModified(),
316 -1 );
317 } catch (Exception e) {
318 throw new IOException(String.format("Problem shading JAR %s entry %s: %s", current, name, e), e);
319 }
320 }
321 }
322
323 private void shadeJar(
324 ShadeRequest shadeRequest,
325 Set<String> resources,
326 List<ResourceTransformer> transformers,
327 DefaultPackageMapper packageMapper,
328 JarOutputStream jos,
329 MultiValuedMap<String, File> duplicates,
330 File jar,
331 List<Filter> jarFilters)
332 throws IOException {
333 try (JarFile jarFile = newJarFile(jar)) {
334
335 for (Enumeration<JarEntry> j = jarFile.entries(); j.hasMoreElements(); ) {
336 final JarEntry entry = j.nextElement();
337
338 String name = entry.getName();
339
340 if (entry.isDirectory() || isFiltered(jarFilters, name) || isExcludedEntry(name)) {
341 continue;
342 }
343
344 try {
345 shadeJarEntry(
346 shadeRequest,
347 resources,
348 transformers,
349 packageMapper,
350 jos,
351 duplicates,
352 jar,
353 new Callable<InputStream>() {
354 @Override
355 public InputStream call() throws Exception {
356 return jarFile.getInputStream(entry);
357 }
358 },
359 name,
360 getTime(entry),
361 entry.getMethod());
362 } catch (Exception e) {
363 throw new IOException(String.format("Problem shading JAR %s entry %s: %s", jar, name, e), e);
364 }
365 }
366 }
367 }
368
369 private boolean isExcludedEntry(final String name) {
370 if ("META-INF/INDEX.LIST".equals(name)) {
371
372
373
374 return true;
375 }
376
377 if ("module-info.class".equals(name)) {
378 logger.warn("Discovered module-info.class. " + "Shading will break its strong encapsulation.");
379 return true;
380 }
381 return false;
382 }
383
384 private void shadeJarEntry(
385 ShadeRequest shadeRequest,
386 Set<String> resources,
387 List<ResourceTransformer> transformers,
388 DefaultPackageMapper packageMapper,
389 JarOutputStream jos,
390 MultiValuedMap<String, File> duplicates,
391 File jar,
392 Callable<InputStream> inputProvider,
393 String name,
394 long time,
395 int method)
396 throws Exception {
397 try (InputStream in = inputProvider.call()) {
398 String mappedName = packageMapper.map(name, true, false);
399
400 int idx = mappedName.lastIndexOf('/');
401 if (idx != -1) {
402
403 String dir = mappedName.substring(0, idx);
404 if (!resources.contains(dir)) {
405 addDirectory(resources, jos, dir, time);
406 }
407 }
408
409 duplicates.put(name, jar);
410 if (name.endsWith(".class")) {
411 addRemappedClass(jos, jar, name, time, in, packageMapper);
412 } else if (shadeRequest.isShadeSourcesContent() && name.endsWith(".java")) {
413
414 if (resources.contains(mappedName)) {
415 return;
416 }
417
418 addJavaSource(resources, jos, mappedName, time, in, shadeRequest.getRelocators());
419 } else {
420 if (!resourceTransformed(transformers, mappedName, in, shadeRequest.getRelocators(), time)) {
421
422 if (resources.contains(mappedName)) {
423 logger.debug("We have a duplicate " + name + " in " + jar);
424 return;
425 }
426
427 addResource(resources, jos, mappedName, inputProvider, time, method);
428 } else {
429 duplicates.removeMapping(name, jar);
430 }
431 }
432 }
433 }
434
435 private void goThroughAllJarEntriesForManifestTransformer(
436 ShadeRequest shadeRequest,
437 Set<String> resources,
438 ManifestResourceTransformer manifestTransformer,
439 JarOutputStream jos)
440 throws IOException {
441 if (manifestTransformer != null) {
442 for (File jar : shadeRequest.getJars()) {
443 try (JarFile jarFile = newJarFile(jar)) {
444 for (Enumeration<JarEntry> en = jarFile.entries(); en.hasMoreElements(); ) {
445 JarEntry entry = en.nextElement();
446 String resource = entry.getName();
447 if (manifestTransformer.canTransformResource(resource)) {
448 resources.add(resource);
449 try (InputStream inputStream = jarFile.getInputStream(entry)) {
450 manifestTransformer.processResource(
451 resource, inputStream, shadeRequest.getRelocators(), getTime(entry));
452 }
453 break;
454 }
455 }
456 }
457 }
458 if (manifestTransformer.hasTransformedResource()) {
459 manifestTransformer.modifyOutputStream(jos);
460 }
461 }
462 }
463
464 private void showOverlappingWarning() {
465 logger.warn("maven-shade-plugin has detected that some files are");
466 logger.warn("present in two or more JARs. When this happens, only one");
467 logger.warn("single version of the file is copied to the uber jar.");
468 logger.warn("Usually this is not harmful and you can skip these warnings,");
469 logger.warn("otherwise try to manually exclude artifacts based on");
470 logger.warn("mvn dependency:tree -Ddetail=true and the above output.");
471 logger.warn("See https://maven.apache.org/plugins/maven-shade-plugin/");
472 }
473
474 private void logSummaryOfDuplicates(MultiValuedMap<Collection<File>, String> overlapping) {
475 for (Collection<File> jarz : overlapping.keySet()) {
476 List<String> jarzS = new ArrayList<>();
477
478 for (File jjar : jarz) {
479 jarzS.add(jjar.getName());
480 }
481
482 Collections.sort(jarzS);
483
484 List<String> classes = new LinkedList<>();
485 List<String> resources = new LinkedList<>();
486
487 for (String name : overlapping.get(jarz)) {
488 if (name.endsWith(".class")) {
489 classes.add(name.replace(".class", "").replace("/", "."));
490 } else {
491 resources.add(name);
492 }
493 }
494
495
496 final Collection<String> overlaps = new ArrayList<>();
497 if (!classes.isEmpty()) {
498 if (resources.size() == 1) {
499 overlaps.add("class");
500 } else {
501 overlaps.add("classes");
502 }
503 }
504 if (!resources.isEmpty()) {
505 if (resources.size() == 1) {
506 overlaps.add("resource");
507 } else {
508 overlaps.add("resources");
509 }
510 }
511
512 final List<String> all = new ArrayList<>(classes.size() + resources.size());
513 all.addAll(classes);
514 all.addAll(resources);
515
516 logger.warn(String.join(", ", jarzS) + " define " + all.size() + " overlapping "
517 + String.join(" and ", overlaps) + ": ");
518
519
520 Collections.sort(all);
521
522 int max = 10;
523
524 for (int i = 0; i < Math.min(max, all.size()); i++) {
525 logger.warn(" - " + all.get(i));
526 }
527
528 if (all.size() > max) {
529 logger.warn(" - " + (all.size() - max) + " more...");
530 }
531 }
532 }
533
534 private JarFile newJarFile(File jar) throws IOException {
535 try {
536 return new JarFile(jar);
537 } catch (ZipException zex) {
538
539
540 throw new ZipException("error in opening zip file " + jar);
541 }
542 }
543
544 private List<Filter> getFilters(File jar, List<Filter> filters) {
545 List<Filter> list = new ArrayList<>();
546
547 for (Filter filter : filters) {
548 if (filter.canFilter(jar)) {
549 list.add(filter);
550 }
551 }
552
553 return list;
554 }
555
556 private void addDirectory(Set<String> resources, JarOutputStream jos, String name, long time) throws IOException {
557 if (name.lastIndexOf('/') > 0) {
558 String parent = name.substring(0, name.lastIndexOf('/'));
559 if (!resources.contains(parent)) {
560 addDirectory(resources, jos, parent, time);
561 }
562 }
563
564
565 JarEntry entry = new JarEntry(name + "/");
566 entry.setTime(time);
567 jos.putNextEntry(entry);
568
569 resources.add(name);
570 }
571
572 private void addRemappedClass(
573 JarOutputStream jos, File jar, String name, long time, InputStream is, DefaultPackageMapper packageMapper)
574 throws IOException, MojoExecutionException {
575 if (packageMapper.relocators.isEmpty()) {
576 try {
577 JarEntry entry = new JarEntry(name);
578 entry.setTime(time);
579 jos.putNextEntry(entry);
580 IOUtil.copy(is, jos);
581 } catch (ZipException e) {
582 logger.debug("We have a duplicate " + name + " in " + jar);
583 }
584
585 return;
586 }
587
588
589
590
591 byte[] originalClass = IOUtil.toByteArray(is);
592
593 ClassReader cr = new ClassReader(new ByteArrayInputStream(originalClass));
594
595
596
597
598
599
600 ClassWriter cw = new ClassWriter(0);
601
602 final String pkg = name.substring(0, name.lastIndexOf('/') + 1);
603 final ShadeClassRemapper cv = new ShadeClassRemapper(cw, pkg, packageMapper);
604
605 try {
606 cr.accept(cv, ClassReader.EXPAND_FRAMES);
607 } catch (Throwable ise) {
608 throw new MojoExecutionException("Error in ASM processing class " + name, ise);
609 }
610
611
612 final byte[] renamedClass;
613 if (cv.remapped) {
614 logger.debug("Rewrote class bytecode: " + name);
615 renamedClass = cw.toByteArray();
616 } else {
617 logger.debug("Keeping original class bytecode: " + name);
618 renamedClass = originalClass;
619 }
620
621
622 String mappedName = packageMapper.map(name.substring(0, name.indexOf('.')), true, false);
623
624 try {
625
626 JarEntry entry = new JarEntry(mappedName + ".class");
627 entry.setTime(time);
628 jos.putNextEntry(entry);
629
630 jos.write(renamedClass);
631 } catch (ZipException e) {
632 logger.debug("We have a duplicate " + mappedName + " in " + jar);
633 }
634 }
635
636 private boolean isFiltered(List<Filter> filters, String name) {
637 for (Filter filter : filters) {
638 if (filter.isFiltered(name)) {
639 return true;
640 }
641 }
642
643 return false;
644 }
645
646 private boolean resourceTransformed(
647 List<ResourceTransformer> resourceTransformers,
648 String name,
649 InputStream is,
650 List<Relocator> relocators,
651 long time)
652 throws IOException {
653 boolean resourceTransformed = false;
654
655 for (ResourceTransformer transformer : resourceTransformers) {
656 if (transformer.canTransformResource(name)) {
657 logger.debug("Transforming " + name + " using "
658 + transformer.getClass().getName());
659
660 if (transformer instanceof ReproducibleResourceTransformer) {
661 ((ReproducibleResourceTransformer) transformer).processResource(name, is, relocators, time);
662 } else {
663 transformer.processResource(name, is, relocators);
664 }
665
666 resourceTransformed = true;
667
668 break;
669 }
670 }
671 return resourceTransformed;
672 }
673
674 private void addJavaSource(
675 Set<String> resources,
676 JarOutputStream jos,
677 String name,
678 long time,
679 InputStream is,
680 List<Relocator> relocators)
681 throws IOException {
682 JarEntry entry = new JarEntry(name);
683 entry.setTime(time);
684 jos.putNextEntry(entry);
685
686 String sourceContent = IOUtil.toString(new InputStreamReader(is, StandardCharsets.UTF_8));
687
688 for (Relocator relocator : relocators) {
689 sourceContent = relocator.applyToSourceContent(sourceContent);
690 }
691
692 final Writer writer = new OutputStreamWriter(jos, StandardCharsets.UTF_8);
693 writer.write(sourceContent);
694 writer.flush();
695
696 resources.add(name);
697 }
698
699 private void addResource(
700 Set<String> resources, JarOutputStream jos, String name, Callable<InputStream> input, long time, int method)
701 throws Exception {
702 ZipHeaderPeekInputStream inputStream = new ZipHeaderPeekInputStream(input.call());
703 try {
704 final JarEntry entry = new JarEntry(name);
705
706
707 if (inputStream.hasZipHeader() && method == ZipEntry.STORED) {
708 new CrcAndSize(inputStream).setupStoredEntry(entry);
709 inputStream.close();
710 inputStream = new ZipHeaderPeekInputStream(input.call());
711 }
712
713 entry.setTime(time);
714
715 jos.putNextEntry(entry);
716
717 IOUtil.copy(inputStream, jos);
718
719 resources.add(name);
720 } finally {
721 inputStream.close();
722 }
723 }
724
725 private interface PackageMapper {
726
727
728
729
730
731
732
733
734 String map(String entityName, boolean mapPaths, boolean mapPackages);
735 }
736
737
738
739
740 private static class DefaultPackageMapper implements PackageMapper {
741 private static final Pattern CLASS_PATTERN = Pattern.compile("(\\[*)?L(.+);");
742
743 private final List<Relocator> relocators;
744
745 private DefaultPackageMapper(final List<Relocator> relocators) {
746 this.relocators = relocators;
747 }
748
749 @Override
750 public String map(String entityName, boolean mapPaths, final boolean mapPackages) {
751 String value = entityName;
752
753 String prefix = "";
754 String suffix = "";
755
756 Matcher m = CLASS_PATTERN.matcher(entityName);
757 if (m.matches()) {
758 prefix = m.group(1) + "L";
759 suffix = ";";
760 entityName = m.group(2);
761 }
762
763 for (Relocator r : relocators) {
764 if (mapPackages && r.canRelocateClass(entityName)) {
765 value = prefix + r.relocateClass(entityName) + suffix;
766 break;
767 } else if (mapPaths && r.canRelocatePath(entityName)) {
768 value = prefix + r.relocatePath(entityName) + suffix;
769 break;
770 }
771 }
772 return value;
773 }
774 }
775
776 private static class LazyInitRemapper extends Remapper {
777 private PackageMapper relocators;
778
779 @Override
780 public Object mapValue(Object object) {
781 return object instanceof String ? relocators.map((String) object, true, true) : super.mapValue(object);
782 }
783
784 @Override
785 public String map(String name) {
786
787
788
789
790
791
792
793
794 return relocators.map(name, true, false);
795 }
796 }
797
798
799
800
801
802
803
804
805
806
807
808 private static class ShadeClassRemapper extends ClassRemapper implements PackageMapper {
809 private final String pkg;
810 private final PackageMapper packageMapper;
811 private boolean remapped;
812
813 ShadeClassRemapper(
814 final ClassVisitor classVisitor, final String pkg, final DefaultPackageMapper packageMapper) {
815 super(classVisitor, new LazyInitRemapper() );
816 this.pkg = pkg;
817 this.packageMapper = packageMapper;
818
819
820 LazyInitRemapper.class.cast(remapper).relocators = this;
821 }
822
823 @Override
824 public void visitSource(final String source, final String debug) {
825 if (source == null) {
826 super.visitSource(null, debug);
827 return;
828 }
829
830 final String fqSource = pkg + source;
831 final String mappedSource = map(fqSource, true, false);
832 final String filename = mappedSource.substring(mappedSource.lastIndexOf('/') + 1);
833 super.visitSource(filename, debug);
834 }
835
836 @Override
837 public String map(final String entityName, boolean mapPaths, final boolean mapPackages) {
838 final String mapped = packageMapper.map(entityName, true, mapPackages);
839 if (!remapped) {
840 remapped = !mapped.equals(entityName);
841 }
842 return mapped;
843 }
844 }
845 }