View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.eclipse.aether.internal.impl.checksum;
20  
21  import javax.inject.Inject;
22  import javax.inject.Named;
23  import javax.inject.Singleton;
24  
25  import java.io.BufferedReader;
26  import java.io.IOException;
27  import java.io.UncheckedIOException;
28  import java.nio.charset.StandardCharsets;
29  import java.nio.file.Files;
30  import java.nio.file.Path;
31  import java.util.ArrayList;
32  import java.util.HashMap;
33  import java.util.List;
34  import java.util.Map;
35  import java.util.Objects;
36  import java.util.concurrent.ConcurrentHashMap;
37  import java.util.concurrent.atomic.AtomicBoolean;
38  
39  import org.eclipse.aether.MultiRuntimeException;
40  import org.eclipse.aether.RepositorySystemSession;
41  import org.eclipse.aether.artifact.Artifact;
42  import org.eclipse.aether.impl.RepositorySystemLifecycle;
43  import org.eclipse.aether.internal.impl.LocalPathComposer;
44  import org.eclipse.aether.repository.ArtifactRepository;
45  import org.eclipse.aether.spi.connector.checksum.ChecksumAlgorithmFactory;
46  import org.eclipse.aether.util.ConfigUtils;
47  import org.eclipse.aether.util.FileUtils;
48  import org.slf4j.Logger;
49  import org.slf4j.LoggerFactory;
50  
51  import static java.util.Objects.requireNonNull;
52  import static java.util.stream.Collectors.toList;
53  
54  /**
55   * Compact file {@link FileTrustedChecksumsSourceSupport} implementation that use specified directory as base
56   * directory, where it expects a "summary" file named as "checksums.${checksumExt}" for each checksum algorithm.
57   * File format is GNU Coreutils compatible: each line holds checksum followed by two spaces and artifact relative path
58   * (from local repository root, without leading "./"). This means that trusted checksums summary file can be used to
59   * validate artifacts or generate it using standard GNU tools like GNU {@code sha1sum} is (for BSD derivatives same
60   * file can be used with {@code -r} switch).
61   * <p>
62   * The format supports comments "#" (hash) and empty lines for easier structuring the file content, and both are
63   * ignored. Also, their presence makes the summary file incompatible with GNU Coreutils format. On save of the
64   * summary file, the comments and empty lines are lost, and file is sorted by path names for easier diffing
65   * (2nd column in file).
66   * <p>
67   * The source by default is "origin aware", and it will factor in origin repository ID as well into summary file name,
68   * for example "checksums-central.sha256".
69   * <p>
70   * Example commands for managing summary file (in examples will use repository ID "central"):
71   * <ul>
72   *     <li>To create summary file: {@code find * -not -name "checksums-central.sha256" -type f -print0 |
73   *       xargs -0 sha256sum | sort -k 2 > checksums-central.sha256}</li>
74   *     <li>To verify artifacts using summary file: {@code sha256sum --quiet -c checksums-central.sha256}</li>
75   * </ul>
76   * <p>
77   * The checksums summary file is lazily loaded and remains cached during lifetime of the component, so file changes
78   * during lifecycle of the component are not picked up. This implementation can be simultaneously used to lookup and
79   * also record checksums. The recorded checksums will become visible for every session, and will be flushed
80   * at repository system shutdown, merged with existing ones on disk.
81   * <p>
82   * The name of this implementation is "summaryFile".
83   *
84   * @see <a href="https://man7.org/linux/man-pages/man1/sha1sum.1.html">sha1sum man page</a>
85   * @see <a href="https://www.gnu.org/software/coreutils/manual/coreutils.html#md5sum-invocation">GNU Coreutils: md5sum</a>
86   * @since 1.9.0
87   */
88  @Singleton
89  @Named(SummaryFileTrustedChecksumsSource.NAME)
90  public final class SummaryFileTrustedChecksumsSource extends FileTrustedChecksumsSourceSupport {
91      public static final String NAME = "summaryFile";
92  
93      private static final String CONFIG_PROPS_PREFIX =
94              FileTrustedChecksumsSourceSupport.CONFIG_PROPS_PREFIX + NAME + ".";
95  
96      /**
97       * Is checksum source enabled?
98       *
99       * @configurationSource {@link RepositorySystemSession#getConfigProperties()}
100      * @configurationType {@link java.lang.Boolean}
101      * @configurationDefaultValue false
102      */
103     public static final String CONFIG_PROP_ENABLED = FileTrustedChecksumsSourceSupport.CONFIG_PROPS_PREFIX + NAME;
104 
105     /**
106      * The basedir where checksums are. If relative, is resolved from local repository root.
107      *
108      * @configurationSource {@link RepositorySystemSession#getConfigProperties()}
109      * @configurationType {@link java.lang.String}
110      * @configurationDefaultValue {@link #LOCAL_REPO_PREFIX_DIR}
111      */
112     public static final String CONFIG_PROP_BASEDIR = CONFIG_PROPS_PREFIX + "basedir";
113 
114     public static final String LOCAL_REPO_PREFIX_DIR = ".checksums";
115 
116     /**
117      * Is source origin aware?
118      *
119      * @configurationSource {@link RepositorySystemSession#getConfigProperties()}
120      * @configurationType {@link java.lang.Boolean}
121      * @configurationDefaultValue true
122      */
123     public static final String CONFIG_PROP_ORIGIN_AWARE = CONFIG_PROPS_PREFIX + "originAware";
124 
125     public static final String CHECKSUMS_FILE_PREFIX = "checksums";
126 
127     private static final Logger LOGGER = LoggerFactory.getLogger(SummaryFileTrustedChecksumsSource.class);
128 
129     private final LocalPathComposer localPathComposer;
130 
131     private final RepositorySystemLifecycle repositorySystemLifecycle;
132 
133     private final ConcurrentHashMap<Path, ConcurrentHashMap<String, String>> checksums;
134 
135     private final ConcurrentHashMap<Path, Boolean> changedChecksums;
136 
137     private final AtomicBoolean onShutdownHandlerRegistered;
138 
139     @Inject
140     public SummaryFileTrustedChecksumsSource(
141             LocalPathComposer localPathComposer, RepositorySystemLifecycle repositorySystemLifecycle) {
142         this.localPathComposer = requireNonNull(localPathComposer);
143         this.repositorySystemLifecycle = requireNonNull(repositorySystemLifecycle);
144         this.checksums = new ConcurrentHashMap<>();
145         this.changedChecksums = new ConcurrentHashMap<>();
146         this.onShutdownHandlerRegistered = new AtomicBoolean(false);
147     }
148 
149     @Override
150     protected boolean isEnabled(RepositorySystemSession session) {
151         return ConfigUtils.getBoolean(session, false, CONFIG_PROP_ENABLED);
152     }
153 
154     private boolean isOriginAware(RepositorySystemSession session) {
155         return ConfigUtils.getBoolean(session, true, CONFIG_PROP_ORIGIN_AWARE);
156     }
157 
158     @Override
159     protected Map<String, String> doGetTrustedArtifactChecksums(
160             RepositorySystemSession session,
161             Artifact artifact,
162             ArtifactRepository artifactRepository,
163             List<ChecksumAlgorithmFactory> checksumAlgorithmFactories) {
164         final HashMap<String, String> result = new HashMap<>();
165         final Path basedir = getBasedir(session, LOCAL_REPO_PREFIX_DIR, CONFIG_PROP_BASEDIR, false);
166         if (Files.isDirectory(basedir)) {
167             final String artifactPath = localPathComposer.getPathForArtifact(artifact, false);
168             final boolean originAware = isOriginAware(session);
169             for (ChecksumAlgorithmFactory checksumAlgorithmFactory : checksumAlgorithmFactories) {
170                 Path summaryFile = summaryFile(
171                         basedir, originAware, artifactRepository.getId(), checksumAlgorithmFactory.getFileExtension());
172                 ConcurrentHashMap<String, String> algorithmChecksums =
173                         checksums.computeIfAbsent(summaryFile, f -> loadProvidedChecksums(summaryFile));
174                 String checksum = algorithmChecksums.get(artifactPath);
175                 if (checksum != null) {
176                     result.put(checksumAlgorithmFactory.getName(), checksum);
177                 }
178             }
179         }
180         return result;
181     }
182 
183     @Override
184     protected Writer doGetTrustedArtifactChecksumsWriter(RepositorySystemSession session) {
185         if (onShutdownHandlerRegistered.compareAndSet(false, true)) {
186             repositorySystemLifecycle.addOnSystemEndedHandler(this::saveRecordedLines);
187         }
188         return new SummaryFileWriter(
189                 checksums,
190                 getBasedir(session, LOCAL_REPO_PREFIX_DIR, CONFIG_PROP_BASEDIR, true),
191                 isOriginAware(session));
192     }
193 
194     /**
195      * Returns the summary file path. The file itself and its parent directories may not exist, this method merely
196      * calculate the path.
197      */
198     private Path summaryFile(Path basedir, boolean originAware, String repositoryId, String checksumExtension) {
199         String fileName = CHECKSUMS_FILE_PREFIX;
200         if (originAware) {
201             fileName += "-" + repositoryId;
202         }
203         return basedir.resolve(fileName + "." + checksumExtension);
204     }
205 
206     private ConcurrentHashMap<String, String> loadProvidedChecksums(Path summaryFile) {
207         ConcurrentHashMap<String, String> result = new ConcurrentHashMap<>();
208         if (Files.isRegularFile(summaryFile)) {
209             try (BufferedReader reader = Files.newBufferedReader(summaryFile, StandardCharsets.UTF_8)) {
210                 String line;
211                 while ((line = reader.readLine()) != null) {
212                     if (!line.startsWith("#") && !line.isEmpty()) {
213                         String[] parts = line.split("  ", 2);
214                         if (parts.length == 2) {
215                             String newChecksum = parts[0];
216                             String artifactPath = parts[1];
217                             String oldChecksum = result.put(artifactPath, newChecksum);
218                             if (oldChecksum != null) {
219                                 if (Objects.equals(oldChecksum, newChecksum)) {
220                                     LOGGER.warn(
221                                             "Checksums file '{}' contains duplicate checksums for artifact {}: {}",
222                                             summaryFile,
223                                             artifactPath,
224                                             oldChecksum);
225                                 } else {
226                                     LOGGER.warn(
227                                             "Checksums file '{}' contains different checksums for artifact {}: "
228                                                     + "old '{}' replaced by new '{}'",
229                                             summaryFile,
230                                             artifactPath,
231                                             oldChecksum,
232                                             newChecksum);
233                                 }
234                             }
235                         } else {
236                             LOGGER.warn("Checksums file '{}' ignored malformed line '{}'", summaryFile, line);
237                         }
238                     }
239                 }
240             } catch (IOException e) {
241                 throw new UncheckedIOException(e);
242             }
243             LOGGER.info("Loaded {} trusted checksums from {}", result.size(), summaryFile);
244         }
245         return result;
246     }
247 
248     private class SummaryFileWriter implements Writer {
249         private final ConcurrentHashMap<Path, ConcurrentHashMap<String, String>> cache;
250 
251         private final Path basedir;
252 
253         private final boolean originAware;
254 
255         private SummaryFileWriter(
256                 ConcurrentHashMap<Path, ConcurrentHashMap<String, String>> cache, Path basedir, boolean originAware) {
257             this.cache = cache;
258             this.basedir = basedir;
259             this.originAware = originAware;
260         }
261 
262         @Override
263         public void addTrustedArtifactChecksums(
264                 Artifact artifact,
265                 ArtifactRepository artifactRepository,
266                 List<ChecksumAlgorithmFactory> checksumAlgorithmFactories,
267                 Map<String, String> trustedArtifactChecksums) {
268             String artifactPath = localPathComposer.getPathForArtifact(artifact, false);
269             for (ChecksumAlgorithmFactory checksumAlgorithmFactory : checksumAlgorithmFactories) {
270                 Path summaryFile = summaryFile(
271                         basedir, originAware, artifactRepository.getId(), checksumAlgorithmFactory.getFileExtension());
272                 String checksum = requireNonNull(trustedArtifactChecksums.get(checksumAlgorithmFactory.getName()));
273 
274                 String oldChecksum = cache.computeIfAbsent(summaryFile, k -> loadProvidedChecksums(summaryFile))
275                         .put(artifactPath, checksum);
276 
277                 if (oldChecksum == null) {
278                     changedChecksums.put(summaryFile, Boolean.TRUE); // new
279                 } else if (!Objects.equals(oldChecksum, checksum)) {
280                     changedChecksums.put(summaryFile, Boolean.TRUE); // replaced
281                     LOGGER.info(
282                             "Trusted checksum for artifact {} replaced: old {}, new {}",
283                             artifact,
284                             oldChecksum,
285                             checksum);
286                 }
287             }
288         }
289     }
290 
291     /**
292      * On-close handler that saves recorded checksums, if any.
293      */
294     private void saveRecordedLines() {
295         if (changedChecksums.isEmpty()) {
296             return;
297         }
298 
299         ArrayList<Exception> exceptions = new ArrayList<>();
300         for (Map.Entry<Path, ConcurrentHashMap<String, String>> entry : checksums.entrySet()) {
301             Path summaryFile = entry.getKey();
302             if (changedChecksums.get(summaryFile) != Boolean.TRUE) {
303                 continue;
304             }
305             ConcurrentHashMap<String, String> recordedLines = entry.getValue();
306             if (!recordedLines.isEmpty()) {
307                 try {
308                     ConcurrentHashMap<String, String> result = new ConcurrentHashMap<>();
309                     result.putAll(loadProvidedChecksums(summaryFile));
310                     result.putAll(recordedLines);
311 
312                     LOGGER.info("Saving {} checksums to '{}'", result.size(), summaryFile);
313                     FileUtils.writeFileWithBackup(
314                             summaryFile,
315                             p -> Files.write(
316                                     p,
317                                     result.entrySet().stream()
318                                             .sorted(Map.Entry.comparingByKey())
319                                             .map(e -> e.getValue() + "  " + e.getKey())
320                                             .collect(toList())));
321                 } catch (IOException e) {
322                     exceptions.add(e);
323                 }
324             }
325         }
326         MultiRuntimeException.mayThrow("session save checksums failure", exceptions);
327     }
328 }