View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *   http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing,
13   * software distributed under the License is distributed on an
14   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15   * KIND, either express or implied.  See the License for the
16   * specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.maven.surefire.its;
20  
21  import org.apache.maven.surefire.its.fixture.Settings;
22  import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
23  import org.apache.maven.surefire.its.fixture.SurefireLauncher;
24  import org.junit.Test;
25  
26  import static org.apache.maven.surefire.its.fixture.Configuration.INCLUDES;
27  import static org.apache.maven.surefire.its.fixture.Configuration.INCLUDES_EXCLUDES;
28  import static org.apache.maven.surefire.its.fixture.Configuration.INCLUDES_EXCLUDES_FILE;
29  import static org.apache.maven.surefire.its.fixture.Configuration.INCLUDES_FILE;
30  import static org.apache.maven.surefire.its.fixture.Configuration.TEST;
31  import static org.apache.maven.surefire.its.fixture.TestFramework.JUNIT47;
32  import static org.apache.maven.surefire.its.fixture.TestFramework.TestNG;
33  import static org.hamcrest.core.AnyOf.anyOf;
34  import static org.hamcrest.core.Is.is;
35  import static org.junit.Assume.assumeThat;
36  
37  /**
38   * Test project using multiple method patterns, including wildcards in class and method names.
39   */
40  public abstract class AbstractTestMultipleMethodPatterns extends SurefireJUnit4IntegrationTestCase {
41      private static final String CSV_DELIMITER_SHORT = ",";
42      private static final String CSV_DELIMITER_LONG = ", ";
43      private static final String NOT_DELIMITER = "!";
44  
45      protected abstract Settings getSettings();
46  
47      protected abstract SurefireLauncher unpack();
48  
49      protected SurefireLauncher prepare(String tests) {
50          SurefireLauncher launcher = unpack().addGoal("-P " + getSettings().profile());
51          String[] includedExcluded = splitIncludesExcludes(tests);
52          switch (getSettings().getConfiguration()) {
53              case TEST:
54                  launcher.setTestToRun(tests);
55                  break;
56              case INCLUDES:
57                  launcher.sysProp("included", tests);
58                  break;
59              case INCLUDES_EXCLUDES:
60                  launcher.sysProp("included", includedExcluded[0]);
61                  launcher.sysProp("excluded", includedExcluded[1]);
62                  break;
63              default:
64                  throw new IllegalArgumentException(
65                          "Unsupported configuration " + getSettings().getConfiguration());
66          }
67          return launcher;
68      }
69  
70      private static String[] splitIncludesExcludes(String patterns) {
71          String included = "";
72          String excluded = "";
73          for (String pattern : patterns.split(CSV_DELIMITER_SHORT)) {
74              pattern = pattern.trim();
75              if (pattern.startsWith(NOT_DELIMITER)) {
76                  excluded += pattern.substring(NOT_DELIMITER.length()).trim();
77                  excluded += CSV_DELIMITER_LONG;
78              } else {
79                  included += pattern;
80                  included += CSV_DELIMITER_LONG;
81              }
82          }
83          return new String[] {trimEndComma(included), trimEndComma(excluded)};
84      }
85  
86      private static String trimEndComma(String pattern) {
87          pattern = pattern.trim();
88          return pattern.endsWith(CSV_DELIMITER_LONG)
89                  ? pattern.substring(0, pattern.length() - CSV_DELIMITER_LONG.length())
90                  : pattern;
91      }
92  
93      @Test
94      public void simpleNameTest() {
95          prepare("TestTwo")
96                  .executeTest()
97                  .verifyErrorFree(2)
98                  .verifyErrorFreeLog()
99                  .verifyTextInLog("jiras.surefire745.TestTwo#testSuccessOne")
100                 .verifyTextInLog("jiras.surefire745.TestTwo#testSuccessTwo");
101     }
102 
103     @Test
104     public void simpleNameTestAsParallel() {
105         assumeThat(getSettings().getFramework(), anyOf(is(JUNIT47), is(TestNG)));
106         prepare("TestTwo")
107                 .parallel("classes")
108                 .useUnlimitedThreads()
109                 .executeTest()
110                 .verifyErrorFree(2)
111                 .verifyErrorFreeLog()
112                 .verifyTextInLog("jiras.surefire745.TestTwo#testSuccessOne")
113                 .verifyTextInLog("jiras.surefire745.TestTwo#testSuccessTwo");
114     }
115 
116     @Test
117     public void simpleNameTestWithJavaExt() {
118         prepare("TestTwo.java")
119                 .executeTest()
120                 .verifyErrorFree(2)
121                 .verifyErrorFreeLog()
122                 .verifyTextInLog("jiras.surefire745.TestTwo#testSuccessOne")
123                 .verifyTextInLog("jiras.surefire745.TestTwo#testSuccessTwo");
124     }
125 
126     @Test
127     public void simpleNameTestWithWildcardPkg() {
128         prepare("**/TestTwo")
129                 .executeTest()
130                 .verifyErrorFree(2)
131                 .verifyErrorFreeLog()
132                 .verifyTextInLog("jiras.surefire745.TestTwo#testSuccessOne")
133                 .verifyTextInLog("jiras.surefire745.TestTwo#testSuccessTwo");
134     }
135 
136     @Test
137     public void simpleNameTestWithJavaExtWildcardPkg() {
138         prepare("**/TestTwo.java")
139                 .executeTest()
140                 .verifyErrorFree(2)
141                 .verifyErrorFreeLog()
142                 .verifyTextInLog("jiras.surefire745.TestTwo#testSuccessOne")
143                 .verifyTextInLog("jiras.surefire745.TestTwo#testSuccessTwo");
144     }
145 
146     @Test
147     public void fullyQualifiedTest() {
148         prepare("jiras/surefire745/TestTwo.java")
149                 .executeTest()
150                 .verifyErrorFree(2)
151                 .verifyErrorFreeLog()
152                 .verifyTextInLog("jiras.surefire745.TestTwo#testSuccessOne")
153                 .verifyTextInLog("jiras.surefire745.TestTwo#testSuccessTwo");
154     }
155 
156     @Test
157     public void shouldMatchSimpleClassNameAndMethod() {
158         assumeThat(getSettings().getConfiguration(), is(TEST));
159         prepare("BasicTest#testSuccessTwo")
160                 .executeTest()
161                 .verifyErrorFree(1)
162                 .verifyErrorFreeLog()
163                 .verifyTextInLog("jiras.surefire745.BasicTest#testSuccessTwo");
164     }
165 
166     /**
167      * This method name was shorten because it cause 261 character long path on Windows with Jenkins Pipeline.
168      */
169     @Test
170     public void matchSimpleClassAndMethodWithJavaExt() {
171         assumeThat(getSettings().getConfiguration(), is(TEST));
172         prepare("BasicTest.java#testSuccessTwo")
173                 .executeTest()
174                 .verifyErrorFree(1)
175                 .verifyErrorFreeLog()
176                 .verifyTextInLog("jiras.surefire745.BasicTest#testSuccessTwo");
177     }
178 
179     /**
180      * This method name was shorten because it cause 261 character long path on Windows with Jenkins Pipeline.
181      */
182     @Test
183     public void matchSimpleClassAndMethodWithWildcardPkg() {
184         assumeThat(getSettings().getConfiguration(), is(TEST));
185         prepare("**/BasicTest#testSuccessTwo")
186                 .executeTest()
187                 .verifyErrorFree(1)
188                 .verifyErrorFreeLog()
189                 .verifyTextInLog("jiras.surefire745.BasicTest#testSuccessTwo");
190     }
191 
192     /**
193      * This method name was shorten because it cause 261 character long path on Windows with Jenkins Pipeline.
194      */
195     @Test
196     public void matchSimpleClassAndMethodWithJavaExtWildcardPkg() {
197         assumeThat(getSettings().getConfiguration(), is(TEST));
198         prepare("**/BasicTest.java#testSuccessTwo")
199                 .executeTest()
200                 .verifyErrorFree(1)
201                 .verifyErrorFreeLog()
202                 .verifyTextInLog("jiras.surefire745.BasicTest#testSuccessTwo");
203     }
204 
205     @Test
206     public void shouldMatchWildcardPackageAndClassAndMethod() {
207         assumeThat(getSettings().getConfiguration(), is(TEST));
208         prepare("jiras/**/BasicTest#testSuccessTwo")
209                 .executeTest()
210                 .verifyErrorFree(1)
211                 .verifyErrorFreeLog()
212                 .verifyTextInLog("jiras.surefire745.BasicTest#testSuccessTwo");
213     }
214 
215     @Test
216     public void regexClass() {
217         prepare("%regex[.*.TestTwo.*]")
218                 .executeTest()
219                 .verifyErrorFree(2)
220                 .verifyErrorFreeLog()
221                 .verifyTextInLog("jiras.surefire745.TestTwo#testSuccessOne")
222                 .verifyTextInLog("jiras.surefire745.TestTwo#testSuccessTwo");
223     }
224 
225     @Test
226     public void testSuccessTwo() {
227         assumeThat(getSettings().getConfiguration(), is(TEST));
228         prepare("#testSuccessTwo")
229                 .maven()
230                 .debugLogging()
231                 .executeTest()
232                 .verifyErrorFree(5)
233                 .verifyErrorFreeLog();
234     }
235 
236     @Test
237     public void testRegexSuccessTwo() {
238         assumeThat(getSettings().getConfiguration(), is(TEST));
239         prepare("%regex[#testSuccessTwo]").executeTest().verifyErrorFree(5).verifyErrorFreeLog();
240     }
241 
242     @Test
243     public void regexClassAndMethod() {
244         assumeThat(getSettings().getConfiguration(), is(TEST));
245         prepare("%regex[.*.BasicTest.*#testSuccessTwo]")
246                 .executeTest()
247                 .verifyErrorFree(1)
248                 .verifyErrorFreeLog()
249                 .verifyTextInLog("jiras.surefire745.BasicTest#testSuccessTwo");
250     }
251 
252     @Test
253     public void shouldMatchExactClassAndMethodWildcard() {
254         assumeThat(getSettings().getConfiguration(), is(TEST));
255         prepare("BasicTest#test*One")
256                 .executeTest()
257                 .verifyErrorFree(1)
258                 .verifyErrorFreeLog()
259                 .verifyTextInLog("jiras.surefire745.BasicTest#testSuccessOne");
260     }
261 
262     @Test
263     public void shouldMatchExactClassAndMethodsWildcard() {
264         assumeThat(getSettings().getConfiguration(), is(TEST));
265         prepare("BasicTest#testSuccess*")
266                 .executeTest()
267                 .verifyErrorFree(2)
268                 .verifyErrorFreeLog()
269                 .verifyTextInLog("jiras.surefire745.BasicTest#testSuccessOne")
270                 .verifyTextInLog("jiras.surefire745.BasicTest#testSuccessTwo");
271     }
272 
273     @Test
274     public void shouldMatchExactClassAndMethodCharacters() {
275         assumeThat(getSettings().getConfiguration(), is(TEST));
276         prepare("BasicTest#test???????One")
277                 .executeTest()
278                 .verifyErrorFree(1)
279                 .verifyErrorFreeLog()
280                 .verifyTextInLog("jiras.surefire745.BasicTest#testSuccessOne");
281     }
282 
283     @Test
284     public void shouldMatchExactClassAndMethodsPostfix() {
285         assumeThat(getSettings().getConfiguration(), is(TEST));
286         prepare("TestFive#testSuccess???")
287                 .executeTest()
288                 .verifyErrorFree(2)
289                 .verifyErrorFreeLog()
290                 .verifyTextInLog("jiras.surefire745.TestFive#testSuccessOne")
291                 .verifyTextInLog("jiras.surefire745.TestFive#testSuccessTwo");
292     }
293 
294     @Test
295     public void shouldMatchExactClassAndMethodPostfix() {
296         assumeThat(getSettings().getConfiguration(), is(TEST));
297         prepare("TestFive#testSuccess?????")
298                 .executeTest()
299                 .verifyErrorFree(1)
300                 .verifyErrorFreeLog()
301                 .verifyTextInLog("jiras.surefire745.TestFive#testSuccessThree");
302     }
303 
304     @Test
305     public void shouldMatchExactClassAndMultipleMethods() {
306         assumeThat(getSettings().getConfiguration(), is(TEST));
307         prepare("TestFive#testSuccessOne+testSuccessThree")
308                 .executeTest()
309                 .verifyErrorFree(2)
310                 .verifyErrorFreeLog()
311                 .verifyTextInLog("jiras.surefire745.TestFive#testSuccessOne")
312                 .verifyTextInLog("jiras.surefire745.TestFive#testSuccessThree");
313     }
314 
315     @Test
316     public void shouldMatchMultiplePatterns() {
317         assumeThat(getSettings().getConfiguration(), is(TEST));
318         String test = "jiras/surefire745/BasicTest#testSuccessOne+testSuccessTwo" // 2
319                 + ',' + "jiras/**/TestTwo" // 2
320                 + ',' + "jiras/surefire745/TestThree#testSuccess*" // 2
321                 + ',' + "TestFour#testSuccess???" // 2
322                 + ',' + "jiras/surefire745/*Five#test*One"; // 1
323 
324         prepare(test).executeTest().verifyErrorFree(9).verifyErrorFreeLog();
325     }
326 
327     @Test
328     public void shouldMatchMultiplePatternsAsParallel() {
329         assumeThat(getSettings().getFramework(), anyOf(is(JUNIT47), is(TestNG)));
330         assumeThat(getSettings().getConfiguration(), is(TEST));
331         String test = "jiras/surefire745/BasicTest#testSuccessOne+testSuccessTwo" // 2
332                 + ',' + "jiras/**/TestTwo" // 2
333                 + ',' + "jiras/surefire745/TestThree#testSuccess*" // 2
334                 + ',' + "TestFour#testSuccess???" // 2
335                 + ',' + "jiras/surefire745/*Five#test*One"; // 1
336 
337         prepare(test)
338                 .parallel("classes")
339                 .useUnlimitedThreads()
340                 .executeTest()
341                 .verifyErrorFree(9)
342                 .verifyErrorFreeLog();
343     }
344 
345     @Test
346     public void shouldMatchMultiplePatternsComplex() {
347         assumeThat(getSettings().getConfiguration(), is(TEST));
348         String test = "**/BasicTest#testSuccessOne+testSuccessTwo" // 2
349                 + ',' + "jiras/**/TestTwo" // 2
350                 + ',' + "?????/surefire745/TestThree#testSuccess*" // 2
351                 + ',' + "jiras/surefire745/TestFour.java#testSuccess???" // 2
352                 + ',' + "jiras/surefire745/*Five#test*One"; // 1
353 
354         prepare(test).executeTest().verifyErrorFree(9).verifyErrorFreeLog();
355     }
356 
357     @Test
358     public void shouldMatchMultiplePatternsComplexAsParallel() {
359         assumeThat(getSettings().getFramework(), anyOf(is(JUNIT47), is(TestNG)));
360         assumeThat(getSettings().getConfiguration(), is(TEST));
361         String test = "**/BasicTest#testSuccessOne+testSuccessTwo" // 2
362                 + ',' + "jiras/**/TestTwo" // 2
363                 + ',' + "?????/surefire745/TestThree#testSuccess*" // 2
364                 + ',' + "jiras/surefire745/TestFour.java#testSuccess???" // 2
365                 + ',' + "jiras/surefire745/*Five#test*One"; // 1
366 
367         prepare(test)
368                 .parallel("classes")
369                 .useUnlimitedThreads()
370                 .executeTest()
371                 .verifyErrorFree(9)
372                 .verifyErrorFreeLog();
373     }
374 
375     @Test
376     public void shouldNotRunExcludedClasses() {
377         prepare("!BasicTest, !**/TestTwo, !**/TestThree.java")
378                 .executeTest()
379                 .verifyErrorFree(6)
380                 .verifyErrorFreeLog();
381     }
382 
383     @Test
384     public void shouldNotRunExcludedClassesIfIncluded() {
385         prepare("TestF*.java, !**/TestFour.java")
386                 .executeTest()
387                 .verifyErrorFree(3)
388                 .verifyErrorFreeLog();
389     }
390 
391     @Test
392     public void shouldNotRunExcludedMethods() {
393         assumeThat(getSettings().getConfiguration(), is(TEST));
394         prepare("!#*Fail*, !%regex[#.*One], !#testSuccessThree")
395                 .executeTest()
396                 .verifyErrorFree(5)
397                 .verifyErrorFreeLog();
398     }
399 
400     @Test
401     public void shouldNotRunExcludedClassesAndMethods() {
402         assumeThat(getSettings().getConfiguration(), is(TEST));
403         prepare("!#*Fail*, !TestFour#testSuccessTwo")
404                 .executeTest()
405                 .verifyErrorFree(11)
406                 .verifyErrorFreeLog();
407     }
408 
409     @Test
410     public void negativeTest() {
411         assumeThat(
412                 getSettings().getConfiguration(),
413                 anyOf(
414                         is(INCLUDES), is(INCLUDES_EXCLUDES),
415                         is(INCLUDES_FILE), is(INCLUDES_EXCLUDES_FILE)));
416         String pattern = "TestFive#testSuccessOne+testSuccessThree";
417         prepare(pattern)
418                 .failNever()
419                 .executeTest()
420                 .verifyTextInLog("Method filter prohibited in includes|excludes parameter: " + pattern);
421     }
422 }