View Javadoc
1   package org.apache.maven.surefire.its;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import org.apache.maven.surefire.its.fixture.OutputValidator;
23  import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
24  import org.apache.maven.surefire.its.fixture.SurefireLauncher;
25  import org.junit.Test;
26  
27  /**
28   * JUnit4 RunListener Integration Test.
29   *
30   * @author <a href="mailto:qingzhouluo@google.com">Qingzhou Luo</a>
31   */
32  public class JUnit4RerunFailingTestsIT
33      extends SurefireJUnit4IntegrationTestCase
34  {
35      private SurefireLauncher unpack()
36      {
37          return unpack( "/junit4-rerun-failing-tests" );
38      }
39  
40      @Test
41      public void testRerunFailingErrorTestsWithOneRetry()
42          throws Exception
43      {
44          OutputValidator outputValidator =
45              unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
46                  "-Dsurefire.rerunFailingTestsCount=1" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0,
47                                                                                                              0 );
48          verifyFailuresOneRetryAllClasses( outputValidator );
49  
50          outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
51              "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
52              "-DforkCount=2" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
53          verifyFailuresOneRetryAllClasses( outputValidator );
54  
55          outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
56              "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
57              "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
58          verifyFailuresOneRetryAllClasses( outputValidator );
59  
60          outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
61              "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
62              "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
63          verifyFailuresOneRetryAllClasses( outputValidator );
64      }
65  
66      @Test
67      public void testRerunFailingErrorTestsTwoRetry()
68          throws Exception
69      {
70          // Four flakes, both tests have been re-run twice
71          OutputValidator outputValidator =
72              unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
73                  "-Dsurefire.rerunFailingTestsCount=2" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
74  
75          verifyFailuresTwoRetryAllClasses( outputValidator );
76  
77          outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
78              "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-DforkCount=3" ).executeTest()
79              .assertTestSuiteResults( 5, 0, 0, 0, 4 );
80  
81          verifyFailuresTwoRetryAllClasses( outputValidator );
82  
83          outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
84              "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=methods" ).addGoal(
85              "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
86  
87          verifyFailuresTwoRetryAllClasses( outputValidator );
88  
89          outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
90              "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=classes" ).addGoal(
91              "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
92  
93          verifyFailuresTwoRetryAllClasses( outputValidator );
94      }
95  
96      @Test
97      public void testRerunFailingErrorTestsFalse()
98          throws Exception
99      {
100         OutputValidator outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion(
101             "4.12" ).maven().withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
102 
103         verifyFailuresNoRetryAllClasses( outputValidator );
104 
105         outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
106             "-DforkCount=3" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
107 
108         verifyFailuresNoRetryAllClasses( outputValidator );
109 
110         outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
111             "-Dparallel=methods" ).addGoal(
112             "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
113 
114         verifyFailuresNoRetryAllClasses( outputValidator );
115 
116         outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
117             "-Dparallel=classes" ).addGoal(
118             "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
119 
120         verifyFailuresNoRetryAllClasses( outputValidator );
121     }
122 
123     @Test
124     public void testRerunOneTestClass()
125         throws Exception
126     {
127         OutputValidator outputValidator =
128             unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
129                 "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
130                 "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
131 
132         verifyFailuresOneRetryOneClass( outputValidator );
133 
134         outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
135             "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal(
136             "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
137 
138         verifyFailuresOneRetryOneClass( outputValidator );
139 
140         outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
141             "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
142             "-DuseUnlimitedThreads=true" ).addGoal(
143             "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
144 
145         verifyFailuresOneRetryOneClass( outputValidator );
146 
147         outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
148             "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
149             "-DuseUnlimitedThreads=true" ).addGoal(
150             "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
151 
152         verifyFailuresOneRetryOneClass( outputValidator );
153     }
154 
155     @Test
156     public void testRerunOneTestMethod()
157         throws Exception
158     {
159         OutputValidator outputValidator =
160             unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
161                 "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
162                 "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1,
163                                                                                                                0, 0 );
164 
165         verifyFailuresOneRetryOneMethod( outputValidator );
166 
167         outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
168             "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal(
169             "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1, 0,
170                                                                                                            0 );
171 
172         verifyFailuresOneRetryOneMethod( outputValidator );
173 
174         outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
175             "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
176             "-DuseUnlimitedThreads=true" ).addGoal(
177             "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1, 0,
178                                                                                                            0 );
179 
180         verifyFailuresOneRetryOneMethod( outputValidator );
181 
182         outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
183             "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
184             "-DuseUnlimitedThreads=true" ).addGoal(
185             "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1, 0,
186                                                                                                            0 );
187 
188         verifyFailuresOneRetryOneMethod( outputValidator );
189     }
190 
191     private void verifyFailuresOneRetryAllClasses( OutputValidator outputValidator )
192     {
193         verifyFailuresOneRetry( outputValidator, 5, 1, 1, 0 );
194     }
195 
196     private void verifyFailuresTwoRetryAllClasses( OutputValidator outputValidator )
197     {
198         verifyFailuresTwoRetry( outputValidator, 5, 0, 0, 2 );
199     }
200 
201     private void verifyFailuresNoRetryAllClasses( OutputValidator outputValidator )
202     {
203         verifyFailuresNoRetry( outputValidator, 5, 1, 1, 0 );
204     }
205 
206     private void verifyFailuresOneRetryOneClass( OutputValidator outputValidator )
207     {
208         verifyFailuresOneRetry( outputValidator, 3, 1, 1, 0 );
209     }
210 
211     private void verifyFailuresOneRetryOneMethod( OutputValidator outputValidator )
212     {
213         verifyOnlyFailuresOneRetry( outputValidator, 1, 1, 0, 0 );
214     }
215 
216     private void verifyFailuresOneRetry( OutputValidator outputValidator, int run, int failures, int errors,
217                                          int flakes )
218     {
219         outputValidator.verifyTextInLog( "Failures:" );
220         outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
221         outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
222 
223         outputValidator.verifyTextInLog( "Errors:" );
224         outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" );
225         outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" );
226 
227         verifyStatistics( outputValidator, run, failures, errors, flakes );
228     }
229 
230     private void verifyOnlyFailuresOneRetry( OutputValidator outputValidator, int run, int failures, int errors,
231                                              int flakes )
232     {
233         outputValidator.verifyTextInLog( "Failures:" );
234         outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
235         outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
236 
237         verifyStatistics( outputValidator, run, failures, errors, flakes );
238     }
239 
240     private void verifyFailuresTwoRetry( OutputValidator outputValidator, int run, int failures, int errors,
241                                          int flakes )
242     {
243         outputValidator.verifyTextInLog( "Flakes:" );
244         outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
245         outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
246         outputValidator.verifyTextInLog( "Run 3: PASS" );
247 
248         outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" );
249         outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" );
250 
251         verifyStatistics( outputValidator, run, failures, errors, flakes );
252     }
253 
254     private void verifyFailuresNoRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
255     {
256         outputValidator.verifyTextInLog( "Failures:" );
257         outputValidator.verifyTextInLog( "testFailingTestOne(junit4.FlakyFirstTimeTest)" );
258         outputValidator.verifyTextInLog( "ERROR" );
259         outputValidator.verifyTextInLog( "testErrorTestOne(junit4.FlakyFirstTimeTest)" );
260 
261         verifyStatistics( outputValidator, run, failures, errors, flakes );
262     }
263 
264     private void verifyStatistics( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
265     {
266         if ( flakes > 0 )
267         {
268             outputValidator.verifyTextInLog(
269                 "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: 0, Flakes: "
270                     + flakes );
271         }
272         else
273         {
274             outputValidator.verifyTextInLog(
275                 "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: 0" );
276         }
277     }
278 }