1 package org.apache.maven.surefire.its;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 import org.apache.maven.surefire.its.fixture.OutputValidator;
23 import org.apache.maven.surefire.its.fixture.SurefireJUnit4IntegrationTestCase;
24 import org.apache.maven.surefire.its.fixture.SurefireLauncher;
25 import org.junit.Test;
26
27
28
29
30
31
32 public class JUnit4RerunFailingTestsIT
33 extends SurefireJUnit4IntegrationTestCase
34 {
35 private SurefireLauncher unpack()
36 {
37 return unpack( "/junit4-rerun-failing-tests" );
38 }
39
40 @Test
41 public void testRerunFailingErrorTestsWithOneRetry()
42 throws Exception
43 {
44 OutputValidator outputValidator =
45 unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
46 "-Dsurefire.rerunFailingTestsCount=1" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0,
47 0 );
48 verifyFailuresOneRetryAllClasses( outputValidator );
49
50 outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
51 "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
52 "-DforkCount=2" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
53 verifyFailuresOneRetryAllClasses( outputValidator );
54
55 outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
56 "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
57 "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
58 verifyFailuresOneRetryAllClasses( outputValidator );
59
60 outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
61 "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
62 "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
63 verifyFailuresOneRetryAllClasses( outputValidator );
64 }
65
66 @Test
67 public void testRerunFailingErrorTestsTwoRetry()
68 throws Exception
69 {
70
71 OutputValidator outputValidator =
72 unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
73 "-Dsurefire.rerunFailingTestsCount=2" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
74
75 verifyFailuresTwoRetryAllClasses( outputValidator );
76
77 outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
78 "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-DforkCount=3" ).executeTest()
79 .assertTestSuiteResults( 5, 0, 0, 0, 4 );
80
81 verifyFailuresTwoRetryAllClasses( outputValidator );
82
83 outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
84 "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=methods" ).addGoal(
85 "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
86
87 verifyFailuresTwoRetryAllClasses( outputValidator );
88
89 outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
90 "-Dsurefire.rerunFailingTestsCount=2" ).addGoal( "-Dparallel=classes" ).addGoal(
91 "-DuseUnlimitedThreads=true" ).executeTest().assertTestSuiteResults( 5, 0, 0, 0, 4 );
92
93 verifyFailuresTwoRetryAllClasses( outputValidator );
94 }
95
96 @Test
97 public void testRerunFailingErrorTestsFalse()
98 throws Exception
99 {
100 OutputValidator outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion(
101 "4.12" ).maven().withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
102
103 verifyFailuresNoRetryAllClasses( outputValidator );
104
105 outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
106 "-DforkCount=3" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
107
108 verifyFailuresNoRetryAllClasses( outputValidator );
109
110 outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
111 "-Dparallel=methods" ).addGoal(
112 "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
113
114 verifyFailuresNoRetryAllClasses( outputValidator );
115
116 outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
117 "-Dparallel=classes" ).addGoal(
118 "-DuseUnlimitedThreads=true" ).withFailure().executeTest().assertTestSuiteResults( 5, 1, 1, 0, 0 );
119
120 verifyFailuresNoRetryAllClasses( outputValidator );
121 }
122
123 @Test
124 public void testRerunOneTestClass()
125 throws Exception
126 {
127 OutputValidator outputValidator =
128 unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
129 "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
130 "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
131
132 verifyFailuresOneRetryOneClass( outputValidator );
133
134 outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
135 "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal(
136 "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
137
138 verifyFailuresOneRetryOneClass( outputValidator );
139
140 outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
141 "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
142 "-DuseUnlimitedThreads=true" ).addGoal(
143 "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
144
145 verifyFailuresOneRetryOneClass( outputValidator );
146
147 outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
148 "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
149 "-DuseUnlimitedThreads=true" ).addGoal(
150 "-Dtest=FlakyFirstTimeTest" ).withFailure().executeTest().assertTestSuiteResults( 3, 1, 1, 0, 0 );
151
152 verifyFailuresOneRetryOneClass( outputValidator );
153 }
154
155 @Test
156 public void testRerunOneTestMethod()
157 throws Exception
158 {
159 OutputValidator outputValidator =
160 unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
161 "-Dsurefire.rerunFailingTestsCount=1" ).addGoal(
162 "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1,
163 0, 0 );
164
165 verifyFailuresOneRetryOneMethod( outputValidator );
166
167 outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
168 "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-DforkCount=3" ).addGoal(
169 "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1, 0,
170 0 );
171
172 verifyFailuresOneRetryOneMethod( outputValidator );
173
174 outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
175 "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=methods" ).addGoal(
176 "-DuseUnlimitedThreads=true" ).addGoal(
177 "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1, 0,
178 0 );
179
180 verifyFailuresOneRetryOneMethod( outputValidator );
181
182 outputValidator = unpack().addGoal( "-Dprovider=surefire-junit4" ).setJUnitVersion( "4.12" ).maven().addGoal(
183 "-Dsurefire.rerunFailingTestsCount=1" ).addGoal( "-Dparallel=classes" ).addGoal(
184 "-DuseUnlimitedThreads=true" ).addGoal(
185 "-Dtest=FlakyFirstTimeTest#testFailing*" ).withFailure().executeTest().assertTestSuiteResults( 1, 0, 1, 0,
186 0 );
187
188 verifyFailuresOneRetryOneMethod( outputValidator );
189 }
190
191 private void verifyFailuresOneRetryAllClasses( OutputValidator outputValidator )
192 {
193 verifyFailuresOneRetry( outputValidator, 5, 1, 1, 0 );
194 }
195
196 private void verifyFailuresTwoRetryAllClasses( OutputValidator outputValidator )
197 {
198 verifyFailuresTwoRetry( outputValidator, 5, 0, 0, 2 );
199 }
200
201 private void verifyFailuresNoRetryAllClasses( OutputValidator outputValidator )
202 {
203 verifyFailuresNoRetry( outputValidator, 5, 1, 1, 0 );
204 }
205
206 private void verifyFailuresOneRetryOneClass( OutputValidator outputValidator )
207 {
208 verifyFailuresOneRetry( outputValidator, 3, 1, 1, 0 );
209 }
210
211 private void verifyFailuresOneRetryOneMethod( OutputValidator outputValidator )
212 {
213 verifyOnlyFailuresOneRetry( outputValidator, 1, 1, 0, 0 );
214 }
215
216 private void verifyFailuresOneRetry( OutputValidator outputValidator, int run, int failures, int errors,
217 int flakes )
218 {
219 outputValidator.verifyTextInLog( "Failures:" );
220 outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
221 outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
222
223 outputValidator.verifyTextInLog( "Errors:" );
224 outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" );
225 outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" );
226
227 verifyStatistics( outputValidator, run, failures, errors, flakes );
228 }
229
230 private void verifyOnlyFailuresOneRetry( OutputValidator outputValidator, int run, int failures, int errors,
231 int flakes )
232 {
233 outputValidator.verifyTextInLog( "Failures:" );
234 outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
235 outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
236
237 verifyStatistics( outputValidator, run, failures, errors, flakes );
238 }
239
240 private void verifyFailuresTwoRetry( OutputValidator outputValidator, int run, int failures, int errors,
241 int flakes )
242 {
243 outputValidator.verifyTextInLog( "Flakes:" );
244 outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testFailingTestOne" );
245 outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testFailingTestOne" );
246 outputValidator.verifyTextInLog( "Run 3: PASS" );
247
248 outputValidator.verifyTextInLog( "Run 1: FlakyFirstTimeTest.testErrorTestOne" );
249 outputValidator.verifyTextInLog( "Run 2: FlakyFirstTimeTest.testErrorTestOne" );
250
251 verifyStatistics( outputValidator, run, failures, errors, flakes );
252 }
253
254 private void verifyFailuresNoRetry( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
255 {
256 outputValidator.verifyTextInLog( "Failures:" );
257 outputValidator.verifyTextInLog( "testFailingTestOne(junit4.FlakyFirstTimeTest)" );
258 outputValidator.verifyTextInLog( "ERROR" );
259 outputValidator.verifyTextInLog( "testErrorTestOne(junit4.FlakyFirstTimeTest)" );
260
261 verifyStatistics( outputValidator, run, failures, errors, flakes );
262 }
263
264 private void verifyStatistics( OutputValidator outputValidator, int run, int failures, int errors, int flakes )
265 {
266 if ( flakes > 0 )
267 {
268 outputValidator.verifyTextInLog(
269 "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: 0, Flakes: "
270 + flakes );
271 }
272 else
273 {
274 outputValidator.verifyTextInLog(
275 "Tests run: " + run + ", Failures: " + failures + ", Errors: " + errors + ", Skipped: 0" );
276 }
277 }
278 }