1 package org.apache.maven.plugin.surefire.report;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 import java.io.File;
23 import java.util.ArrayDeque;
24 import java.util.ArrayList;
25 import java.util.Collection;
26 import java.util.List;
27 import java.util.Queue;
28
29 import junit.framework.TestCase;
30
31 import org.apache.maven.plugin.surefire.StartupReportConfiguration;
32 import org.apache.maven.plugin.surefire.extensions.SurefireConsoleOutputReporter;
33 import org.apache.maven.plugin.surefire.extensions.SurefireStatelessReporter;
34 import org.apache.maven.plugin.surefire.extensions.SurefireStatelessTestsetInfoReporter;
35 import org.apache.maven.plugin.surefire.log.api.ConsoleLogger;
36 import org.apache.maven.surefire.shared.utils.logging.MessageUtils;
37 import org.apache.maven.surefire.report.RunStatistics;
38 import org.apache.maven.surefire.api.report.SafeThrowable;
39 import org.apache.maven.surefire.api.report.StackTraceWriter;
40 import org.apache.maven.surefire.api.suite.RunResult;
41
42 import static java.nio.charset.StandardCharsets.UTF_8;
43 import static java.util.Arrays.asList;
44 import static java.util.Collections.emptyList;
45 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.error;
46 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.failure;
47 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.flake;
48 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.skipped;
49 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.success;
50 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.TestResultType.unknown;
51 import static org.apache.maven.plugin.surefire.report.DefaultReporterFactory.getTestResultType;
52 import static org.mockito.Mockito.mock;
53 import static org.mockito.Mockito.when;
54 import static org.powermock.reflect.Whitebox.getInternalState;
55 import static org.powermock.reflect.Whitebox.invokeMethod;
56
57
58
59
60 public class DefaultReporterFactoryTest
61 extends TestCase
62 {
63 private static final String TEST_ONE = "testOne";
64
65 private static final String TEST_TWO = "testTwo";
66
67 private static final String TEST_THREE = "testThree";
68
69 private static final String TEST_FOUR = "testFour";
70
71 private static final String TEST_FIVE = "testFive";
72
73 private static final String ASSERTION_FAIL = "assertionFail";
74
75 private static final String ERROR = "error";
76
77 public void testMergeTestHistoryResult()
78 throws Exception
79 {
80 MessageUtils.setColorEnabled( false );
81 File target = new File( System.getProperty( "user.dir" ), "target" );
82 File reportsDirectory = new File( target, "tmp5" );
83 StartupReportConfiguration reportConfig =
84 new StartupReportConfiguration( true, true, "PLAIN", false, reportsDirectory, false, null,
85 new File( reportsDirectory, "TESTHASH" ), false, 1, null, null, false,
86 new SurefireStatelessReporter(), new SurefireConsoleOutputReporter(),
87 new SurefireStatelessTestsetInfoReporter() );
88
89 DummyTestReporter reporter = new DummyTestReporter();
90
91 DefaultReporterFactory factory = new DefaultReporterFactory( reportConfig, reporter );
92
93
94 Queue<TestMethodStats> firstRunStats = new ArrayDeque<>();
95 firstRunStats.add( new TestMethodStats( TEST_ONE, ReportEntryType.ERROR, new DummyStackTraceWriter( ERROR ) ) );
96 firstRunStats.add( new TestMethodStats( TEST_TWO, ReportEntryType.ERROR, new DummyStackTraceWriter( ERROR ) ) );
97 firstRunStats.add(
98 new TestMethodStats( TEST_THREE, ReportEntryType.FAILURE, new DummyStackTraceWriter( ASSERTION_FAIL ) ) );
99 firstRunStats.add(
100 new TestMethodStats( TEST_FOUR, ReportEntryType.FAILURE, new DummyStackTraceWriter( ASSERTION_FAIL ) ) );
101 firstRunStats.add(
102 new TestMethodStats( TEST_FIVE, ReportEntryType.SUCCESS, null ) );
103
104
105 Queue<TestMethodStats> secondRunStats = new ArrayDeque<>();
106 secondRunStats.add(
107 new TestMethodStats( TEST_ONE, ReportEntryType.FAILURE, new DummyStackTraceWriter( ASSERTION_FAIL ) ) );
108 secondRunStats.add( new TestMethodStats( TEST_TWO, ReportEntryType.SUCCESS, null ) );
109 secondRunStats.add(
110 new TestMethodStats( TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter( ERROR ) ) );
111 secondRunStats.add( new TestMethodStats( TEST_FOUR, ReportEntryType.SUCCESS, null ) );
112
113
114 Queue<TestMethodStats> thirdRunStats = new ArrayDeque<>();
115 thirdRunStats.add( new TestMethodStats( TEST_ONE, ReportEntryType.SUCCESS, null ) );
116 thirdRunStats.add(
117 new TestMethodStats( TEST_THREE, ReportEntryType.ERROR, new DummyStackTraceWriter( ERROR ) ) );
118
119 TestSetRunListener firstRunListener = mock( TestSetRunListener.class );
120 TestSetRunListener secondRunListener = mock( TestSetRunListener.class );
121 TestSetRunListener thirdRunListener = mock( TestSetRunListener.class );
122 when( firstRunListener.getTestMethodStats() ).thenReturn( firstRunStats );
123 when( secondRunListener.getTestMethodStats() ).thenReturn( secondRunStats );
124 when( thirdRunListener.getTestMethodStats() ).thenReturn( thirdRunStats );
125
126 factory.addListener( firstRunListener );
127 factory.addListener( secondRunListener );
128 factory.addListener( thirdRunListener );
129
130 invokeMethod( factory, "mergeTestHistoryResult" );
131 RunStatistics mergedStatistics = factory.getGlobalRunStatistics();
132
133
134 assertEquals( 5, mergedStatistics.getCompletedCount() );
135 assertEquals( 1, mergedStatistics.getErrors() );
136 assertEquals( 0, mergedStatistics.getFailures() );
137 assertEquals( 3, mergedStatistics.getFlakes() );
138 assertEquals( 0, mergedStatistics.getSkipped() );
139
140
141 factory.printTestFailures( flake );
142 String[] expectedFlakeOutput =
143 { "Flakes: ", TEST_FOUR, " Run 1: " + ASSERTION_FAIL, " Run 2: PASS", "", TEST_ONE,
144 " Run 1: " + ERROR, " Run 2: " + ASSERTION_FAIL, " Run 3: PASS", "", TEST_TWO, " Run 1: " + ERROR,
145 " Run 2: PASS", "" };
146 assertEquals( asList( expectedFlakeOutput ), reporter.getMessages() );
147
148 reporter.reset();
149 factory.printTestFailures( error );
150 String[] expectedFailureOutput =
151 { "Errors: ", TEST_THREE, " Run 1: " + ASSERTION_FAIL, " Run 2: " + ERROR, " Run 3: " + ERROR, "" };
152 assertEquals( asList( expectedFailureOutput ), reporter.getMessages() );
153
154 reporter.reset();
155 factory.printTestFailures( failure );
156 assertEquals( emptyList(), reporter.getMessages() );
157 }
158
159 static final class DummyTestReporter implements ConsoleLogger
160 {
161 private final List<String> messages = new ArrayList<>();
162
163 @Override
164 public boolean isDebugEnabled()
165 {
166 return true;
167 }
168
169 @Override
170 public void debug( String message )
171 {
172 messages.add( message );
173 }
174
175 @Override
176 public boolean isInfoEnabled()
177 {
178 return true;
179 }
180
181 @Override
182 public void info( String message )
183 {
184 messages.add( message );
185 }
186
187 @Override
188 public boolean isWarnEnabled()
189 {
190 return true;
191 }
192
193 @Override
194 public void warning( String message )
195 {
196 messages.add( message );
197 }
198
199 @Override
200 public boolean isErrorEnabled()
201 {
202 return true;
203 }
204
205 @Override
206 public void error( String message )
207 {
208 messages.add( message );
209 }
210
211 @Override
212 public void error( String message, Throwable t )
213 {
214 messages.add( message + " " + t.getLocalizedMessage() );
215 }
216
217 @Override
218 public void error( Throwable t )
219 {
220 messages.add( t.getLocalizedMessage() );
221 }
222
223 List<String> getMessages()
224 {
225 return messages;
226 }
227
228 void reset()
229 {
230 messages.clear();
231 }
232 }
233
234 public void testGetTestResultType()
235 {
236 List<ReportEntryType> emptyList = new ArrayList<>();
237 assertEquals( unknown, getTestResultType( emptyList, 1 ) );
238
239 List<ReportEntryType> successList = new ArrayList<>();
240 successList.add( ReportEntryType.SUCCESS );
241 successList.add( ReportEntryType.SUCCESS );
242 assertEquals( success, getTestResultType( successList, 1 ) );
243
244 List<ReportEntryType> failureErrorList = new ArrayList<>();
245 failureErrorList.add( ReportEntryType.FAILURE );
246 failureErrorList.add( ReportEntryType.ERROR );
247 assertEquals( error, getTestResultType( failureErrorList, 1 ) );
248
249 List<ReportEntryType> errorFailureList = new ArrayList<>();
250 errorFailureList.add( ReportEntryType.ERROR );
251 errorFailureList.add( ReportEntryType.FAILURE );
252 assertEquals( error, getTestResultType( errorFailureList, 1 ) );
253
254 List<ReportEntryType> flakeList = new ArrayList<>();
255 flakeList.add( ReportEntryType.SUCCESS );
256 flakeList.add( ReportEntryType.FAILURE );
257 assertEquals( flake, getTestResultType( flakeList, 1 ) );
258
259 assertEquals( failure, getTestResultType( flakeList, 0 ) );
260
261 flakeList = new ArrayList<>();
262 flakeList.add( ReportEntryType.ERROR );
263 flakeList.add( ReportEntryType.SUCCESS );
264 flakeList.add( ReportEntryType.FAILURE );
265 assertEquals( flake, getTestResultType( flakeList, 1 ) );
266
267 assertEquals( error, getTestResultType( flakeList, 0 ) );
268
269 List<ReportEntryType> skippedList = new ArrayList<>();
270 skippedList.add( ReportEntryType.SKIPPED );
271 assertEquals( skipped, getTestResultType( skippedList, 1 ) );
272 }
273
274 public void testLogger()
275 {
276 MessageUtils.setColorEnabled( false );
277 File target = new File( System.getProperty( "user.dir" ), "target" );
278 File reportsDirectory = new File( target, "tmp6" );
279 StartupReportConfiguration reportConfig =
280 new StartupReportConfiguration( true, true, "PLAIN", false, reportsDirectory, false, null,
281 new File( reportsDirectory, "TESTHASH" ), false, 1, null, null, false,
282 new SurefireStatelessReporter(), new SurefireConsoleOutputReporter(),
283 new SurefireStatelessTestsetInfoReporter() );
284
285 DummyTestReporter reporter = new DummyTestReporter();
286
287 DefaultReporterFactory factory = new DefaultReporterFactory( reportConfig, reporter );
288
289 TestSetRunListener runListener = (TestSetRunListener) factory.createReporter();
290
291 assertTrue( runListener.isDebugEnabled() );
292 assertTrue( runListener.isInfoEnabled() );
293 assertTrue( runListener.isWarnEnabled() );
294 assertTrue( runListener.isErrorEnabled() );
295
296 runListener.debug( "msg" );
297 assertEquals( 1, reporter.getMessages().size() );
298 assertEquals( "msg", reporter.getMessages().get( 0 ) );
299 reporter.reset();
300
301 runListener.info( "msg\n" );
302 assertEquals( 1, reporter.getMessages().size() );
303 assertEquals( "msg", reporter.getMessages().get( 0 ) );
304 reporter.reset();
305
306 runListener.warning( "msg\r\n" );
307 assertEquals( 1, reporter.getMessages().size() );
308 assertEquals( "msg", reporter.getMessages().get( 0 ) );
309 reporter.reset();
310
311 runListener.error( "msg" );
312 assertEquals( 1, reporter.getMessages().size() );
313 assertEquals( "msg", reporter.getMessages().get( 0 ) );
314 reporter.reset();
315
316 runListener.error( "msg\n", new Exception( "e" ) );
317 assertEquals( 1, reporter.getMessages().size() );
318 assertEquals( "msg e", reporter.getMessages().get( 0 ) );
319 reporter.reset();
320
321 runListener.error( new Exception( "e" ) );
322 assertEquals( 1, reporter.getMessages().size() );
323 assertEquals( "e", reporter.getMessages().get( 0 ) );
324 reporter.reset();
325 }
326
327 public void testCreateReporterWithZeroStatistics()
328 {
329 MessageUtils.setColorEnabled( false );
330 File target = new File( System.getProperty( "user.dir" ), "target" );
331 File reportsDirectory = new File( target, "tmp7" );
332 StartupReportConfiguration reportConfig =
333 new StartupReportConfiguration( true, true, "PLAIN", false, reportsDirectory, false, null,
334 new File( reportsDirectory, "TESTHASH" ), false, 0, null, null, false,
335 new SurefireStatelessReporter(), new SurefireConsoleOutputReporter(),
336 new SurefireStatelessTestsetInfoReporter() );
337
338 assertTrue( reportConfig.isUseFile() );
339 assertTrue( reportConfig.isPrintSummary() );
340 assertEquals( "PLAIN", reportConfig.getReportFormat() );
341 assertFalse( reportConfig.isRedirectTestOutputToFile() );
342 assertEquals( reportsDirectory, reportConfig.getReportsDirectory() );
343 assertFalse( reportConfig.isTrimStackTrace() );
344 assertNull( reportConfig.getReportNameSuffix() );
345 assertEquals( new File( reportsDirectory, "TESTHASH" ), reportConfig.getStatisticsFile() );
346 assertFalse( reportConfig.isRequiresRunHistory() );
347 assertEquals( 0, reportConfig.getRerunFailingTestsCount() );
348 assertNull( reportConfig.getXsdSchemaLocation() );
349 assertEquals( UTF_8, reportConfig.getEncoding() );
350 assertFalse( reportConfig.isForkMode() );
351 assertNotNull( reportConfig.getXmlReporter() );
352 assertNotNull( reportConfig.getConsoleOutputReporter() );
353 assertNotNull( reportConfig.getTestsetReporter() );
354 assertNull( reportConfig.getStatisticsReporter() );
355
356 DummyTestReporter reporter = new DummyTestReporter();
357
358 DefaultReporterFactory factory = new DefaultReporterFactory( reportConfig, reporter );
359 assertEquals( reportsDirectory, factory.getReportsDirectory() );
360
361 TestSetRunListener runListener = (TestSetRunListener) factory.createReporter();
362 Collection listeners = getInternalState( factory, "listeners" );
363 assertEquals( 1, listeners.size() );
364 assertTrue( listeners.contains( runListener ) );
365
366 assertNotNull( runListener.getTestMethodStats() );
367
368 factory.runStarting();
369
370 factory.close();
371
372 RunStatistics statistics = factory.getGlobalRunStatistics();
373 assertEquals( 0, statistics.getCompletedCount() );
374 assertEquals( new RunResult( 0, 0, 0, 0 ), statistics.getRunResult() );
375 assertEquals( 0, statistics.getFailures() );
376 assertEquals( 0, statistics.getErrors() );
377 assertEquals( 0, statistics.getSkipped() );
378 assertEquals( 0, statistics.getFlakes() );
379 assertEquals( "Tests run: 0, Failures: 0, Errors: 0, Skipped: 0", statistics.getSummary() );
380 assertEquals( 0, statistics.getCompletedCount() );
381
382 List<String> messages = reporter.getMessages();
383 assertEquals( "", messages.get( 0 ) );
384 assertEquals( "-------------------------------------------------------", messages.get( 1 ) );
385 assertEquals( " T E S T S", messages.get( 2 ) );
386 assertEquals( "-------------------------------------------------------", messages.get( 3 ) );
387 assertEquals( "", messages.get( 4 ) );
388 assertEquals( "Results:", messages.get( 5 ) );
389 assertEquals( "", messages.get( 6 ) );
390 assertEquals( "Tests run: 0, Failures: 0, Errors: 0, Skipped: 0", messages.get( 7 ) );
391 assertEquals( "", messages.get( 8 ) );
392 assertEquals( 9, messages.size() );
393 }
394
395 static class DummyStackTraceWriter
396 implements StackTraceWriter
397 {
398
399 private final String stackTrace;
400
401 DummyStackTraceWriter( String stackTrace )
402 {
403 this.stackTrace = stackTrace;
404 }
405
406 @Override
407 public String writeTraceToString()
408 {
409 return "";
410 }
411
412 @Override
413 public String writeTrimmedTraceToString()
414 {
415 return "";
416 }
417
418 @Override
419 public String smartTrimmedStackTrace()
420 {
421 return stackTrace;
422 }
423
424 @Override
425 public SafeThrowable getThrowable()
426 {
427 return null;
428 }
429 }
430 }