1 /*
2 * JBoss, Home of Professional Open Source
3 * Copyright 2009, Red Hat Middleware LLC, and individual contributors
4 * by the @authors tag. See the copyright.txt in the distribution for a
5 * full listing of individual contributors.
6 *
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at
10 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
17 package org.apache.maven.surefire.junitcore;
18
19 import java.io.File;
20 import java.util.HashMap;
21 import org.apache.maven.plugin.surefire.report.FileReporterFactory;
22 import org.apache.maven.surefire.booter.StartupReportConfiguration;
23 import org.apache.maven.surefire.report.DefaultConsoleReporter;
24 import org.apache.maven.surefire.report.ReporterConfiguration;
25 import org.apache.maven.surefire.report.ReporterFactory;
26 import org.apache.maven.surefire.report.RunListener;
27
28 import junit.framework.Assert;
29 import junit.framework.TestCase;
30 import org.junit.BeforeClass;
31 import org.junit.Test;
32 import org.junit.runner.Computer;
33 import org.junit.runner.JUnitCore;
34 import org.junit.runner.Result;
35 import org.junit.runner.notification.Failure;
36
37 /**
38 * TestCase that expose "No tests were executed!" on Test failure using Maven Surefire 2.6-SNAPSHOT
39 * and the JUnit 4.7 Runner.
40 * <p/>
41 * -------------------------------------------------------
42 * T E S T S
43 * -------------------------------------------------------
44 * <p/>
45 * Results :
46 * <p/>
47 * Tests run: 0, Failures: 0, Errors: 0, Skipped: 0
48 * <p/>
49 * [INFO] ------------------------------------------------------------------------
50 * [INFO] BUILD FAILURE
51 * [INFO] ------------------------------------------------------------------------
52 * [INFO] Total time: 11.011s
53 * [INFO] Finished at: Thu Jul 15 13:59:14 CEST 2010
54 * [INFO] Final Memory: 24M/355M
55 * [INFO] ------------------------------------------------------------------------
56 * [ERROR] Failed to execute goal org.apache.maven.plugins:maven-surefire-plugin:2.5:test
57 * (default-test) on project xxxxxx: No tests were executed! (Set -DfailIfNoTests=false to
58 * ignore this error.) -> [Help 1]
59 * <p/>
60 * <p/>
61 * <dependency>
62 * <groupId>junit</groupId>
63 * <artifactId>junit</artifactId>
64 * <version>4.8.1</version>
65 * <scope>test</scope>
66 * </dependency>
67 * <p/>
68 * <dependency>
69 * <groupId>org.apache.maven.surefire</groupId>
70 * <artifactId>surefire-booter</artifactId>
71 * <version>2.6-SNAPSHOT</version>
72 * <scope>test</scope>
73 * </dependency>
74 * <dependency>
75 * <groupId>org.apache.maven.plugins</groupId>
76 * <artifactId>maven-surefire-plugin</artifactId>
77 * <version>2.6-SNAPSHOT</version>
78 * <scope>test</scope>
79 * </dependency>
80 * <dependency>
81 * <groupId>org.apache.maven.surefire</groupId>
82 * <artifactId>surefire-junit47</artifactId>
83 * <version>2.6-SNAPSHOT</version>
84 * <scope>test</scope>
85 * </dependency>
86 *
87 * @author <a href="mailto:aslak@redhat.com">Aslak Knutsen</a>
88 * @version $Revision: $
89 */
90 public class MavenSurefireJUnit47RunnerTest
91 extends TestCase
92 {
93
94 /*
95 * Assumption:
96 * The ConcurrentReportingRunListener assumes a Test will be Started before it Fails or Finishes.
97 *
98 * Reality:
99 * JUnits ParentRunner is responsible for adding the BeforeClass/AfterClass statements to the
100 * statement execution chain. After BeforeClass is executed, a Statement that delegates to the
101 * abstract method: runChild(T child, RunNotifier notifier) is called. As the JavaDoc explains:
102 * "Subclasses are responsible for making sure that relevant test events are reported through {@code notifier}".
103 * When a @BeforeClass fail, the child that should handle the relevant test events(Started, Failed, Finished)
104 * is never executed.
105 *
106 * Result:
107 * When Test Failed event is received in ConcurrentReportingRunListener without a Started event received first,
108 * it causes a NullPointException because there is no ClassReporter setup for that class yet. When this Exception
109 * is thrown from the ConcurrentReportingRunListener, JUnit catches the exception and reports is as a Failed test.
110 * But to avoid a wild loop, it removes the failing Listener before calling Failed test again. Since the
111 * ConcurrentReportingRunListener now is removed from the chain it will never receive the RunFinished event
112 * and the recorded state will never be replayed on the ReportManager.
113 *
114 * The End result: ReporterManager falsely believe no Test were run.
115 *
116 */
117 @SuppressWarnings( { "unchecked", "ThrowableResultOfMethodCallIgnored" } )
118 public void testSurefireShouldBeAbleToReportRunStatusEvenWithFailingTests()
119 throws Exception
120 {
121 ReporterFactory reporterManagerFactory = new FileReporterFactory( StartupReportConfiguration.defaultNoXml() );
122
123 final HashMap<String, TestSet> classMethodCounts = new HashMap<String, TestSet>();
124 RunListener reporter =
125 ConcurrentReporterManager.createInstance( classMethodCounts, reporterManagerFactory, false, false,
126 new DefaultConsoleReporter( System.out ) );
127
128 org.junit.runner.notification.RunListener concurrentReportingRunListener =
129 new JUnitCoreRunListener( reporter, classMethodCounts );
130 Computer computer = new Computer();
131
132 JUnitCore junitCore = new JUnitCore();
133
134 junitCore.addListener( concurrentReportingRunListener );
135
136 Result result = junitCore.run( computer, FailingTestClassTestNot.class );
137
138 junitCore.removeListener( concurrentReportingRunListener );
139
140 Assert.assertEquals( "JUnit should report correctly number of test ran(Finished)", 0, result.getRunCount() );
141
142 // Sys.out swallowed in ConsoleReporter..
143 for ( Failure failure : result.getFailures() )
144 {
145 System.out.println( failure.getException().getMessage() );
146 }
147
148 Assert.assertEquals( "There should only be one Exception reported, the one from the failing TestCase", 1,
149 result.getFailureCount() );
150
151 Assert.assertEquals( "The exception thrown by the failing TestCase", RuntimeException.class,
152 result.getFailures().get( 0 ).getException().getClass() );
153
154 reporterManagerFactory.close();
155 }
156
157 private ReporterConfiguration getReporterConfiguration()
158 {
159 return new ReporterConfiguration( new File( "." ), true );
160 }
161
162 /**
163 * Simple TestCase to force a Exception in @BeforeClass.
164 */
165 public static class FailingTestClassTestNot
166 {
167 @BeforeClass
168 public static void failingBeforeClass()
169 throws Exception
170 {
171 throw new RuntimeException( "Opps, we failed in @BeforeClass" );
172 }
173
174 @Test
175 public void shouldNeverBeCalled()
176 throws Exception
177 {
178 Assert.assertTrue( true );
179 }
180 }
181 }