/*
* Copyright (C) 2016 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.test.internal.events.client;
import static androidx.test.internal.util.Checks.checkNotNull;
import static androidx.test.services.events.ParcelableConverter.getFailure;
import static androidx.test.services.events.ParcelableConverter.getFailuresFromList;
import static androidx.test.services.events.ParcelableConverter.getTestCaseFromDescription;
import static java.util.Collections.emptyList;
import android.util.Log;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.test.services.events.FailureInfo;
import androidx.test.services.events.TestCaseInfo;
import androidx.test.services.events.TestEventException;
import androidx.test.services.events.run.TestAssumptionFailureEvent;
import androidx.test.services.events.run.TestFailureEvent;
import androidx.test.services.events.run.TestFinishedEvent;
import androidx.test.services.events.run.TestIgnoredEvent;
import androidx.test.services.events.run.TestRunFinishedEvent;
import androidx.test.services.events.run.TestRunStartedEvent;
import androidx.test.services.events.run.TestStartedEvent;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.runner.Description;
import org.junit.runner.Result;
import org.junit.runner.notification.Failure;
import org.junit.runner.notification.RunListener;
/**
* A {@link RunListener} for the orchestrated instrumentation to communicate test run notifications
* back to the remote {@link androidx.test.orchestrator.OrchestratorService} or {@link
* androidx.test.services.events.run.ITestRunEvent} service.
*/
public final class OrchestratedInstrumentationListener extends RunListener {
// TODO(b/161754141): replace references to the word "orchestrator" with "test event service"
private static final String TAG = "OrchestrationListener";
private final TestRunEventService notificationService;
private final AtomicBoolean isTestFailed = new AtomicBoolean(false);
private Description description = Description.EMPTY; // Cached test description
/**
* Creates the {@link OrchestratedInstrumentationListener} to communicate with the remote test run
* events service.
*
* @param notificationService the remote service to send test run events to
*/
public OrchestratedInstrumentationListener(@NonNull TestRunEventService notificationService) {
super();
checkNotNull(notificationService, "notificationService cannot be null");
this.notificationService = notificationService;
}
/** {@inheritDoc} */
@Override
public void testRunStarted(Description description) {
try {
notificationService.send(new TestRunStartedEvent(getTestCaseFromDescription(description)));
} catch (TestEventException e) {
Log.e(TAG, "Unable to send TestRunStartedEvent to Orchestrator", e);
}
}
/** {@inheritDoc} */
@Override
public void testRunFinished(Result result) {
List<FailureInfo> failures = emptyList();
try {
failures = getFailuresFromList(result.getFailures());
} catch (TestEventException e) {
Log.w(TAG, "Failure event doesn't contain a test case", e);
}
try {
notificationService.send(
new TestRunFinishedEvent(
result.getRunCount(), result.getIgnoreCount(), result.getRunTime(), failures));
} catch (TestEventException e) {
Log.e(TAG, "Unable to send TestRunFinishedEvent to Orchestrator", e);
}
}
/** {@inheritDoc} */
@Override
public void testStarted(Description description) {
this.description = description; // Caches the test description in case of a crash
if (!JUnitValidator.validateDescription(description)) {
Log.w(
TAG,
"testStarted: JUnit reported "
+ description.getClassName()
+ "#"
+ description.getMethodName()
+ "; discarding as bogus.");
return;
}
try {
notificationService.send(new TestStartedEvent(getTestCaseFromDescription(description)));
} catch (TestEventException e) {
Log.e(TAG, "Unable to send TestStartedEvent to Orchestrator", e);
}
}
/** {@inheritDoc} */
@Override
public void testFinished(Description description) {
if (!JUnitValidator.validateDescription(description)) {
Log.w(
TAG,
"testFinished: JUnit reported "
+ description.getClassName()
+ "#"
+ description.getMethodName()
+ "; discarding as bogus.");
return;
}
try {
notificationService.send(new TestFinishedEvent(getTestCaseFromDescription(description)));
} catch (TestEventException e) {
Log.e(TAG, "Unable to send TestFinishedEvent to Orchestrator", e);
}
}
/** {@inheritDoc} */
@Override
public void testFailure(Failure failure) {
// This block can be called by the JUnit test framework when a failure happened in the test,
// or {@link #reportProcessCrash(Throwable)} when we'd like to report a process crash as a
// failure.
// We'd like to make sure only one failure gets sent so that the isTestFailed variable is
// checked and set without possibly racing between two thread calls.
if (isTestFailed.compareAndSet(false, true)) {
Description description = failure.getDescription();
if (!JUnitValidator.validateDescription(description)) {
// The call stack from failure.getException() will be logged by the LogRunListener; look for
// the "TestRunner" tag in the logcat.
Log.w(
TAG,
"testFailure: JUnit reported "
+ description.getClassName()
+ "#"
+ description.getMethodName()
+ "; discarding as bogus.");
return;
}
TestFailureEvent event;
try {
event =
new TestFailureEvent(
getTestCaseFromDescription(failure.getDescription()), getFailure(failure));
} catch (TestEventException e) {
Log.d(TAG, "Unable to determine test case from failure [" + failure + "]", e);
event = getTestFailureEventFromCachedDescription(failure);
if (event == null) {
return;
}
}
try {
notificationService.send(event);
} catch (TestEventException e) {
throw new IllegalStateException("Unable to send TestFailureEvent, terminating", e);
}
}
}
@Nullable
private TestFailureEvent getTestFailureEventFromCachedDescription(@NonNull Failure failure) {
checkNotNull(failure, "failure cannot be null");
TestCaseInfo testCase;
try {
// Get the testCase from the cached description instead.
testCase = getTestCaseFromDescription(description);
} catch (TestEventException ex) {
Log.e(TAG, "Unable to determine test case from description [" + description + "]", ex);
return null;
}
return new TestFailureEvent(
testCase,
new FailureInfo(
failure.getMessage(), failure.getTestHeader(), failure.getTrace(), testCase));
}
/** {@inheritDoc} */
@Override
public void testAssumptionFailure(Failure failure) {
try {
notificationService.send(
new TestAssumptionFailureEvent(
getTestCaseFromDescription(failure.getDescription()), getFailure(failure)));
} catch (TestEventException e) {
Log.e(TAG, "Unable to send TestAssumptionFailureEvent to Orchestrator", e);
}
}
/** {@inheritDoc} */
@Override
public void testIgnored(Description description) {
try {
TestCaseInfo info = getTestCaseFromDescription(description);
Log.i(
TAG,
"TestIgnoredEvent("
+ description.getDisplayName()
+ "): "
+ description.getClassName()
+ "#"
+ description.getMethodName()
+ " = "
+ info.getClassAndMethodName());
notificationService.send(new TestIgnoredEvent(info));
} catch (TestEventException e) {
Log.e(TAG, "Unable to send TestIgnoredEvent to Orchestrator", e);
}
}
/** Reports the process crash event with a given exception. */
public boolean reportProcessCrash(Throwable t, long timeoutMillis) {
// Need to report the process crashed event to the orchestrator.
// This is to handle the case when the test body finishes but process crashes during
// Instrumentation cleanup (e.g. stopping the app). Otherwise, the test will be marked as
// passed.
if (!isTestFailed.get()) {
Log.i(TAG, "No test failure has been reported. Report the process crash.");
reportProcessCrash(t);
return true;
}
return false;
}
/** Reports the process crash event with a given exception. */
private void reportProcessCrash(Throwable t) {
testFailure(new Failure(description, t));
testFinished(description);
}
}