TestPlatformListener.java

/*
 * Copyright (C) 2021 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package androidx.test.internal.events.client;

import static androidx.test.internal.util.Checks.checkNotNull;

import android.util.Log;
import androidx.annotation.NonNull;
import androidx.test.services.events.ErrorInfo;
import androidx.test.services.events.ParcelableConverter;
import androidx.test.services.events.TestCaseInfo;
import androidx.test.services.events.TestEventException;
import androidx.test.services.events.TestRunInfo;
import androidx.test.services.events.TestStatus;
import androidx.test.services.events.TestStatus.Status;
import androidx.test.services.events.TimeStamp;
import androidx.test.services.events.platform.TestCaseErrorEvent;
import androidx.test.services.events.platform.TestCaseFinishedEvent;
import androidx.test.services.events.platform.TestCaseStartedEvent;
import androidx.test.services.events.platform.TestPlatformEvent;
import androidx.test.services.events.platform.TestRunErrorEvent;
import androidx.test.services.events.platform.TestRunFinishedEvent;
import androidx.test.services.events.platform.TestRunStartedEvent;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import org.junit.runner.Description;
import org.junit.runner.Result;
import org.junit.runner.notification.Failure;
import org.junit.runner.notification.RunListener;

/**
 * This {@link RunListener} is for Android-based JUnit clients to speak with services that use the
 * {@link TestPlatformEvent} protocol.
 */
public final class TestPlatformListener extends RunListener {
  private static final String TAG = "TestPlatformListener";
  private static final String INIT_ERROR = "initializationError";
  private final TestPlatformEventService notificationService;
  private Map<Description, Status> testCaseToStatus;
  private Set<Description> foundTestCases;
  private Set<Description> finishedTestCases;
  private Set<Description> startedTestCases;
  // The {@link Description} for the parent TestRunner/TestSuite. May contain many nested
  // {@link Description}s for other {@link Runner}s and individual test methods.
  private Description testRunDescription = Description.EMPTY;
  private final AtomicReference<Description> currentTestCase =
      new AtomicReference<>(Description.EMPTY);
  private TestRunInfo memoizedTestRun;
  private final AtomicBoolean processCrashed = new AtomicBoolean(false);
  /*
   * ongoingResult and ongoingResultListener enable us to generate a final test run result in the
   * event of an application crash.  This is a bit messy, but Result's internal state is completely
   * dependent on its listener.  If this API changes, we can just subclass Result directly and
   * populate it throughout this RunListener.
   */
  private final AtomicReference<Result> ongoingResult = new AtomicReference<>(new Result());
  private final AtomicReference<RunListener> ongoingResultListener =
      new AtomicReference<>(ongoingResult.get().createListener());

  /**
   * Creates the {@link TestPlatformListener} to communicate with the remote test platform event
   * service.
   *
   * @param notificationService the remote service to send test run events to
   */
  public TestPlatformListener(@NonNull TestPlatformEventService notificationService) {
    super();
    // Instantiates everything on creation so that we can correctly report errors before
    // {@link #testRunStarted} is called.
    initListener();
    this.notificationService =
        checkNotNull(notificationService, "notificationService cannot be null");
  }

  /** {@inheritDoc} */
  @Override
  public void testRunStarted(Description description) throws Exception {
    initListener();
    ongoingResultListener.get().testRunStarted(description);
    setRunDescription(description);
    List<Description> testCases =
        JUnitDescriptionParser.getAllTestCaseDescriptions(testRunDescription);
    for (Description testCase : testCases) {
      foundTestCases.add(testCase);
      // Tests are considered passed if nothing changes their status.
      testCaseToStatus.put(testCase, Status.PASSED);
    }
    try {
      memoizedTestRun = convertToTestRun(testRunDescription);
      notificationService.send(new TestRunStartedEvent(memoizedTestRun, TimeStamp.now()));
    } catch (TestEventException e) {
      Log.e(TAG, "Unable to send TestRunStartedEvent to Test Platform", e);
    }
  }

  /** {@inheritDoc} */
  @Override
  public void testRunFinished(Result result) throws Exception {
    ongoingResultListener.get().testRunFinished(result);
    Status status = result.wasSuccessful() ? Status.PASSED : Status.FAILED;
    // If the process crashed at any point, this is failed.
    status = processCrashed.get() ? Status.FAILED : status;
    // Mark all test cases that haven't run as CANCELLED or ABORTED (if started)
    if (foundTestCases.size() > finishedTestCases.size()) {
      // This was aborted mid test run. Mark it if this isn't already failing for some other
      // reason.
      status = status.equals(Status.PASSED) ? Status.ABORTED : status;
      for (Description test :
          JUnitDescriptionParser.getAllTestCaseDescriptions(testRunDescription)) {
        if (!finishedTestCases.contains(test)) {
          if (startedTestCases.contains(test)) {
            // The test isn't completed, but it was started and not finished.
            testCaseToStatus.put(test, Status.ABORTED);
          } else {
            // The test was supposed to be run but was never finished
            testCaseToStatus.put(test, Status.CANCELLED);
          }
          testFinishedInternal(test, TimeStamp.now());
        }
      }
    }
    try {
      notificationService.send(
          new TestRunFinishedEvent(memoizedTestRun, new TestStatus(status), TimeStamp.now()));
    } catch (TestEventException e) {
      Log.e(TAG, "Unable to send TestRunFinishedEvent to Test Platform", e);
    }
  }

  /** {@inheritDoc} */
  @Override
  public void testStarted(Description description) throws Exception {
    if (isInitError(description)) {
      return; // This isn't a real test method, don't send an update to the service
    }
    ongoingResultListener.get().testStarted(description);
    startedTestCases.add(description);
    currentTestCase.set(description); // Caches the test description in case of a crash
    try {
      notificationService.send(
          new TestCaseStartedEvent(convertToTestCase(description), TimeStamp.now()));
    } catch (TestEventException e) {
      Log.e(TAG, "Unable to send TestStartedEvent to Test Platform", e);
    }
  }

  /** {@inheritDoc} */
  @Override
  public void testFinished(Description description) throws Exception {
    testFinishedInternal(description, TimeStamp.now());
  }

  // If the test is marked as finished during the test run finish, we use the same timestamp
  private void testFinishedInternal(Description description, TimeStamp timeStamp) throws Exception {
    if (isInitError(description)) {
      return; // This isn't a real test method, don't send an update to the service
    }
    ongoingResultListener.get().testFinished(description);
    finishedTestCases.add(description);
    try {
      notificationService.send(
          new TestCaseFinishedEvent(
              convertToTestCase(description),
              new TestStatus(testCaseToStatus.get(description)),
              timeStamp));
    } catch (TestEventException e) {
      Log.e(TAG, "Unable to send TestFinishedEvent to Test Platform", e);
    } finally {
      // reset test case
      currentTestCase.set(Description.EMPTY);
    }
  }

  /** {@inheritDoc} */
  @Override
  public void testFailure(Failure failure) throws Exception {
    Description description = failure.getDescription();
    ongoingResultListener.get().testFailure(failure);
    if (description.isTest() && !isInitError(description)) {
      testCaseToStatus.put(description, Status.FAILED);
    }
    try {
      TestPlatformEvent event = createErrorEvent(failure, TimeStamp.now());
      notificationService.send(event);
    } catch (TestEventException e) {
      throw new IllegalStateException("Unable to send error event", e);
    }
  }

  /** {@inheritDoc} */
  @Override
  public void testAssumptionFailure(Failure failure) {
    ongoingResultListener.get().testAssumptionFailure(failure);
    if (failure.getDescription().isTest()) {
      testCaseToStatus.put(failure.getDescription(), Status.SKIPPED);
    }
    try {
      TestPlatformEvent event = createErrorEvent(failure, TimeStamp.now());
      notificationService.send(event);
    } catch (TestEventException e) {
      Log.e(TAG, "Unable to send TestAssumptionFailureEvent to Test Platform", e);
    }
  }

  /** {@inheritDoc} */
  @Override
  public void testIgnored(Description description) throws Exception {
    ongoingResultListener.get().testIgnored(description);
    Log.i(
        TAG,
        "TestIgnoredEvent("
            + description.getDisplayName()
            + "): "
            + description.getClassName()
            + "#"
            + description.getMethodName());
    testCaseToStatus.put(description, Status.IGNORED);
    testFinishedInternal(description, TimeStamp.now());
  }

  /**
   * Reports the process crash event with a given exception. It is assumed that AJUR is crashing and
   * not recovering from this. This will inform all clients that:
   *
   * <ol>
   *   <li>A test has encountered an error (or the run has encountered an error if no test is in
   *       progress)
   *   <li>The currently running test has finished (if it didn't already finished normally)
   *   <li>The test run has finished.
   * </ol>
   */
  public boolean reportProcessCrash(Throwable t) {
    processCrashed.set(true);
    boolean isTestCase = true;
    Description failingDescription = currentTestCase.get();
    if (failingDescription.equals(Description.EMPTY)) {
      isTestCase = false;
      failingDescription = testRunDescription;
    }
    try {
      Log.e("TestPlatformListener", "reporting crash as testfailure", t);
      testFailure(new Failure(failingDescription, t));
      if (isTestCase) {
        testFinished(failingDescription);
      }
      testRunFinished(ongoingResult.get());
    } catch (Exception e) {
      Log.e(TAG, "An exception was encountered while reporting the process crash", e);
      return false;
    }
    return true;
  }

  private void initListener() {
    finishedTestCases = new HashSet<>();
    foundTestCases = new HashSet<>();
    startedTestCases = new HashSet<>();
    testCaseToStatus = new HashMap<>();
    currentTestCase.set(Description.EMPTY);
    testRunDescription = Description.EMPTY;
    memoizedTestRun = null;
    processCrashed.set(false);
    ongoingResult.set(new Result());
    ongoingResultListener.set(ongoingResult.get().createListener());
  }

  private void setRunDescription(Description description) {
    testRunDescription = description;
    // Ignore around the "null" top-level Runner Description in AJUR or any unnecessarily nested
    // Runner structures.
    while (testRunDescription.getDisplayName().equals("null")
        && testRunDescription.getChildren().size() == 1) {
      testRunDescription = testRunDescription.getChildren().get(0);
    }
  }

  private static TestCaseInfo convertToTestCase(Description testCase) throws TestEventException {
    return ParcelableConverter.getTestCaseFromDescription(testCase);
  }

  private static TestRunInfo convertToTestRun(Description testRun) throws TestEventException {
    List<TestCaseInfo> testCases = new ArrayList<>();
    for (Description testCase : JUnitDescriptionParser.getAllTestCaseDescriptions(testRun)) {
      testCases.add(convertToTestCase(testCase));
    }
    return new TestRunInfo(testRun.getDisplayName(), testCases);
  }

  private static boolean isInitError(Description description) {
    return description.getMethodName() != null && description.getMethodName().equals(INIT_ERROR);
  }

  private TestPlatformEvent createErrorEvent(Failure failure, TimeStamp timeStamp)
      throws TestEventException {
    Description descriptionToUse = failure.getDescription();
    if (!descriptionToUse.isTest() || isInitError(descriptionToUse)) {
      descriptionToUse = testRunDescription;
    }
    ErrorInfo errorInfo = ErrorInfo.createFromFailure(failure);
    // If the description is a run description, report a run error. Otherwise report a test error.
    if (!descriptionToUse.equals(testRunDescription)) {
      try {
        return new TestCaseErrorEvent(convertToTestCase(descriptionToUse), errorInfo, timeStamp);
      } catch (TestEventException e) {
        Log.e(TAG, "Unable to create TestCaseErrorEvent", e);
      }
    }
    if (memoizedTestRun == null) {
      Log.d(TAG, "No test run info. Reporting an error before test run has ever started.");
      memoizedTestRun = convertToTestRun(Description.EMPTY);
    }
    return new TestRunErrorEvent(memoizedTestRun, errorInfo, timeStamp);
  }

}