MetricResultExtensions.kt

/*
 * Copyright 2021 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package androidx.benchmark.macro

import android.util.Log
import androidx.benchmark.BenchmarkResult
import androidx.benchmark.MetricResult
import kotlin.math.abs

/**
 * Asserts that the two lists of Measurements are equal with a threshold for data,
 * ignoring list order.
 *
 * @throws AssertionError
 */
@ExperimentalMetricApi
fun assertEqualMeasurements(
    expected: List<Metric.Measurement>,
    observed: List<Metric.Measurement>,
    threshold: Double
) {
    val expectedSorted = expected.sortedBy { it.name }
    val observedSorted = observed.sortedBy { it.name }
    val expectedNames = listOf(expectedSorted.map { it.name })
    val observedNames = listOf(observedSorted.map { it.name })
    if (expectedNames != observedNames) {
        throw AssertionError("expected same measurement names, " +
            "expected = $expectedNames, observed = $observedNames")
    }

    var errorString = ""
    expectedSorted.zip(observedSorted) { expectedMeasurement, observedMeasurement ->
        val name = expectedMeasurement.name
        if (expectedMeasurement.requireSingleValue !=
            observedMeasurement.requireSingleValue
        ) {
            errorString += "expected value of requireSingleValue " +
                "(${expectedMeasurement.requireSingleValue}) does not match observed " +
                "value ${observedMeasurement.requireSingleValue}\n"
        }

        val expectedSamples = expectedMeasurement.data
        val observedSamples = observedMeasurement.data
        if (expectedSamples.size != observedSamples.size) {
            errorString += "$name expected ${expectedSamples.size} samples," +
                " observed ${observedSamples.size}\n"
        } else {
            expectedSamples.zip(observedSamples).forEachIndexed { index, pair ->
                if (abs(pair.first - pair.second) > threshold) {
                    errorString += "$name sample $index observed ${pair.first}" +
                        " more than $threshold from expected ${pair.second}\n"
                }
            }
        }
    }

    if (errorString.isNotBlank()) {
        throw AssertionError(errorString)
    }
}

internal fun List<Metric.Measurement>.merge(
    other: List<Metric.Measurement>
): List<Metric.Measurement> {
    val nameSet = this.map { it.name }.toSet()
    val otherNameSet = other.map { it.name }.toSet()
    val intersectingNames = nameSet.intersect(otherNameSet)
    if (intersectingNames.isNotEmpty()) {
        throw IllegalStateException(
            "Multiple metrics produced " +
                "measurements with overlapping names: $intersectingNames"
        )
    }
    return this + other
}

/**
 * Takes a `List<List<Measurement>>`, one for each iteration, and transposes the data to be
 * organized by Measurement name, with data merged into a `MetricResult`.
 *
 * For requireSingleValue Measurements, this becomes a MetricResult used to extract min/med/max.
 *
 * For !requireSingleValue SubResults, this becomes a MetricResult used to extract
 * P50/P90/P95/P99 from a flattened list of all samples, pooled together.
 */
internal fun List<List<Metric.Measurement>>.mergeMultiIterResults() = BenchmarkResult.Measurements(
    singleMetrics = this.map {
        it.filter { measurement ->
            measurement.requireSingleValue
        }.associate { singleResult ->
            singleResult.name to singleResult.data.first()
        }
    }.mergeToSingleMetricResults(),
    sampledMetrics = this.map {
        it.filter { measurement ->
            !measurement.requireSingleValue
        }.associate { singleResult ->
            singleResult.name to singleResult.data
        }
    }.mergeToSampledMetricResults()
)

/**
 * Merge the Map<String, Long> results from each iteration into one List<MetricResult>
 */
internal fun List<Map<String, Double>>.mergeToSingleMetricResults(): List<MetricResult> {
    val setOfAllKeys = flatMap { it.keys }.toSet()

    // build Map<String, List<Long>>
    val listResults: Map<String, List<Double>> = setOfAllKeys.associateWith { key ->
        mapIndexedNotNull { iteration, resultMap ->
            if (resultMap.keys != setOfAllKeys) {
                // TODO: assert that metrics are always captured (b/193827052)
                Log.d(TAG, "Skipping results from iter $iteration, it didn't capture all metrics")
                null
            } else {
                resultMap[key] ?: error("Metric $key not observed in iteration")
            }
        }
    }

    // transform to List<MetricResult>, sorted by metric name
    return listResults.map { (metricName, values) ->
        MetricResult(name = metricName, data = values)
    }.sortedBy { it.name }
}

/**
 * Merge the Map<String, List<Long>> results from each iteration into one List<MetricResult>
 */
internal fun List<Map<String, List<Double>>>.mergeToSampledMetricResults(): List<MetricResult> {
    val setOfAllKeys = flatMap { it.keys }.toSet()

    // build Map<String, List<List<Long>>>
    val listResults = setOfAllKeys.associateWith { key ->
        mapIndexed { index: Int, iterationSamples: Map<String, List<Double>> ->
            iterationSamples[key]
                ?: throw IllegalStateException("Iteration $index didn't capture metric $key")
        }
    }

    // transform to List<MetricResult>, sorted by metric name
    return listResults.map { (metricName, values) ->
        MetricResult(
            name = metricName,
            data = values.flatten(),
            iterationData = values
        )
    }.sortedBy { it.name }
}