From c87f23db833365d3630a7fd86b1fcd071e7deac2 Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Wed, 7 May 2025 18:01:39 +0200 Subject: [PATCH 1/2] fix(ci/eval-stats): resolve prResult symlink (cherry picked from commit c8276114ac18237c8fcf5a2e416578aab58eb393) --- .github/workflows/eval.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.yml index 1c823b519585..e5fa2732a757 100644 --- a/.github/workflows/eval.yml +++ b/.github/workflows/eval.yml @@ -219,7 +219,7 @@ jobs: # Use the target branch to get accurate maintainer info nix-build target/ci -A eval.compare \ --arg beforeResultDir ./targetResult \ - --arg afterResultDir ./prResult \ + --arg afterResultDir $(realpath prResult) \ --arg touchedFilesJson ./touched-files.json \ -o comparison From ff678dcec901e8eb9d18dd83bce578b264b74c7d Mon Sep 17 00:00:00 2001 From: Johannes Kirschbauer Date: Wed, 7 May 2025 18:05:07 +0200 Subject: [PATCH 2/2] ci/eval-stats: sort output table by metric name (cherry picked from commit 4f64ebd02b0cf1623bf4bf13a27a7a2d1b186b2e) --- ci/eval/compare/cmp-stats.py | 154 +++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 ci/eval/compare/cmp-stats.py diff --git a/ci/eval/compare/cmp-stats.py b/ci/eval/compare/cmp-stats.py new file mode 100644 index 000000000000..0ef9c773163a --- /dev/null +++ b/ci/eval/compare/cmp-stats.py @@ -0,0 +1,154 @@ +import json +import os +from scipy.stats import ttest_rel +import pandas as pd +import numpy as np +from pathlib import Path + +# Define metrics of interest (can be expanded as needed) +METRIC_PREFIXES = ("nr", "gc") + +def flatten_data(json_data: dict) -> dict: + """ + Extracts and flattens metrics from JSON data. + This is needed because the JSON data can be nested. + For example, the JSON data entry might look like this: + + "gc":{"cycles":13,"heapSize":5404549120,"totalBytes":9545876464} + + Flattened: + + "gc.cycles": 13 + "gc.heapSize": 5404549120 + ... + + Args: + json_data (dict): JSON data containing metrics. + Returns: + dict: Flattened metrics with keys as metric names. + """ + flat_metrics = {} + for k, v in json_data.items(): + if isinstance(v, (int, float)): + flat_metrics[k] = v + elif isinstance(v, dict): + for sub_k, sub_v in v.items(): + flat_metrics[f"{k}.{sub_k}"] = sub_v + return flat_metrics + + + + +def load_all_metrics(directory: Path) -> dict: + """ + Loads all stats JSON files in the specified directory and extracts metrics. + + Args: + directory (Path): Directory containing JSON files. + Returns: + dict: Dictionary with filenames as keys and extracted metrics as values. + """ + metrics = {} + for system_dir in directory.iterdir(): + assert system_dir.is_dir() + + for chunk_output in system_dir.iterdir(): + with chunk_output.open() as f: + data = json.load(f) + metrics[f"{system_dir.name}/${chunk_output.name}"] = flatten_data(data) + + return metrics + +def dataframe_to_markdown(df: pd.DataFrame) -> str: + df = df.sort_values(by=df.columns[0], ascending=True) + markdown_lines = [] + + # Header (get column names and format them) + header = '\n| ' + ' | '.join(df.columns) + ' |' + markdown_lines.append(header) + markdown_lines.append("| - " * (len(df.columns)) + "|") # Separator line + + # Iterate over rows to build Markdown rows + for _, row in df.iterrows(): + # TODO: define threshold for highlighting + highlight = False + + fmt = lambda x: f"**{x}**" if highlight else f"{x}" + + # Check for no change and NaN in p_value/t_stat + row_values = [] + for val in row: + if isinstance(val, float) and np.isnan(val): # For NaN values in p-value or t-stat + row_values.append("-") # Custom symbol for NaN + elif isinstance(val, float) and val == 0: # For no change (mean_diff == 0) + row_values.append("-") # Custom symbol for no change + else: + row_values.append(fmt(f"{val:.4f}" if isinstance(val, float) else str(val))) + + markdown_lines.append('| ' + ' | '.join(row_values) + ' |') + + return '\n'.join(markdown_lines) + + +def perform_pairwise_tests(before_metrics: dict, after_metrics: dict) -> pd.DataFrame: + common_files = sorted(set(before_metrics) & set(after_metrics)) + all_keys = sorted({ metric_keys for file_metrics in before_metrics.values() for metric_keys in file_metrics.keys() }) + + results = [] + + for key in all_keys: + before_vals, after_vals = [], [] + + for fname in common_files: + if key in before_metrics[fname] and key in after_metrics[fname]: + before_vals.append(before_metrics[fname][key]) + after_vals.append(after_metrics[fname][key]) + + if len(before_vals) >= 2: + before_arr = np.array(before_vals) + after_arr = np.array(after_vals) + + diff = after_arr - before_arr + pct_change = 100 * diff / before_arr + t_stat, p_val = ttest_rel(after_arr, before_arr) + + results.append({ + "metric": key, + "mean_before": np.mean(before_arr), + "mean_after": np.mean(after_arr), + "mean_diff": np.mean(diff), + "mean_%_change": np.mean(pct_change), + "p_value": p_val, + "t_stat": t_stat + }) + + df = pd.DataFrame(results).sort_values("p_value") + return df + + +if __name__ == "__main__": + before_dir = os.environ.get("BEFORE_DIR") + after_dir = os.environ.get("AFTER_DIR") + + if not before_dir or not after_dir: + print("Error: Environment variables 'BEFORE_DIR' and 'AFTER_DIR' must be set.") + exit(1) + + before_stats = Path(before_dir) / "stats" + after_stats = Path(after_dir) / "stats" + + # This may happen if the pull request target does not include PR#399720 yet. + if not before_stats.exists(): + print("⚠️ Skipping comparison: stats directory is missing in the target commit.") + exit(0) + + # This should never happen, but we're exiting gracefully anyways + if not after_stats.exists(): + print("⚠️ Skipping comparison: stats directory missing in current PR evaluation.") + exit(0) + + before_metrics = load_all_metrics(before_stats) + after_metrics = load_all_metrics(after_stats) + df1 = perform_pairwise_tests(before_metrics, after_metrics) + markdown_table = dataframe_to_markdown(df1) + print(markdown_table)