Summary Tables

This notebook loads the committed benchmarks/results/paper-reproduction.json and renders two tables:

1. Headline table — per-dataset metric for each flavor (torch/switch, torch/absolute, jax/switch, keras/switch) alongside the paper-quoted CMNN number and XGBoost baseline.

2. Cross-backend agreement table — max absolute difference in metric across backends for each (dataset, mode) pair, verifying numerical equivalence.

Run the per-dataset notebooks first (or tools/execute-benchmarks.sh) to generate benchmarks/results/paper-reproduction.json.

Paper reference: Runje & Shankaranarayana (2023), arXiv:2205.11775. Zenodo archive: https://zenodo.org/records/7968969.

import json
from pathlib import Path

import pandas as pd

RESULTS_FILE = Path("../../../benchmarks/results/paper-reproduction.json")

if not RESULTS_FILE.exists():
    raise FileNotFoundError(
        f"{RESULTS_FILE} not found. "
        "Run the per-dataset notebooks or tools/execute-benchmarks.sh first."
    )

with RESULTS_FILE.open() as f:
    data = json.load(f)

print(f"Loaded results for {len(data)} entries from {RESULTS_FILE}")

# XGBoost baseline — shows how the XGBoost column is produced.
# Run the per-dataset notebooks (or tools/execute-benchmarks.sh) which call
# run_xgboost and store results under the "xgboost" key in paper-reproduction.json.
# The import below documents the entry-point; results are read from data[] below.
from benchmarks.baselines.xgboost import run_xgboost  # noqa: F401

# Preview: XGBoost mean metric per dataset (None = not yet run)
xgb_means = {
    dataset: data.get(dataset, {}).get("xgboost", {}).get("mean", None)
    for dataset in PAPER_NUMBERS
}
print("XGBoost baseline means:", xgb_means)

# Paper-quoted baseline numbers (Tables 1 & 2 of Runje & Shankaranarayana 2023)
PAPER_NUMBERS = {
    "auto":   {"metric": "mse",      "paper": 8.37,  "lower_is_better": True},
    "heart":  {"metric": "accuracy", "paper": 0.852, "lower_is_better": False},
    "compas": {"metric": "accuracy", "paper": 0.672, "lower_is_better": False},
    "blog":   {"metric": "rmse",     "paper": None,  "lower_is_better": True},   # TBD
    "loan":   {"metric": "accuracy", "paper": 0.945, "lower_is_better": False},
}

FLAVORS = [
    ("torch", "switch",   False),
    ("torch", "absolute", False),
    ("jax",   "switch",   False),
    ("keras", "switch",   False),
]

headline_rows = []
for dataset, info in PAPER_NUMBERS.items():
    metric = info["metric"]
    row: dict = {"dataset": dataset, "metric": metric, "paper (CMNN)": info["paper"]}
    for backend, mode, residual in FLAVORS:
        flavor_key = f"{backend}/{mode}"
        entry = data.get(dataset, {}).get(flavor_key, {})
        row[flavor_key] = round(entry.get("mean", float("nan")), 4)
    # XGBoost baseline column — None when not yet run
    xgb_entry = data.get(dataset, {}).get("xgboost", {})
    xgb_mean = xgb_entry.get("mean", None)
    row["xgboost"] = round(xgb_mean, 4) if xgb_mean is not None else None
    headline_rows.append(row)

df_headline = pd.DataFrame(headline_rows).set_index("dataset")
print("Headline table — per-dataset metric vs paper:")
df_headline

import itertools

# Cross-backend agreement: max |metric_i - metric_j| across backend pairs
agreement_rows = []
for dataset, info in PAPER_NUMBERS.items():
    metric = info["metric"]
    for mode in ("switch", "absolute"):
        vals = {}
        for backend in ("torch", "jax", "keras"):
            flavor_key = f"{backend}/{mode}"
            entry = data.get(dataset, {}).get(flavor_key, {})
            v = entry.get("mean", None)
            if v is not None:
                vals[backend] = v
        if len(vals) >= 2:
            max_diff = max(
                abs(a - b)
                for a, b in itertools.combinations(vals.values(), 2)
            )
            agreement_rows.append({
                "dataset": dataset,
                "mode": mode,
                "metric": metric,
                "max_abs_diff": round(max_diff, 6),
                "backends": ", ".join(sorted(vals)),
            })

df_agreement = pd.DataFrame(agreement_rows).set_index(["dataset", "mode"])
print("Cross-backend agreement table:")
df_agreement