{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "tables-desc",
   "metadata": {},
   "source": [
    "# Summary Tables\n",
    "\n",
    "This notebook loads the committed `benchmarks/results/paper-reproduction.json` and\n",
    "renders two tables:\n",
    "\n",
    "1. **Headline table** \u2014 per-dataset metric for each flavor (torch/switch, torch/absolute,\n",
    "   jax/switch, keras/switch) alongside the paper-quoted CMNN number and XGBoost baseline.\n",
    "2. **Cross-backend agreement table** \u2014 max absolute difference in metric across backends\n",
    "   for each (dataset, mode) pair, verifying numerical equivalence.\n",
    "\n",
    "Run the per-dataset notebooks first (or `tools/execute-benchmarks.sh`) to generate\n",
    "`benchmarks/results/paper-reproduction.json`.\n",
    "\n",
    "> **Paper reference:** Runje & Shankaranarayana (2023),\n",
    "> [arXiv:2205.11775](https://arxiv.org/abs/2205.11775).\n",
    "> Zenodo archive: <https://zenodo.org/records/7968969>."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "tables-load",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from pathlib import Path\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "RESULTS_FILE = Path(\"../../../benchmarks/results/paper-reproduction.json\")\n",
    "\n",
    "if not RESULTS_FILE.exists():\n",
    "    raise FileNotFoundError(\n",
    "        f\"{RESULTS_FILE} not found. \"\n",
    "        \"Run the per-dataset notebooks or tools/execute-benchmarks.sh first.\"\n",
    "    )\n",
    "\n",
    "with RESULTS_FILE.open() as f:\n",
    "    data = json.load(f)\n",
    "\n",
    "print(f\"Loaded results for {len(data)} entries from {RESULTS_FILE}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "tables-xgboost-ref",
   "metadata": {},
   "outputs": [],
   "source": [
    "# XGBoost baseline \u2014 shows how the XGBoost column is produced.\n",
    "# Run the per-dataset notebooks (or tools/execute-benchmarks.sh) which call\n",
    "# run_xgboost and store results under the \"xgboost\" key in paper-reproduction.json.\n",
    "# The import below documents the entry-point; results are read from data[] below.\n",
    "from benchmarks.baselines.xgboost import run_xgboost  # noqa: F401\n",
    "\n",
    "# Preview: XGBoost mean metric per dataset (None = not yet run)\n",
    "xgb_means = {\n",
    "    dataset: data.get(dataset, {}).get(\"xgboost\", {}).get(\"mean\", None)\n",
    "    for dataset in PAPER_NUMBERS\n",
    "}\n",
    "print(\"XGBoost baseline means:\", xgb_means)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "tables-headline",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Paper-quoted baseline numbers (Tables 1 & 2 of Runje & Shankaranarayana 2023)\n",
    "PAPER_NUMBERS = {\n",
    "    \"auto\":   {\"metric\": \"mse\",      \"paper\": 8.37,  \"lower_is_better\": True},\n",
    "    \"heart\":  {\"metric\": \"accuracy\", \"paper\": 0.852, \"lower_is_better\": False},\n",
    "    \"compas\": {\"metric\": \"accuracy\", \"paper\": 0.672, \"lower_is_better\": False},\n",
    "    \"blog\":   {\"metric\": \"rmse\",     \"paper\": None,  \"lower_is_better\": True},   # TBD\n",
    "    \"loan\":   {\"metric\": \"accuracy\", \"paper\": 0.945, \"lower_is_better\": False},\n",
    "}\n",
    "\n",
    "FLAVORS = [\n",
    "    (\"torch\", \"switch\",   False),\n",
    "    (\"torch\", \"absolute\", False),\n",
    "    (\"jax\",   \"switch\",   False),\n",
    "    (\"keras\", \"switch\",   False),\n",
    "]\n",
    "\n",
    "headline_rows = []\n",
    "for dataset, info in PAPER_NUMBERS.items():\n",
    "    metric = info[\"metric\"]\n",
    "    row: dict = {\"dataset\": dataset, \"metric\": metric, \"paper (CMNN)\": info[\"paper\"]}\n",
    "    for backend, mode, residual in FLAVORS:\n",
    "        flavor_key = f\"{backend}/{mode}\"\n",
    "        entry = data.get(dataset, {}).get(flavor_key, {})\n",
    "        row[flavor_key] = round(entry.get(\"mean\", float(\"nan\")), 4)\n",
    "    # XGBoost baseline column \u2014 None when not yet run\n",
    "    xgb_entry = data.get(dataset, {}).get(\"xgboost\", {})\n",
    "    xgb_mean = xgb_entry.get(\"mean\", None)\n",
    "    row[\"xgboost\"] = round(xgb_mean, 4) if xgb_mean is not None else None\n",
    "    headline_rows.append(row)\n",
    "\n",
    "df_headline = pd.DataFrame(headline_rows).set_index(\"dataset\")\n",
    "print(\"Headline table \u2014 per-dataset metric vs paper:\")\n",
    "df_headline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "tables-agreement",
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "\n",
    "# Cross-backend agreement: max |metric_i - metric_j| across backend pairs\n",
    "agreement_rows = []\n",
    "for dataset, info in PAPER_NUMBERS.items():\n",
    "    metric = info[\"metric\"]\n",
    "    for mode in (\"switch\", \"absolute\"):\n",
    "        vals = {}\n",
    "        for backend in (\"torch\", \"jax\", \"keras\"):\n",
    "            flavor_key = f\"{backend}/{mode}\"\n",
    "            entry = data.get(dataset, {}).get(flavor_key, {})\n",
    "            v = entry.get(\"mean\", None)\n",
    "            if v is not None:\n",
    "                vals[backend] = v\n",
    "        if len(vals) >= 2:\n",
    "            max_diff = max(\n",
    "                abs(a - b)\n",
    "                for a, b in itertools.combinations(vals.values(), 2)\n",
    "            )\n",
    "            agreement_rows.append({\n",
    "                \"dataset\": dataset,\n",
    "                \"mode\": mode,\n",
    "                \"metric\": metric,\n",
    "                \"max_abs_diff\": round(max_diff, 6),\n",
    "                \"backends\": \", \".join(sorted(vals)),\n",
    "            })\n",
    "\n",
    "df_agreement = pd.DataFrame(agreement_rows).set_index([\"dataset\", \"mode\"])\n",
    "print(\"Cross-backend agreement table:\")\n",
    "df_agreement"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}