{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "auto-mpg-desc",
   "metadata": {},
   "source": "# Auto MPG\n\n**Task:** regression — predict fuel efficiency (MPG) for 398 cars from the UCI Auto MPG dataset.\n\n**Metric:** MSE. The paper reports ≈ 8.37 for the CMNN baseline, but under a test-selected\nprotocol (HP search, early stopping, and best-epoch all on the test set, then best-5-of-10).\nA correctly-wired *held-out* harness should **not** reproduce 8.37 — it reports a somewhat\nhigher, honest number. See [Protocol](../protocol.md).\n\n**Monotone features** (non-increasing in MPG — heavier/more powerful → lower fuel economy):\n- `weight` — vehicle curb weight (lbs)\n- `displacement` — engine displacement (cubic inches)\n- `horsepower` — engine power output\n\nNon-monotone features: `cylinders`, `acceleration`, `model_year`, `origin`.\n\n**Reference:** Table 1, row \"Auto MPG\" of Runje & Shankaranarayana (2023),\n[arXiv:2205.11775](https://arxiv.org/abs/2205.11775).\n\n> **Before running:** fetch the datasets with\n> `python -m benchmarks.datasets.download`."
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "auto-mpg-run",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "from benchmarks._common.config_io import load_config\n",
    "from benchmarks._common.results import aggregate\n",
    "from benchmarks._common.runner import run\n",
    "from benchmarks.datasets.download import default_dest\n",
    "from benchmarks.datasets.registry import load\n",
    "\n",
    "DATA_DIR = default_dest()\n",
    "CONFIG_DIR = Path(\"../../../benchmarks/configs\")\n",
    "\n",
    "PAPER_MSE = 8.37  # Table 1 value from Runje & Shankaranarayana (2023)\n",
    "\n",
    "# Small budget for scaffold execution; maintainer uses full seeds/epochs.\n",
    "QUICK_SEEDS = (0, 1)\n",
    "QUICK_EPOCHS = 5\n",
    "\n",
    "bundle = load(\"auto\", data_dir=DATA_DIR)\n",
    "\n",
    "flavors = [\n",
    "    (\"torch\", \"switch\", False),\n",
    "    (\"torch\", \"absolute\", False),\n",
    "    (\"jax\", \"switch\", False),\n",
    "    (\"keras\", \"switch\", False),\n",
    "]\n",
    "\n",
    "results = {}\n",
    "for backend, mode, residual in flavors:\n",
    "    cfg = load_config(\n",
    "        CONFIG_DIR / \"auto.toml\",\n",
    "        backend=backend,\n",
    "        mode=mode,\n",
    "        residual=residual,\n",
    "    )\n",
    "    # Override for quick scaffold run\n",
    "    import dataclasses\n",
    "    cfg = dataclasses.replace(cfg, seeds=QUICK_SEEDS, epochs=QUICK_EPOCHS)\n",
    "    rows = run(cfg, bundle)\n",
    "    agg = aggregate(rows, metric=\"mse\", lower_is_better=True, top_k=len(rows))\n",
    "    flavor_key = f\"{backend}/{mode}{'[residual]' if residual else ''}\"\n",
    "    results[flavor_key] = agg\n",
    "    print(f\"{flavor_key}: MSE = {agg.mean:.4f} ± {agg.std:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "auto-mpg-table",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "rows_table = [{\"flavor\": \"paper (CMNN)\", \"MSE\": PAPER_MSE, \"std\": \"-\"}]\n",
    "for flavor, agg in results.items():\n",
    "    rows_table.append({\"flavor\": flavor, \"MSE\": round(agg.mean, 4), \"std\": round(agg.std, 4)})\n",
    "\n",
    "df = pd.DataFrame(rows_table).set_index(\"flavor\")\n",
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}