Loan Defaulter

Task: binary classification — predict loan default from a synthetic loan dataset (≈ 255,000 instances, 8 engineered features).

Metric: accuracy (paper reports ≈ 0.945 for the CMNN baseline).

Monotone features:

• Non-decreasing in default probability: feature_1 (income-to-debt ratio proxy), feature_4 (number of late payments)

• Non-increasing in default probability (higher value → lower risk): feature_0 (credit score), feature_2 (employment duration), feature_3 (collateral value)

Non-monotone features: feature_5, feature_6, feature_7.

Reference: Table 2, row “Loan Defaulter” of Runje & Shankaranarayana (2023), arXiv:2205.11775.

Before running: fetch the datasets with python -m benchmarks.datasets.download.

from pathlib import Path

from benchmarks._common.config_io import load_config
from benchmarks._common.results import aggregate
from benchmarks._common.runner import run
from benchmarks.datasets.download import default_dest
from benchmarks.datasets.registry import load

DATA_DIR = default_dest()
CONFIG_DIR = Path("../../../benchmarks/configs")

PAPER_ACCURACY = 0.945  # Table 2 value from Runje & Shankaranarayana (2023)

# Small budget for scaffold execution; maintainer uses full seeds/epochs.
QUICK_SEEDS = (0, 1)
QUICK_EPOCHS = 5

bundle = load("loan", data_dir=DATA_DIR)

flavors = [
    ("torch", "switch", False),
    ("torch", "absolute", False),
    ("jax", "switch", False),
    ("keras", "switch", False),
]

results = {}
for backend, mode, residual in flavors:
    cfg = load_config(
        CONFIG_DIR / "loan.toml",
        backend=backend,
        mode=mode,
        residual=residual,
    )
    import dataclasses
    cfg = dataclasses.replace(cfg, seeds=QUICK_SEEDS, epochs=QUICK_EPOCHS)
    rows = run(cfg, bundle)
    agg = aggregate(rows, metric="accuracy", lower_is_better=False, top_k=len(rows))
    flavor_key = f"{backend}/{mode}{'[residual]' if residual else ''}"
    results[flavor_key] = agg
    print(f"{flavor_key}: accuracy = {agg.mean:.4f} ± {agg.std:.4f}")

import pandas as pd

rows_table = [{"flavor": "paper (CMNN)", "accuracy": PAPER_ACCURACY, "std": "-"}]
for flavor, agg in results.items():
    rows_table.append({"flavor": flavor, "accuracy": round(agg.mean, 4), "std": round(agg.std, 4)})

df = pd.DataFrame(rows_table).set_index("flavor")
df