Analysis

The trainer's penalty, and better ones

In the demos' source, the trainer's fixture holds 150 windows of embeddings as integers, balanced labels, and the penalty the Python API chose for them. Reproduce the penalty, then compare it with others by contiguous five-fold cross-validation, choose one by nested cross-validation, and fit shrinkage LDA. Then cut the data to the light switch's size.

import json
import numpy as np
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

F = "tests/fixtures/train/"                  # in the demos' source
X = np.array(json.load(open(F + "embeddings.json")), float)   # 150 windows of 76, as integers
y = np.array(json.load(open(F + "targets.json")), float)      # 0 and 1
trainer = json.load(open(F + "ridge.json"))["ridge"]

def gap_ridge(X):
    """The demo's penalty: the geometric mean of the two smallest eigenvalues."""
    ev = np.linalg.eigvalsh((X - X.mean(axis=0)).T @ (X - X.mean(axis=0)))
    return np.sqrt(max(ev[0], 1) * ev[1])

Xc = X - X.mean(axis=0)
ev, vecs = np.linalg.eigh(Xc.T @ Xc)                          # ascending
gap = gap_ridge(X)
print(f"smallest eigenvalues {ev[0]:.2f} and {ev[1]:.3g}: "
      f"gap ridge {gap:.2f}, the trainer's {trainer:.2f}")
cosine = abs(vecs[:, 0].sum()) / 76 ** 0.5
print(f"the smallest one's direction against all ones: cosine {cosine:.3f}")

def fit(X, y, lam):
    """Ridge with the intercept left unpenalized, as the demo's fitLinear does."""
    m = X.mean(axis=0)
    xc = X - m
    w = np.linalg.solve(xc.T @ xc + lam * np.eye(X.shape[1]), xc.T @ (y - y.mean()))
    return w, y.mean() - m @ w

def blocks(n, k=5):
    size = -(-n // k)
    return [np.arange(i * size, min(n, (i + 1) * size)) for i in range(k)]

def blocked(X, y, lam, k=5):
    """Accuracy and squared error on contiguous held-out blocks: a score above 0.5 means 1.
    lam="gap" chooses the demo's penalty afresh from each training set."""
    hits = err = 0.0
    for test in blocks(len(y), k):
        train = np.setdiff1d(np.arange(len(y)), test)
        penalty = gap_ridge(X[train]) if lam == "gap" else lam
        w, b = fit(X[train], y[train], penalty)
        s = X[test] @ w + b
        hits += np.sum((s > 0.5) == (y[test] == 1))
        err += np.sum((s - y[test]) ** 2)
    return hits / len(y), err / len(y)

grid = [10.0 ** p for p in range(5, 11)]
for lam in [gap] + grid:
    acc, mse = blocked(X, y, lam)
    print(f"lambda {lam:8.2g}  effective df {np.sum(ev / (ev + lam)):5.1f}  "
          f"accuracy {acc:.2f}  mse {mse:.3f}")

# Nested: choose lambda inside each outer training set, then score the outer block.
hits, chosen = 0, []
for test in blocks(len(y)):
    train = np.setdiff1d(np.arange(len(y)), test)
    best = min(grid, key=lambda lam: blocked(X[train], y[train], lam, k=4)[1])
    w, b = fit(X[train], y[train], best)
    hits += np.sum((X[test] @ w + b > 0.5) == (y[test] == 1))
    chosen.append(f"{best:.0e}")
print(f"nested: accuracy {hits / len(y):.2f}, lambdas chosen {chosen}")

# Shrinkage LDA, its shrinkage set analytically from each training set (Ledoit and Wolf).
hits = 0
for test in blocks(len(y)):
    train = np.setdiff1d(np.arange(len(y)), test)
    lda = LinearDiscriminantAnalysis(solver="lsqr", shrinkage="auto").fit(X[train], y[train])
    hits += np.sum(lda.predict(X[test]) == y[test])
print(f"shrinkage LDA: accuracy {hits / len(y):.2f}")

# Cut to the light switch's size: 88 windows held out a round of 11 at a time, so every
# training set has 77 windows for 76 weights.
for lam in ("gap", 1e8, 1e9):
    acc = blocked(X[:88], y[:88], lam, k=8)[0]
    name = lam if lam == "gap" else f"{lam:.0e}"
    print(f"88 windows in 8 folds, lambda {name}: accuracy {acc:.2f}")

You should seeThe trainer's penalty reproduced exactly, 20,131.58, between eigenvalues of 5.07 and 7.99 × 10⁷, and the smallest one's direction all ones. At that penalty the fit spends 75 of its 76 degrees of freedom and scores 0.67. Squared error is lowest near 10⁹, at 31 degrees of freedom and 0.70. Nested cross-validation chooses 10⁸ or 10⁹ and scores 0.71, and shrinkage LDA 0.69. Cut to 88 windows held out a round of 11 at a time, the trainer's penalty scores 0.50, chance, where 10⁸ scores 0.61 and 10⁹ 0.66.

Then report: how large a difference in accuracy 150 windows can resolve, why the trainer's penalty does well at 150 windows and fails at 88, and what penalty you would choose without labels.