Analysis

Three tests on sessions with and without a signal

Simulate sessions shaped like the light switch's, 8 rounds of 11 embeddings with an offset shared within each round, with no signal and with one. On each, test a ridge head three ways: folds that mix the rounds with a binomial test, rounds held out with a binomial test, and rounds held out against all 70 labelings. Do it with the demo's penalty, and with a heavier one that also uses no labels.

import numpy as np
from itertools import combinations
from scipy.stats import binom

rng = np.random.default_rng(6)
ROUNDS, PER, D = 8, 11, 76
rounds = np.repeat(np.arange(ROUNDS), PER)
labels = (rounds % 2).astype(float)                   # Off, On, Off, On, as the demo teaches
u = rng.normal(size=D)
u = (u - u.mean()) / np.linalg.norm(u - u.mean())    # sums to zero, like every embedding

def session(signal, drift=0.3):
    """88 embeddings: noise, an offset shared within each round, and the label times u."""
    x = rng.normal(size=(ROUNDS * PER, D))
    x += np.repeat(rng.normal(0, drift, (ROUNDS, D)), PER, axis=0)
    return x - x.mean(axis=1, keepdims=True) + signal * labels[:, None] * u

def accuracy(x, folds, Y, penalty):
    """Held-out accuracy of ridge for every labeling in Y, one row per labeling. Neither
    penalty uses labels: "gap" is the demo's, "mean" the Gram matrix's mean eigenvalue."""
    right = np.zeros(len(Y))
    for train, test in folds:
        xc = x[train] - x[train].mean(0)
        ev = np.linalg.eigvalsh(xc.T @ xc)
        lam = np.sqrt(max(ev[0], 1) * ev[1]) if penalty == "gap" else ev.mean()
        fit = np.linalg.solve(xc.T @ xc + lam * np.eye(D), xc.T)
        yt = Y[:, train]
        w = (yt - yt.mean(1, keepdims=True)) @ fit.T              # a weight vector per labeling
        score = (x[test] - x[train].mean(0)) @ w.T + yt.mean(1)
        right += ((score > 0.5) == (Y[:, test].T == 1)).sum(0)
    return right / x.shape[0]

n = len(labels)
by_round = [(np.flatnonzero(rounds != r), np.flatnonzero(rounds == r)) for r in range(ROUNDS)]
relabel = np.array([np.isin(rounds, c) for c in combinations(range(ROUNDS), 4)], float)  # 70
taught = next(i for i, r in enumerate(relabel) if np.array_equal(r, labels))

def tests(x, penalty):
    mixed = rng.permutation(n).reshape(ROUNDS, PER)              # folds that ignore the rounds
    folds = [(np.setdiff1d(np.arange(n), m), m) for m in mixed]
    leaky = accuracy(x, folds, labels[None], penalty)
    scores = accuracy(x, by_round, relabel, penalty)
    return {"leaky folds, binomial": binom.sf(round(leaky[0] * n) - 1, n, 0.5),
            "rounds, binomial": binom.sf(round(scores[taught] * n) - 1, n, 0.5),
            "rounds, 70 labelings": np.mean(scores >= scores[taught] - 1e-12),
            "null's mean accuracy": scores.mean()}

for penalty, signal in (("gap", 0.0), ("mean", 0.0), ("mean", 2.0)):
    runs = [tests(session(signal), penalty) for _ in range(300)]
    print(f"penalty {penalty}, signal {signal}:")
    for k in runs[0]:
        v = np.array([r[k] for r in runs])
        shown = f"{v.mean():.2f}" if "mean" in k else f"p < 0.05 in {np.mean(v < 0.05):.0%}"
        print(f"   {k:21s} {shown}")

You should seeWith no signal, mixed folds declare a head clearly better than guessing in 36 and 89 percent of sessions. Rounds held out with a binomial test do it in 10 percent with the demo's penalty and 1 percent with the heavier one, whose null averages 0.37. The 70 labelings do it in 3 percent with either. With a signal, the labelings find it in 58 percent of sessions and the binomial in 48.

Then report: why the binomial errs in opposite directions with the two penalties, and what the drift between rounds does to each test when you raise it.