第11章 - experiment模板-数据与实验

// packages/templates/src/experiment/verifier.ts
function modeOf(detail: LevelDetail): 'tally' | 'duel' | 'sampling' | 'inference' {
  const script = detail.levelScript as { mode?: string } | null;
  if (script?.mode === 'sampling') return 'sampling';
  if (script?.mode === 'inference') return 'inference';
  return 'tally';
}

export const experimentVerifier = {
  parseScript(input: unknown): unknown {
    const script = input as { mode?: string } | null;
    if (script?.mode === 'sampling') return samplingScriptSchema.parse(input);
    if (script?.mode === 'inference') return inferenceScriptSchema.parse(input);
    return experiment.parseScript(input);
  },
  // ...
};

'probe/sample': { n?: number };
'probe/clear': Record<string, never>;
'probe/focus': { id: string };
'probe/claim': { claim: ClaimInput };
'probe/tally': { counts: Record<string, number> };

export type ClaimInput =
  | { kind: 'count-compare'; category: string; direction: 'most' | 'least'; tiedWith?: string[]; n: number }
  | { kind: 'stat'; statistic: 'mean' | 'proportion'; value: number; n: number; ci?: [number, number] }
  | { kind: 'difference'; delta: number; ci: [number, number]; n: number };

// packages/templates/src/experiment/logic.ts
const truth = truthCounts(script);
for (const category of script.categories) {
  if ((truth[category.id] ?? 0) !== category.count) {
    issues.push('类别数量与 items 不符：' + category.id + ' 声明 ' + category.count
      + '，实际 ' + (truth[category.id] ?? 0));
  }
}

/** 一次点击把整筐倒出来：复用 probe/sample 的 payload.n（运行时按剩余量切片）。 */
async function dumpAll(): Promise<void> {
  await dispatch({ type: 'probe/sample', payload: { n: remaining.value } });
}

case 'probe/sample': {
  if (state.collected.length >= script.items.length) {
    return { state, error: '这一筐已经采完了，去看统计条。' };
  }
  const requested = action.payload.n;
  const take = typeof requested === 'number' && Number.isFinite(requested)
    ? Math.max(1, Math.floor(requested))
    : script.batchSize;
  const next = script.items.slice(state.collected.length, state.collected.length + take);
  // ...
}

/** 数字框：blur / Enter 时把整份统计条一次 dispatch（运行时只认整份 counts）。 */
async function commitTally(): Promise<void> {
  if (busy.value || !collectionComplete.value) return;
  if (Object.keys(drafts.value).length === 0) return;
  const counts: Record<string, number> = {};
  for (const item of categories.value) {
    counts[item.id] = clampCount(drafts.value[item.id], recorded.value[item.id] ?? 0);
  }
  const ok = await dispatch({ type: 'probe/tally', payload: { counts } });
  if (ok) drafts.value = {};
}

/** 合并答案：一个 chip 同时给出「最多/最少 + 类别」，点一次就写结论。 */
async function answer(direction: 'most' | 'least', categoryId: string): Promise<void> {
  if (!categoryId) return;
  const claim: ClaimInput = {
    kind: 'count-compare',
    category: categoryId,
    direction,
    tiedWith: tiedWithFor(categoryId),
    n: collected.value.length,
  };
  await dispatch({ type: 'probe/claim', payload: { claim } });
}

/**
 * 结论必须和数据对得上：
 * - 方向 most / least 的类别要在对应极值集合里；
 * - 并列时 tiedWith 要把另外几个极值类别都列出来（多列、漏列都算错）；
 * - n 要正好等于采集总数。
 */
export function claimMatches(counts: TallyCounts, claim: ClaimInput | undefined, total: number): boolean

/**
 * duel（10 回合对撞实验）的纯逻辑：
 * - 每局按固定系数把 attack/defense/economy 摊成攻守分，双方同时出手；
 * - 噪声只来自注入的种子（mulberry32），同一 (seed, 局号, 回合, 位置) 结果唯一；
 * - 1000 局后给胜率与 Wilson 95% 区间。
 * 不 import Phaser/Vue/DOM，不调用 Math.random / Date.now。
 */

/** 噪声：同一种子下每局每回合四个位置各取一次，映射到 [noiseLow, noiseHigh]。 */
function noise(base: number, round: number, side: number, coefficients: DuelCoefficients): number {
  const random = mulberry32(base + round * 131 + side)();
  return coefficients.noiseLow + random * (coefficients.noiseHigh - coefficients.noiseLow);
}

/** 跑 n 局，返回胜率与 Wilson 95% 区间；完全确定性。 */
export function simulateDuel(
  script: DuelScript,
  params: DuelParams,
  seed: number,
  n: number,
): DuelResult

/** 参数必须正好用完资源点、且每个值都是 step 的整数倍。 */
export function paramsValid(script: DuelScript, params: DuelParams | undefined): boolean {
  if (!params) return false;
  for (const value of [params.attack, params.defense, params.economy]) {
    if (!Number.isInteger(value) || value < 0 || value % script.step !== 0) return false;
  }
  return paramsSum(params) === script.resources;
}

// packages/templates/src/experiment/duel.ts
const rawN = input.dataset?.n;
const datasetN = typeof rawN === 'number' && Number.isInteger(rawN) ? rawN : 0;
if (datasetN <= 0) { /* exp-not-run */ }
if (datasetN > MAX_MATCHES) { /* exp-not-run：采样局数超出这一关的上限 */ }
if (datasetN < script.matches) { /* exp-sample-small */ }

const result = simulateDuel(script, input.params, input.seed, datasetN);
// ...
const target = input.checkClaim ? script.targets.winRate : script.targets.baselineB;
if (result.winRate < target) {
  return { passed: false, failureClass: 'exp-below-baseline', /* ... */ };
}
if (input.checkClaim && result.ci[0] < script.targets.winRateLow) {
  return { passed: false, failureClass: 'exp-below-baseline', /* ... */ };
}

const tolerance = 0.01;
const inInterval = claim.value >= result.ci[0] - tolerance && claim.value <= result.ci[1] + tolerance;
if (!inInterval || Math.abs(claim.value - result.winRate) > 0.05) {
  return { passed: false, failureClass: 'exp-claim-beyond-ci', /* ... */ };
}

/**
 * 迁移门：tally 只换外观（不做 transfer）；duel 换到 B 种子再验一次胜率基线。
 */
export function transferVariants(detail: LevelDetail): RuntimeVariant[] {
  const script = parseScript(detail.levelScript);
  if (script.mode !== 'duel') return [];
  if (script.seeds.length < 2) return [];
  return [{ id: 'v-seed-b', axis: 'transfer', patch: { seedIndex: 1 } }];
}

const isTransfer = variant.axis === 'transfer';
const check = checkDuelArtifact(script, {
  params: asParams(raw.params),
  dataset: raw.dataset,
  claim: raw.claim,
  seed,
  checkClaim: !isTransfer,
});

if (isTransfer && failureClass === 'exp-below-baseline') failureClass = 'exp-overfit-seed';

transferVariants(detail: LevelDetail) {
  return isSampling(detail) || isInference(detail) ? [] : experiment.transferVariants(detail);
}

export function drawSample(script: SamplingScript, n: number, seed: number): SamplingSample {
  const total = script.population.red + script.population.blue;
  const size = Math.max(1, Math.min(total, Math.trunc(n)));
  const rng = mulberry32(seed);
  const pool: boolean[] = [];
  for (let i = 0; i < total; i += 1) pool.push(i < script.population.red);
  // Fisher-Yates 只抽前 size 个
  for (let i = 0; i < size; i += 1) {
    const j = i + Math.floor(rng() * (total - i));
    // swap ...
  }
  // ...
  const interval = wilsonInterval(red, size);
  return { n: size, red, proportion, ci: [interval.low, interval.high], trueProportion: trueProportion(script) };
}

const trueValue = trueProportion(script);
if (Math.abs(claim.value - trueValue) > script.tolerance) {
  return { passed: false, failures: ['sampling-far'], metrics };
}
if (claim.ci[0] > trueValue || claim.ci[1] < trueValue) {
  return { passed: false, failures: ['sampling-ci-miss'], metrics };
}

/** Wilson 95% 区间；小样本与 0/1 边界都比正态近似稳。 */
export function wilsonInterval(successes: number, n: number, z = 1.96): Interval {
  if (n <= 0) return { low: 0, high: 1 };
  const p = Math.min(1, Math.max(0, successes / n));
  const z2 = z * z;
  const denom = 1 + z2 / n;
  const center = (p + z2 / (2 * n)) / denom;
  const margin = (z * Math.sqrt((p * (1 - p) + z2 / (4 * n)) / n)) / denom;
  return { low: Math.max(0, center - margin), high: Math.min(1, center + margin) };
}

export function inferenceStats(script: InferenceScript): InferenceStats {
  const a = statsOf(script.groups.a);
  const b = statsOf(script.groups.b);
  const overlap = a.ci[0] <= b.ci[1] + script.overlapTolerance && b.ci[0] <= a.ci[1] + script.overlapTolerance;
  a.overlaps = overlap;
  b.overlaps = overlap;
  let truthfulClaim: InferenceStats['truthfulClaim'] = 'no-clear';
  if (!overlap) truthfulClaim = a.proportion > b.proportion ? 'a-higher' : 'b-higher';
  return { a, b, truthfulClaim };
}

const claimDirection = raw.claim.delta > 0 ? 'a-higher' : raw.claim.delta < 0 ? 'b-higher' : 'no-clear';
if (claimDirection !== truth.truthfulClaim) {
  return { passed: false, failures: ['inference-claim-wrong'], metrics };
}
// ...
if (truth.truthfulClaim !== 'no-clear' && raw.claim.ci[0] <= 0) {
  return { passed: false, failures: ['inference-ci-includes-zero'], metrics };
}