{"ok": true, "corpus_meta": {"schema_version": 1, "built_at": "2026-08-15T09:06:23.238536+00:00", "generator": {"seed": 20260809, "battery_n": 800, "n_templates": 44, "script": "apps/ava-factory/scripts/build_orchestration_corpus.py"}, "tier_vocab": ["deterministic", "llm", "deep_research", "action_operator", "agentic_epic"], "dense_features": ["n_words", "n_chain_signals", "has_code_terms", "latency_ms", "tokens_est", "attempt"], "reward_config": {"weights": {"status": 0.6, "latency": 0.25, "tokens": 0.15}, "weights_rationale": "0.6 > 0.25 + 0.15: speed can never buy back a failure", "failure_statuses": ["error", "fail", "failed", "timeout"], "status_score": "S = -1.0 if status in failure_statuses; S = 1.0/max(1, attempt) if status == 'ok'; else 0.0", "node_scale": {"latency": "R_lat = clip(1 - latency_ms/100.0, 0, 1)", "tokens": "R_tok = 1 - min(log(1+tokens_est)/log(1+256), 1)", "latency_scale_ms": 100.0, "tokens_log_cap": 256}, "agent_scale": {"latency": "R_lat = exp(-duration_s/600.0)", "tokens": "R_tok = 1 - min(log(1+output_tokens)/log(1+32768), 1)", "duration_tau_s": 600.0, "tokens_log_cap": 32768}, "reward": "clip(0.6*S + 0.25*R_lat + 0.15*R_tok, -1, 1)", "synthetic_battery_reward": "1.0 flat (labels are rule-derived from the same heuristic - label-match by construction), provenance simulated", "note": "node-scale constants fitted on latency distribution min 0 / p50 35 / p95 55 / max 55 ms and tokens max 200 - but 12/15 of those latencies are SCRIPTED constants (45/30/35/55 hardcoded at apps/scout-cli/bigbang/plugins/agents/cli.py:146,167,171,223)"}, "label_corrections": {"path": "apps/ava-factory/data/orchestration/label_corrections.jsonl", "n_corrections": 0, "n_records_corrected": 0}, "counts": {"total": 1613, "by_source": {"ultra_timeline": 813, "synthetic_battery": 800}, "by_provenance": {"simulated": 839, "measured": 774}, "by_tier": {"deterministic": 253, "deep_research": 451, "llm": 110, "agentic_epic": 490, "action_operator": 309}, "by_split": {"train": 1397, "val": 151, "test": 65}, "by_label_tier": {"simulated": 820, "measured-behavior": 752, "measured-outcome": 41}, "measured_holdout_by_label_tier": {"measured-behavior": 120, "measured-outcome": 10}}, "sources": {"ultra_timeline": {"included": true, "dir": "bundles/ultra/runs", "n_runs": 251, "n_records": 813}, "workflow_journal": {"included": false, "reason": "journal dir not provided"}, "synthetic_battery": {"included": true, "n_records": 800, "seed": 20260809, "n_templates": 44, "note": "seeded template grammar; labels rule-derived from the harness heuristic; provenance simulated"}}}, "champion": {"schema_version": 1, "built_at": "2026-08-15T09:07:05.317117+00:00", "corpus_source": "l2_corpus", "trainer": "orchestrator_model", "champion": {"name": "v4", "model_version": "orch-mlp-v1-v4", "val_tier_accuracy": 0.966887, "tier_accuracy_all": 0.7692307692307693, "tier_accuracy_measured": 0.819672131147541, "n_holdout": 65, "n_measured_holdout": 61, "agreement": {"agreement_rate": 0.7692307692307693, "mean_reward_all": 0.8702535692307694, "mean_reward_on_agreement": 0.9318594400000002, "mean_reward_on_disagreement": 0.6649006666666666, "note": "counterfactual rewards unobserved; agreement-conditional statistics reported in place of true regret"}, "risk_calibration_note": "per-decile detail dropped at vendor time; see the full eval_report.json in apps/ava-factory/reports/orchestrator/"}, "gate": {"gate_passed": false, "reason": "champion measured accuracy 0.819672 does not strictly beat both baselines (freq prior 0.245902, heuristic 0.836066) on n=61 measured held-out records"}, "notes": ["counterfactual rewards unobserved; agreement-conditional statistics reported in place of true regret", "synthetic_battery labels ARE the heuristic's outputs, so heuristic accuracy is 1.0 on battery records by construction; the meaningful comparison is the measured subset", "gate compares measured-subset accuracy only; simulated battery records share the heuristic's labeling and cannot certify the model", "corpus_source=l2_corpus; trainer=orchestrator_model; all metrics measured from this run"], "vendored_from": "apps/ava-factory/reports/orchestrator/eval_report.json"}}