{ "experiments": [ { "decision": { "ep": { "production": "ep_source_replay", "reason": "no independent arm beat source replay on the frozen gates", "verdict": "RETAIN_BASELINE" }, "wp": { "production": "wp_source_replay", "reason": "no independent arm beat source replay on the frozen gates", "verdict": "RETAIN_BASELINE" } }, "design": "docs/validation/EP_WP_TOURNAMENT_DESIGN.md", "excluded": { "kneel_or_spike": 4520, "missing_state_or_source": 42432, "not_snap": 53942, "play_deleted": 1 }, "experiment_id": "ep_wp_tournament_v1", "holdout_2025_accessed": false, "library_versions": { "numpy": "2.5.1", "polars": "1.43.2", "python": "3.14.6", "sklearn": "1.9.0" }, "ran_at": "2026-08-05T23:52:38+00:00", "results": { "ep": { "ep_empirical_bucket": { "select": { "arm": "ep_empirical_bucket", "calibration_by_yardline_decile": { "0": 0.10544139360254068, "1": 0.04693019284164901, "2": 0.09483728552239068, "3": 0.09700886740900216, "4": 0.062239853870992956, "5": 0.03376102130679581, "6": 0.13427215391156588, "7": 0.014891661215654328, "8": 0.22758087822706527, "9": 0.4882822043456968 }, "max_calibration_gap": 0.4882822043456968, "mse": 20.659325412279763 }, "test": { "arm": "ep_empirical_bucket", "calibration_by_yardline_decile": { "0": 0.039880270263111584, "1": 0.01149319323990916, "2": 0.0008297358676685462, "3": 0.020684514406954513, "4": 0.16978290566868415, "5": 0.01055597223437954, "6": 0.0203228090473051, "7": 0.059494015534415357, "8": 0.0383458214093598, "9": 0.7469263712634749 }, "max_calibration_gap": 0.7469263712634749, "mse": 21.32687010730149 } }, "ep_gradient_boosted": { "select": { "arm": "ep_gradient_boosted", "calibration_by_yardline_decile": { "0": 0.20404183545397903, "1": 0.08428442297355154, "2": 0.08626657748792699, "3": 0.07619687252685248, "4": 0.014714128627152157, "5": 0.018694626023378458, "6": 0.1725812129193669, "7": 0.08685837130596075, "8": 0.36139415465753183, "9": 0.7089114815597612 }, "log_loss_multiclass": 1.288887915397228, "max_calibration_gap": 0.7089114815597612, "mse": 20.25909289778927 }, "test": { "arm": "ep_gradient_boosted", "calibration_by_yardline_decile": { "0": 0.059636626064825116, "1": 0.02047784162672972, "2": 0.008821743074098887, "3": 0.019788858188400926, "4": 0.21175416172464123, "5": 0.04838303350764117, "6": 0.03131575888465821, "7": 0.09486285441252384, "8": 0.16289327778142434, "9": 0.5199686317980595 }, "log_loss_multiclass": 1.296161424887337, "max_calibration_gap": 0.5199686317980595, "mse": 20.93508525341922 } }, "ep_multinomial_logistic": { "select": { "arm": "ep_multinomial_logistic", "calibration_by_yardline_decile": { "0": 1.2993532420371778, "1": 0.5677856493834228, "2": 0.36357739031919767, "3": 0.041026950931696504, "4": 0.12708632570889034, "5": 0.14799723065586745, "6": 0.11398581768031524, "7": 0.3151768827971977, "8": 0.6142840937917338, "9": 0.815206262326966 }, "log_loss_multiclass": 1.3538091432707886, "max_calibration_gap": 1.2993532420371778, "mse": 21.89858699179205 }, "test": { "arm": "ep_multinomial_logistic", "calibration_by_yardline_decile": { "0": 1.237301143613435, "1": 0.5191612153786309, "2": 0.27025570621172434, "3": 0.05384830346622582, "4": 0.35470237492567414, "5": 0.20882246403973181, "6": 0.02722639887287115, "7": 0.47757171835080353, "8": 0.2801527708122865, "9": 0.4716624643474897 }, "log_loss_multiclass": 1.3646131442712068, "max_calibration_gap": 1.237301143613435, "mse": 22.557042486985814 } }, "ep_source_replay": { "select": { "arm": "ep_source_replay", "calibration_by_yardline_decile": { "0": 0.12206356071786661, "1": 0.0368144534262731, "2": 0.05628224949642924, "3": 0.05997642454887009, "4": 0.02136334034421239, "5": 0.00897587667624089, "6": 0.1434412156128897, "7": 0.07200491149802735, "8": 0.17603441609453213, "9": 0.5333532524254528 }, "max_calibration_gap": 0.5333532524254528, "mse": 20.209638027253845 }, "test": { "arm": "ep_source_replay", "calibration_by_yardline_decile": { "0": 0.008538603031217562, "1": 0.0177577663998969, "2": 0.009098940138423828, "3": 0.02714106629429347, "4": 0.2022889056095738, "5": 0.023849080319559013, "6": 0.06559610004935457, "7": 0.07380310477771973, "8": 0.011099680133424161, "9": 0.6986094115958311 }, "max_calibration_gap": 0.6986094115958311, "mse": 20.931054273729913 } } }, "wp": { "wp_gradient_boosted": { "select": { "arm": "wp_gradient_boosted", "brier": 0.16835756919892741, "log_loss": 0.4981086075100995, "log_loss_final_5min": 0.30905271159937925, "max_reliability_gap": 0.07642947505104664, "reliability_by_bin": { "0": 0.0008852901845132176, "1": 0.030115626745577356, "2": 0.07642947505104664, "3": 0.06114081304111757, "4": 0.003901578096207392, "5": 0.004518852459544531, "6": 0.033380209159699414, "7": 0.03864838150354721, "8": 0.04206547423672491, "9": 0.0079901109515641 }, "ties_excluded": 0 }, "test": { "arm": "wp_gradient_boosted", "brier": 0.1662563468458739, "log_loss": 0.4881413959043057, "log_loss_final_5min": 0.2693418325216312, "max_reliability_gap": 0.07061288843590642, "reliability_by_bin": { "0": 0.022496059470019092, "1": 0.07061288843590642, "2": 0.038377828240318906, "3": 0.028326050476679687, "4": 0.024908114512384782, "5": 0.0032884083809551212, "6": 0.0073033402782020085, "7": 0.021458108315410396, "8": 0.06780973975356241, "9": 0.020997528804732823 }, "ties_excluded": 0 } }, "wp_logistic": { "select": { "arm": "wp_logistic", "brier": 0.1611266845417531, "log_loss": 0.48037620085758004, "log_loss_final_5min": 0.2911871898081383, "max_reliability_gap": 0.0313522169463637, "reliability_by_bin": { "0": 0.0028505667253894904, "1": 0.0313522169463637, "2": 0.016053014864696263, "3": 0.008637795312373242, "4": 0.0009082876046255794, "5": 0.0056393493006744455, "6": 0.02459624101434954, "7": 0.01010364956376908, "8": 0.026610904578644123, "9": 0.003206024225784976 }, "ties_excluded": 0 }, "test": { "arm": "wp_logistic", "brier": 0.15975525903751, "log_loss": 0.47193333624172734, "log_loss_final_5min": 0.26252021834665207, "max_reliability_gap": 0.03468042379145819, "reliability_by_bin": { "0": 0.026861889461470334, "1": 0.034358981952357745, "2": 0.01095652610380976, "3": 0.023415410644086587, "4": 0.0075907201016326975, "5": 0.004304946696282674, "6": 0.02049081678741682, "7": 0.018963708540026047, "8": 0.03468042379145819, "9": 0.03051608871396061 }, "ties_excluded": 0 } }, "wp_source_replay": { "select": { "arm": "wp_source_replay", "brier": 0.16051854763761184, "log_loss": 0.47843808574422925, "log_loss_final_5min": 0.28054903123085306, "max_reliability_gap": 0.03720621566625597, "reliability_by_bin": { "0": 0.0015611447312602264, "1": 0.03720621566625597, "2": 0.014278017601075749, "3": 0.01436961755194982, "4": 0.013791980594609288, "5": 0.006446557528554719, "6": 0.014843969486460828, "7": 0.018955823483621237, "8": 0.017212695868699446, "9": 0.0008406594733940675 }, "ties_excluded": 0 }, "test": { "arm": "wp_source_replay", "brier": 0.15801246671205457, "log_loss": 0.46635982832031647, "log_loss_final_5min": 0.2481667953057666, "max_reliability_gap": 0.03923076788434883, "reliability_by_bin": { "0": 0.02890623017820952, "1": 0.03923076788434883, "2": 0.0015098688289186812, "3": 0.010865055651072819, "4": 0.00976364414866765, "5": 0.0015207400102011936, "6": 0.016243358136915798, "7": 0.016856219704065367, "8": 0.0347878137986497, "9": 0.025403800563027823 }, "ties_excluded": 0 } } } }, "row_mask_sha256": "5e0ce834cb951844d4a103a35299bd463ccc4e6b30fbc476c4812c349e088c49", "rows": { "select": 38933, "test": 38188, "train": 261988 }, "seed": 7 }, { "design_constants": { "bootstrap_draws": 2000, "calibration_fit_seasons": "1999-2023", "cohort_variants": { "fringe_10_40": [ 0.1, 0.4 ], "fringe_5_25": [ 0.05, 0.25 ], "fringe_5_35": [ 0.05, 0.35 ] }, "replacement_seasons": "2016-2024", "test_once": 2024 }, "experiment_id": "forge_foundations_v1", "holdout_2025_accessed": false, "library_versions": { "numpy": "2.5.1", "polars": "1.43.2", "python": "3.14.6" }, "points_to_wins": { "eras": { "1999-2010": { "points_per_win": 36.348, "pythagorean_exponent_fit": 2.53 }, "2011-2020": { "points_per_win": 35.113, "pythagorean_exponent_fit": 2.84 }, "2021-2024": { "points_per_win": 38.881, "pythagorean_exponent_fit": 2.49 } }, "linear": { "ci90": [ 34.979, 37.184 ], "method": "linear wins~point_diff, season-block bootstrap", "points_per_win": 36.11, "team_seasons": 797 }, "note": "era exponents drift upward with scoring environment; the folklore 2.37 constant is a comparator only (Week-1 verification, contested source)", "pythagorean_exponent_fit_1999_2023": 2.65, "test_once_2024_mae_wins": { "fitted_exponent": 1.2392, "folklore_2.37": 1.3086 } }, "ran_at": "2026-08-06T00:31:11+00:00", "replacement_cohorts": { "QB": { "league_rate_epa_per_opp": 0.0184, "sensitivity_spread": 0.0406, "variants": { "fringe_10_40": { "below_league_average_by": 0.1367, "opportunities": 21412, "player_seasons": 156, "replacement_rate_epa_per_opp": -0.1183 }, "fringe_5_25": { "below_league_average_by": 0.1772, "opportunities": 12575, "player_seasons": 167, "replacement_rate_epa_per_opp": -0.1589 }, "fringe_5_35": { "below_league_average_by": 0.1469, "opportunities": 21667, "player_seasons": 217, "replacement_rate_epa_per_opp": -0.1285 } } }, "RB": { "league_rate_epa_per_opp": -0.0321, "sensitivity_spread": 0.0288, "variants": { "fringe_10_40": { "below_league_average_by": -0.0085, "opportunities": 49880, "player_seasons": 559, "replacement_rate_epa_per_opp": -0.0237 }, "fringe_5_25": { "below_league_average_by": -0.0372, "opportunities": 37220, "player_seasons": 714, "replacement_rate_epa_per_opp": 0.0051 }, "fringe_5_35": { "below_league_average_by": -0.0263, "opportunities": 52317, "player_seasons": 836, "replacement_rate_epa_per_opp": -0.0059 } } }, "WR_TE": { "league_rate_epa_per_opp": 0.1752, "sensitivity_spread": 0.0375, "variants": { "fringe_10_40": { "below_league_average_by": -0.0524, "opportunities": 98947, "player_seasons": 1089, "replacement_rate_epa_per_opp": 0.2276 }, "fringe_5_25": { "below_league_average_by": -0.0149, "opportunities": 117822, "player_seasons": 1832, "replacement_rate_epa_per_opp": 0.1901 }, "fringe_5_35": { "below_league_average_by": -0.0258, "opportunities": 132518, "player_seasons": 1929, "replacement_rate_epa_per_opp": 0.201 } } } }, "status": "FORGE remains scaffold: this packet is the calibration/cohort evidence a future FORGE release must cite; no player values produced" }, { "design": "docs/validation/QB_CORE_PILOT_DESIGN.md", "experiment_id": "qb_core_pilot_v1", "fitted_state_sha256_2024": "95ab76b78a28c9c7af5d4987759bc7bce367c5ea2d8cccd6b67751c9148b41ee", "gates": { "deterministic_replay": true, "no_mechanical_block": true, "stability_beats_raw": true, "test_confirms_vs_raw": true, "yoy_beats_all_baselines": true }, "holdout_2025_accessed": false, "library_versions": { "numpy": "2.5.1", "polars": "1.43.2", "python": "3.14.6", "sklearn": "1.9.0" }, "ran_at": "2026-08-06T00:11:58+00:00", "sanity_board": { "authority_note": "board may block mechanical impossibilities; it may never adjust a rating because a result looks surprising", "bottom_10": [ { "dropbacks": 414, "qb": "00-0039150", "raw_epa_db": -0.0732, "ridge": -0.0325 }, { "dropbacks": 321, "qb": "00-0033662", "raw_epa_db": -0.1015, "ridge": -0.0419 }, { "dropbacks": 374, "qb": "00-0039851", "raw_epa_db": -0.0839, "ridge": -0.0439 }, { "dropbacks": 371, "qb": "00-0035710", "raw_epa_db": -0.0979, "ridge": -0.044 }, { "dropbacks": 277, "qb": "00-0039164", "raw_epa_db": -0.1593, "ridge": -0.0501 }, { "dropbacks": 636, "qb": "00-0039918", "raw_epa_db": -0.089, "ridge": -0.0512 }, { "dropbacks": 333, "qb": "00-0035289", "raw_epa_db": -0.1372, "ridge": -0.0527 }, { "dropbacks": 344, "qb": "00-0039152", "raw_epa_db": -0.1905, "ridge": -0.0784 }, { "dropbacks": 250, "qb": "00-0033537", "raw_epa_db": -0.2462, "ridge": -0.0915 }, { "dropbacks": 250, "qb": "00-0039376", "raw_epa_db": -0.3391, "ridge": -0.1104 } ], "flags": [ { "code": "TEAM_ATTRIBUTION", "detail": "context adjustment moved rate by >0.15 EP/dropback", "qb": "00-0034796" } ], "mechanical_blocks": [], "qualifying_qbs": 39, "season_reviewed": 2024, "top_10": [ { "dropbacks": 494, "qb": "00-0034796", "raw_epa_db": 0.3413, "ridge": 0.1681 }, { "dropbacks": 568, "qb": "00-0033106", "raw_epa_db": 0.2765, "ridge": 0.1448 }, { "dropbacks": 497, "qb": "00-0034857", "raw_epa_db": 0.2621, "ridge": 0.1235 }, { "dropbacks": 612, "qb": "00-0034855", "raw_epa_db": 0.1754, "ridge": 0.0936 }, { "dropbacks": 421, "qb": "00-0036212", "raw_epa_db": 0.2034, "ridge": 0.0843 }, { "dropbacks": 701, "qb": "00-0036442", "raw_epa_db": 0.1434, "ridge": 0.0838 }, { "dropbacks": 482, "qb": "00-0037834", "raw_epa_db": 0.1623, "ridge": 0.0771 }, { "dropbacks": 442, "qb": "00-0036264", "raw_epa_db": 0.1203, "ridge": 0.0635 }, { "dropbacks": 617, "qb": "00-0033873", "raw_epa_db": 0.1088, "ridge": 0.0588 }, { "dropbacks": 545, "qb": "00-0036355", "raw_epa_db": 0.1202, "ridge": 0.0578 } ] }, "split_half_stability": { "anya": { "correlation": 0.4396, "qb_seasons": 278 }, "epa_cpoe_composite": { "correlation": 0.4647, "qb_seasons": 278 }, "epa_db": { "correlation": 0.4683, "qb_seasons": 278 }, "qb_context_ridge": { "correlation": 0.4731, "qb_seasons": 278 } }, "universe": { "dropback_rows": 173656, "min_dropbacks": 200, "ridge_alpha_predeclared": 500.0, "seasons": [ 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024 ] }, "verdict": "PROMOTE_TO_RESEARCH", "year_over_year_selection": { "anya": { "correlation": 0.409, "pairs": 178 }, "epa_cpoe_composite": { "correlation": 0.3844, "pairs": 178 }, "epa_db": { "correlation": 0.4251, "pairs": 178 }, "qb_context_ridge": { "correlation": 0.4325, "pairs": 178 } }, "year_over_year_test_once": { "anya": { "correlation": 0.4375, "pairs": 26 }, "epa_cpoe_composite": { "correlation": 0.5366, "pairs": 26 }, "epa_db": { "correlation": 0.5151, "pairs": 26 }, "qb_context_ridge": { "correlation": 0.5256, "pairs": 26 } } }, { "cross_family_agreement": { "ol_event_share~ol_role_prior": 0.9252, "ol_player_ridge~ol_event_share": 0.9286, "ol_player_ridge~ol_role_prior": 0.9962, "ol_player_ridge~ol_unit_ridge": 0.8276, "ol_unit_ridge~ol_event_share": 0.8804, "ol_unit_ridge~ol_role_prior": 0.8247 }, "design": "docs/validation/TRENCH_VISIBILITY_DESIGN.md", "experiment_id": "trench_visibility_v1", "gates": { "cross_family_agreement_ge_0.5": true, "no_mechanical_block": true, "sim_regime_a_recovery_ge_0.5": true, "sim_regime_c_expected_failure_observed": true }, "holdout_2025_accessed": false, "identifiability_summary": { "lineup_entropy_median": 1.3819, "max_cooccurrence_median": 1.0, "max_cooccurrence_p90": 1.0, "qualifying_player_seasons": 1398 }, "implementation_notes": [ "slot prior uses T/G/C (participation position lists are unordered; LT/RT split unobservable \u2014 recorded deviation, not a gate change)", "OL membership detected via roster position_group==OL from the players capture: per-play listed positions exist only in FTN-era participation files, so roster identity is the uniform cross-season detector", "orientation: lower = better (bad-outcome rate); flip at display only" ], "library_versions": { "numpy": "2.5.1", "polars": "1.43.2", "python": "3.14.6", "sklearn": "1.9.0" }, "mean_cross_family_agreement": 0.8971, "qualifying_player_seasons_common": 1398, "ran_at": "2026-08-06T00:25:24+00:00", "ridge_alpha_predeclared": 1000.0, "rows_per_season": { "2016": 18479, "2017": 18235, "2018": 18719, "2019": 18812, "2020": 18957, "2021": 19670, "2022": 19174, "2023": 19505, "2024": 18946 }, "simulation_expectations": { "a_realistic": "expected >= 0.5", "b_high_churn": "expected best", "c_pathological_stability": "EXPECTED TO FAIL (< 0.3) \u2014 method must report its own blindness" }, "simulation_recovery_spearman": { "a": 0.5306, "b": 0.5928, "c": 0.279 }, "verdict": "INDIVIDUAL_RESEARCH_VALUES_PERMITTED" }, { "cross_family_agreement": { "ol_event_share~ol_role_prior": 0.3282, "ol_player_ridge~ol_event_share": 0.332, "ol_player_ridge~ol_role_prior": 0.9981, "ol_player_ridge~ol_unit_ridge": 0.6728, "ol_unit_ridge~ol_event_share": 0.262, "ol_unit_ridge~ol_role_prior": 0.6699 }, "design": "docs/validation/TRENCH_VISIBILITY_DESIGN.md (second wave clause)", "experiment_id": "trench_visibility_v2_front", "gates": { "cross_family_agreement_ge_0.5": true, "no_mechanical_block": true, "sim_bound_from_wave1": true }, "holdout_2025_accessed": false, "identifiability_summary": { "idl_count": 70, "lb_count": 250, "max_cooccurrence_median": 0.8043, "max_cooccurrence_p90": 0.9883, "qualifying_player_seasons": 320 }, "implementation_notes": [ "outcome = offense rush success (lower = better defender)", "offense team-season occupies the QB control slot (blocking proxy)", "wave-1 simulation is the conservative recovery bound: fronts rotate far more than OL (regime-b-like), and regime b recovered BEST (0.593); no separate front simulation was run \u2014 disclosed reliance", "front size 2-6 on designed rushes; equal 1/n event allocation" ], "mean_cross_family_agreement": 0.5439, "qualifying_player_seasons_common": 320, "ran_at": "2026-08-06T00:33:35+00:00", "rows_per_season": { "2016": 11741, "2017": 12365, "2018": 11791, "2019": 11896, "2020": 12158, "2021": 12801, "2022": 13006, "2023": 12855, "2024": 12692 }, "verdict": "INDIVIDUAL_RESEARCH_VALUES_PERMITTED" } ], "generated_at_utc": "2026-08-06T00:36:55+00:00", "note": "Every model decision on this platform, including the NULL results. Verdicts come from gates frozen before scoring.", "schema_version": "gridiron-experiments-v1" }