"""Rule-based interpretation of framework results. Deterministic: the same results always produce the same diagnostics and text. Thresholds follow ``write_report`` in ppi_benchmark_controls.py v1.0.0 and §10 of the manuscript repository README. Each diagnostic states what was measured, what it implies for building a model on the dataset, and the caveat that limits it. Status vocabulary: ``pass`` (no concern), ``warn`` (needs a control when reporting), ``fail`` (the benchmark's apparent signal is materially explained by this artefact), ``info`` (context, not scored), ``skipped`` (not computable for these inputs). """ from __future__ import annotations THRESHOLDS = { "smd_warn": 0.25, "smd_fail": 0.8, "degree_auc_warn": 0.65, "degree_auc_fail": 0.75, "length_auc_warn": 0.65, "novelty_drop_warn": 0.03, "novelty_drop_fail": 0.10, "chance_band": 0.05, "modest_band": 0.10, "components_warn": 30, "components_fail": 10, "species_auc_warn": 0.6, "species_auc_fail": 0.7, "cross_species_warn": 0.10, "positive_cross_max": 0.05, } CAVEAT_NEGATIVES = ("Constructed negatives are putative (unobserved) pairs, not experimentally established " "non-interactions.") CAVEAT_DEGREE = ("Degree is computed from the same incomplete positive graph that supplies the labels; it is a " "retrospective audit feature, not an input available for a protein with no known interactions.") CAVEAT_INFERENCE = ("A non-significant test does not establish equivalence to chance; few connected components " "limit what can be concluded.") def _f(x, nd=3): return "n/a" if x is None else f"{x:.{nd}f}" def _index(rows, *keys): return {tuple(r.get(k) for k in keys): r for r in rows or []} def _diag(id_, title, status, headline, detail, implication, caveat="", evidence=None, chapter=""): return {"id": id_, "title": title, "status": status, "headline": headline, "detail": detail, "ml_implication": implication, "caveat": caveat, "evidence": evidence or {}, "chapter": chapter} def _preferred_model(rows, rep): models = {r["model"] for r in rows if r["representation"] == rep} for m in ("mlp", "rf", "xgb", "lr"): if m in models: return m return None def interpret(res: dict) -> dict: T = THRESHOLDS diags = [] audit = res["audit"] reps = res["inputs"]["representations"] rep0 = reps[0] supplied = res["inputs"]["supplied_negatives"] # 1. data integrity -------------------------------------------------------------- iss = audit["issues"] removed = iss["duplicate_pairs_removed"] + iss["self_pairs_removed"] + iss["conflicting_label_pairs_removed"] frac = removed / max(audit["input"]["n_pairs"], 1) status = "pass" if removed == 0 else ("warn" if frac > 0.01 or iss["conflicting_label_pairs_removed"] else "pass") diags.append(_diag( "integrity", "Data integrity", status, "Clean input" if removed == 0 else f"{removed:,} rows removed during cleaning ({frac:.1%})", f"{iss['duplicate_pairs_removed']:,} duplicate pairs ({iss['reversed_duplicate_rows']:,} reversed), " f"{iss['self_pairs_removed']:,} self-pairs and {iss['conflicting_label_pairs_removed']:,} pairs with conflicting " f"labels were removed; {audit['canonical']['n_pairs']:,} canonical pairs remain.", "Reversed duplicates (A–B and B–A) placed on both sides of a random split leak labels directly. " "Train on the canonical pair list, not the raw file.", evidence={"removed": removed, "fraction": frac}, chapter="audit")) # 2. negative degree bias ------------------------------------------------------------ bal = {r["negative_set"]: r for r in res["balance"]} if supplied and "N0" in bal: smd = bal["N0"]["smd"] or 0.0 st = "fail" if abs(smd) > T["smd_fail"] else "warn" if abs(smd) > T["smd_warn"] else "pass" diags.append(_diag( "negative_bias", "Degree bias in supplied negatives", st, f"Endpoint degree SMD {smd:.2f} between positives and supplied negatives", f"Positive pairs join proteins with mean positive-network degree {bal['N0']['positive_mean']:.2f}, " f"supplied negatives {bal['N0']['negative_mean']:.2f}. Degree-matched N2 brings the SMD to " f"{_f(bal.get('N2', {}).get('smd'), 2)}.", "A model can tell the classes apart by how well-studied the proteins are, without learning anything " "about interaction. Compare against degree-matched negatives before trusting a score." if st != "pass" else "Supplied negatives are balanced on endpoint degree.", CAVEAT_NEGATIVES, {"smd": smd, "smd_n2": bal.get("N2", {}).get("smd")}, "negatives")) else: diags.append(_diag("negative_bias", "Degree bias in supplied negatives", "info", "No supplied negatives: random unobserved pairs (N1) are the reference set", "Without label-0 rows there is no original negative set to audit.", "Choose a negative-construction rule deliberately; random pairs are degree-biased too " f"(N1 SMD {_f(bal.get('N1', {}).get('smd'), 2)}).", CAVEAT_NEGATIVES, chapter="negatives")) # 3. degree shortcut ------------------------------------------------------------- topo = _index(res["topology"], "feature", "model", "evaluated_on") d0 = (topo.get(("degree", "lr", "N0")) or {}).get("roc_auc") d2 = (topo.get(("degree", "lr", "N2")) or {}).get("roc_auc") if d0 is not None: st = "fail" if d0 > T["degree_auc_fail"] else "warn" if d0 > T["degree_auc_warn"] else "pass" diags.append(_diag( "degree_shortcut", "Degree-only shortcut", st, f"Degree alone reaches ROC-AUC {d0:.3f} on the reference labels", f"A logistic model on four endpoint-degree summaries, with no sequence information, scores " f"{d0:.3f} on N0 and {_f(d2)} on degree-matched N2 negatives.", "Most of the benchmark's label separation is reproducible from network degree. Headline scores on " "these labels mix interaction signal with annotation density." if st == "fail" else "Degree carries part of the label signal; report a degree-only baseline next to any model." if st == "warn" else "Degree alone does not separate the labels.", CAVEAT_DEGREE, {"degree_auc_n0": d0, "degree_auc_n2": d2}, "topology")) l0 = (topo.get(("length", "lr", "N0")) or {}).get("roc_auc") if l0 is not None: st = "warn" if l0 > T["length_auc_warn"] else "pass" diags.append(_diag( "length_shortcut", "Length-only shortcut", st, f"Sequence length alone: ROC-AUC {l0:.3f}", "Endpoint sequence-length summaries used as the only features.", "Length carries label information; length-matched negatives (N3/N5) are advisable." if st == "warn" else "Sequence length does not separate the labels.", chapter="topology", evidence={"length_auc_n0": l0})) # 4. protein-novelty drop ----------------------------------------------------------- nov = _index(res["novelty"], "representation", "regime", "model") m = _preferred_model(res["novelty"], rep0) r1 = (nov.get((rep0, "R1", m)) or {}).get("roc_auc") r3 = (nov.get((rep0, "R3", m)) or {}).get("roc_auc") r0 = (nov.get((rep0, "R0", m)) or {}).get("roc_auc") if r1 is not None and r3 is not None: drop = r1 - r3 st = "fail" if drop > T["novelty_drop_fail"] else "warn" if drop > T["novelty_drop_warn"] else "pass" diags.append(_diag( "novelty_drop", "Performance on unseen proteins", st, f"ROC-AUC falls by {drop:.3f} from seen (R1) to unseen proteins (R3)", f"{rep0} {m.upper()} with the training set held fixed: R0 {_f(r0)}, R1 {_f(r1)}, R3 {_f(r3)}.", "Random pair splits overstate performance on new proteins. Report the protein-disjoint R3 split " "if the model is meant for proteins without known interactions." if st != "pass" else "Discrimination holds on unseen proteins.", "R3 test sets are small; see the uncertainty chapter.", {"r0": r0, "r1": r1, "r3": r3, "drop": drop}, "novelty")) # 5. signal after consistent degree matching ------------------------------------------ rv = _index(res["revalidation"], "representation", "regime", "negative_set") unc = _index(res["uncertainty"], "representation", "negative_set") best = None for rep in reps: v = (rv.get((rep, "R3", "N2")) or {}).get("roc_auc") if v is not None and (best is None or v > best[1]): best = (rep, v) if best: rep, v = best u = unc.get((rep, "N2")) or {} sig = bool(u.get("above_chance_both_methods")) n0v = (rv.get((rep, "R3", "N0")) or {}).get("roc_auc") if v < 0.5 - T["chance_band"]: st, head = "fail", f"Below chance ({v:.3f}) with degree-matched negatives on unseen proteins" elif abs(v - 0.5) < T["chance_band"]: st, head = "fail", f"Close to chance ({v:.3f}) with degree-matched negatives on unseen proteins" elif v - 0.5 < T["modest_band"] or not sig: st, head = "warn", f"Modest signal ({v:.3f}) with degree-matched negatives on unseen proteins" else: st, head = "pass", f"Sequence signal survives degree matching on unseen proteins ({v:.3f})" diags.append(_diag( "matched_signal", "Signal after degree matching", st, head, f"Best representation {rep}, MLP trained and tested with N2 negatives at R3: ROC-AUC {v:.3f} " f"(N0: {_f(n0v)}). Component bootstrap 95% CI {_f(u.get('boot_low'))}–{_f(u.get('boot_high'))}; " f"above chance under both corrected tests: {'yes' if sig else 'no'}.", "With degree shortcuts removed and proteins unseen, there is little sequence signal for a model to " "learn from this dataset as constructed." if st == "fail" else "Some sequence signal remains, but it is small or uncertain; treat performance claims cautiously." if st == "warn" else "The dataset supports learning sequence-encoded interaction signal.", CAVEAT_INFERENCE, {"representation": rep, "auc_n2": v, "auc_n0": n0v, "significant": sig}, "revalidation")) # 6. independent units --------------------------------------------------------------- if res["uncertainty"]: comps = [r["n_components"] for r in res["uncertainty"]] cmin, cmax = min(comps), max(comps) st = "fail" if cmin < T["components_fail"] else "warn" if cmin < T["components_warn"] else "pass" diags.append(_diag( "independent_units", "Independent test units", st, f"R3 test graphs contain {cmin}–{cmax} connected components", "Test pairs that share proteins are not independent. Uncertainty is computed by resampling whole " "connected components of the test-pair graph.", "Confidence intervals are wide and tests have little power; differences between models of a few " "hundredths are not interpretable." if st != "pass" else "There are enough independent units for component-aware intervals to be informative.", CAVEAT_INFERENCE, {"min_components": cmin, "max_components": cmax}, "uncertainty")) # 7-8. species ------------------------------------------------------------------------ sp = res.get("species") if sp: ss = {r["evaluated_on"]: r for r in sp["same_species_shortcut"]} s2 = (ss.get("N2") or {}).get("roc_auc") s2s = (ss.get("N2-species") or {}).get("roc_auc") if s2 is not None: st = "fail" if s2 > T["species_auc_fail"] else "warn" if s2 > T["species_auc_warn"] else "pass" diags.append(_diag( "taxonomic_shortcut", "Taxonomic shortcut", st, f"'Same species?' alone scores ROC-AUC {s2:.3f} on pooled degree-matched negatives", f"Scoring a pair 1 when both proteins come from the same organism gives {s2:.3f} on N2 and " f"{_f(s2s)} on species-matched N2-species negatives.", "Pooled negatives that join organisms can be recognised from taxonomy alone; pooled performance " "partly measures species identification. Use species-matched negatives (N2-species) or " "evaluate within species." if st != "pass" else "Taxonomy does not separate the labels.", CAVEAT_NEGATIVES, {"same_species_auc_n2": s2, "same_species_auc_n2_species": s2s}, "species")) cross = {r["set"]: r for r in sp["cross_species"]} pc = (cross.get("positives") or {}).get("cross_species_fraction") or 0.0 nc = (cross.get("N2") or {}).get("cross_species_fraction") if nc is not None: st = "warn" if nc > T["cross_species_warn"] and pc < T["positive_cross_max"] else "pass" diags.append(_diag( "cross_species", "Cross-species negatives", st, f"{nc:.1%} of pooled N2 negatives join two species (positives: {pc:.1%})", "Fraction of pairs whose endpoints are assigned to different organisms, per negative set.", "Negatives and positives differ in species composition, which a model can exploit." if st == "warn" else "Species composition is similar between positives and negatives.", evidence={"positive_cross": pc, "n2_cross": nc}, chapter="species")) wm = sp.get("within_metrics") or [] within = [r for r in wm if r["representation"] == rep0 and r["model"] == "mlp" and r["negative_set"] == "N2"] if within: parts = ", ".join(f"{r['species']} {_f(r['roc_auc'])}" for r in within) diags.append(_diag( "within_species", "Within-species signal", "info", f"Within-species {rep0} MLP with N2 negatives: {parts}", "Degree statistics and negatives are rebuilt inside each organism; positives and negatives are " "split 80/20 at random (proteins may recur, as in the source protocol).", "Within-species estimates remove the taxonomic shortcut but not protein reuse; compare with the " "pooled R3 result.", evidence={"rows": within}, chapter="species")) # summary ------------------------------------------------------------------------- scored = [d for d in diags if d["status"] in ("pass", "warn", "fail")] n_fail = sum(d["status"] == "fail" for d in scored) n_warn = sum(d["status"] == "warn" for d in scored) score = max(0, min(100, 100 - 15 * n_fail - 5 * n_warn)) ids = {d["id"]: d["status"] for d in diags} if ids.get("degree_shortcut") == "fail" and ids.get("matched_signal") == "fail": verdict, line = "Shortcut-dominated", ("Most of this benchmark's apparent signal is reproduced by network " "degree, and little survives when that shortcut is removed.") elif n_fail: verdict, line = "Needs controls", "At least one artefact materially affects what models trained here will learn." elif n_warn: verdict, line = "Usable with care", "No dominant artefact, but some diagnostics need reporting alongside results." else: verdict, line = "Robust", "No diagnostic crossed its threshold." summary = {"verdict": verdict, "headline": line, "score": score, "n_fail": n_fail, "n_warn": n_warn, "n_pass": sum(d["status"] == "pass" for d in scored), "score_note": "Heuristic summary of the diagnostics below (100 − 15 per concern − 5 per caution); " "not a statistical quantity."} # recommendations ---------------------------------------------------------------- recs = [] if ids.get("integrity") in ("warn", "pass") and audit["issues"]["duplicate_pairs_removed"]: recs.append({"title": "Train on the canonical pair list", "file": "canonical_pairs.tsv", "text": "Duplicates and reversed duplicates have been removed; use this file instead of the raw input."}) if ids.get("negative_bias") in ("warn", "fail") or ids.get("degree_shortcut") in ("warn", "fail"): recs.append({"title": "Use degree-matched negatives", "file": "negatives/N2.tsv", "text": "Train and evaluate with the same degree-matched definition (N2, or N5 to also match " "length) and report the degree-only baseline."}) if ids.get("novelty_drop") in ("warn", "fail"): recs.append({"title": "Report a protein-disjoint split", "file": "splits/R3/", "text": "R1/R2/R3 share one training set; R3 test pairs contain only proteins never seen in " "training. Report R3 for claims about new proteins."}) if ids.get("independent_units") in ("warn", "fail"): recs.append({"title": "Report component-aware uncertainty", "file": "tables/component_aware_uncertainty.csv", "text": "Quote bootstrap and jackknife intervals over connected components, and add an " "independent benchmark where possible."}) if ids.get("taxonomic_shortcut") in ("warn", "fail") or ids.get("cross_species") == "warn": recs.append({"title": "Match negatives within species", "file": "negatives/N2-species.tsv", "text": "Species-matched degree negatives remove the 'same organism' shortcut; also report " "within-species results."}) recs.append({"title": "Report both controls", "file": "results.json", "text": "Protein reuse and negative construction are distinct problems: a protein-disjoint split can " "keep biased negatives, and balanced negatives can still let proteins recur."}) return {"diagnostics": diags, "summary": summary, "recommendations": recs, "methods_text": methods_text(res), "thresholds": THRESHOLDS} def methods_text(res: dict) -> str: a = res["audit"] cfg = res["config"] splits = {r["regime"]: r for r in res["splits"]} sets = ", ".join(r["negative_set"] for r in res["balance"]) models = ", ".join(res["inputs"]["models"]) txt = (f"The dataset ({a['input']['n_proteins']:,} proteins, {a['input']['n_pairs']:,} pairs) was audited and " f"canonicalized to {a['canonical']['n_pairs']:,} unique undirected pairs " f"({a['canonical']['n_positive']:,} positive, {a['canonical']['n_negative']:,} negative). " f"Following the protein-novelty and negative-sampling control framework, a random pair split (R0) and a " f"matched protein-novelty curve (R1 both proteins seen, R2 one unseen, R3 both unseen; shared training set " f"of {splits.get('R1', {}).get('train_pairs', 0):,} pairs) were built from one partition of " f"identical-sequence groups, with automated leakage gates. Putative negative sets ({sets}) were constructed " f"by one-sided substitution matched on positive-network degree and/or sequence length deciles. AAC+CTD " f"descriptors (167-d) with symmetric pair fusion were used with {models} models. Degree-only and " f"length-only classifiers were trained on the reference negatives and evaluated on each alternative set, " f"and an MLP was re-evaluated with each negative definition used consistently for training and testing at " f"R0 and R3. Uncertainty was quantified with a connected-component bootstrap ({cfg['n_boot']} replicates) " f"and a delete-one-component jackknife, with Benjamini–Hochberg adjustment. Seed {cfg['seed']}.") if res.get("species"): txt += (" Species diagnostics measured the cross-species fraction of each negative set, the ROC-AUC of a " "same-species indicator, species-matched degree negatives (N2-species), and within-species " "replication of the degree-only and sequence controls.") return txt