{"path": "scripts/check-briefs.py", "filename": "check-briefs.py", "size_bytes": 128491, "ext": ".py", "content": "#!/usr/bin/env python3\n\"\"\"\nShadow Dynamics — unified QUALITY runner.\n\nSingle source of truth for all editorial/structural checks against a brief\nHTML. Usable as:\n\n CLI: python3 scripts/check-briefs.py \n Lib: from check_briefs import run_checks\n HTTP: POST /quality-check {\"htmlContent\": \"...\"} (brief-saver)\n\nNOTE — module caching: brief-saver.py imports this file once and caches the\nmodule in sys.modules['sd_check_briefs'] for the lifetime of the process.\nEditing this file does NOT affect the HTTP endpoint until the service is\nrestarted: `sudo systemctl restart brief-saver`. The CLI re-imports per\ninvocation, so `python3 scripts/check-briefs.py` always reflects the\non-disk version. Test battery (`scripts/test-system.py`) also runs CLI-side,\nso 72/72 passing does not prove the live API picked up the changes.\n\nEach check returns either None (passed) or a string (failure reason).\nChecks are categorised:\n - BLOCKERS: hard fails — would refuse publication\n - WARNINGS: soft fails — logged for review\n\nVeracity (numerical-claim correctness) is OUT OF SCOPE for this runner;\nthat requires an LLM-judge or live web verification. The runner only\ncatches structural/editorial regressions and known patterns observed in\nprior failure modes.\n\nWhen a check fails in production (called from n8n), the runner appends to\n/var/log/sd-quality-failures.log so failure modes can be reviewed and\nthe prompt template iterated. That log is the input side of the feedback\nloop.\n\"\"\"\n\nfrom __future__ import annotations\n\nimport argparse\nimport glob\nimport json\nimport os\nimport re\nimport sys\nfrom dataclasses import dataclass, asdict\nfrom datetime import date, datetime, timezone\nfrom typing import Callable\n\n# ── BLOCKERS ─────────────────────────────────────────────────────────────────\n\ndef check_scaffold_pseudo_citations(html: str) -> str | None:\n \"\"\"True scaffold leaks: bare markers without content, or pseudo-pipe table breaks.\n\n Content-bearing markers like [WEB: FT] or [INFERENCE: based on X+Y] are\n intentional citations per EDITORIAL-CITE-DETAIL-01 and must remain in\n output. This check only blocks markers that are clearly unfilled templates\n or markers broken across HTML cells.\n \"\"\"\n patterns = [\n # Bare markers — model emitted the template label but no content.\n (r'\\[WEB\\s*\\]|\\[WEB:\\s*\\]', 'bare_web_marker'),\n (r'\\[INFERENCE\\s*\\]|\\[INFERENCE:\\s*\\]', 'bare_inference_marker'),\n (r'\\[POLL\\s*\\]|\\[POLL:\\s*\\]', 'bare_poll_marker'),\n # Pseudo-pipe form that breaks tables when rendered:\n (r'\\[\\w+\\s*\\\\?', 'table_broken_citation'),\n ]\n hits = [name for rx, name in patterns if re.search(rx, html)]\n return f'scaffold pseudo-citations: {\", \".join(hits)}' if hits else None\n\n\ndef check_result_label_duplicate(html: str) -> str | None:\n \"\"\"RESULT/RESULTADO duplicate prefix from prompt template.\"\"\"\n if re.search(r'RESULT:\\s*RESULT(?:ADO)?', html):\n return 'duplicated RESULT/RESULTADO label'\n return None\n\n\ndef check_preflight_section_leak(html: str) -> str | None:\n \"\"\"Pre-flight section §0 should never reach published HTML.\"\"\"\n if re.search(r'PRE-FLIGHT VERIFICATION|VERIFICACI[ÓO]N PREVIA', html):\n return 'pre-flight verification section leaked'\n return None\n\n\ndef check_event_date_leak(html: str) -> str | None:\n \"\"\"`EVENT DATE:` and `FORMAT DECISION` markers from prompt scaffolding.\"\"\"\n hits = []\n if re.search(r'EVENT DATE:\\s', html):\n hits.append('EVENT_DATE')\n if re.search(r'FORMAT DECISION', html):\n hits.append('FORMAT_DECISION')\n return f'prompt scaffolding leaked: {\", \".join(hits)}' if hits else None\n\n\ndef check_inline_paywall(html: str) -> str | None:\n \"\"\"Mid-content paywall divs — distinct from the legitimate footer CTA.\"\"\"\n if re.search(\n r'
]*style=\"margin:32px 0 0\"',\n html,\n ):\n return 'inline paywall div mid-content'\n if 'class=\"pw-full\"' in html or 'class=\"pw-btn\"' in html:\n return 'pw-full/pw-btn paywall block present'\n if re.search(r'filter:\\s*blur\\(\\s*5px', html):\n return 'blur(5px) paywall overlay present'\n if re.search(r'\\$19/(month|mes)\\b', html) and \\\n 'cta-box' in html.split('$19/', 1)[0][-200:]:\n return 'inline price tag mid-content'\n return None\n\n\ndef check_old_branding(html: str) -> str | None:\n \"\"\"Brand renamed Forecaster → Intelligence on 2026-04-26.\"\"\"\n if re.search(r'Shadow Dynamics Forecaster|SHADOW DYNAMICS FORECASTER',\n html):\n return 'old branding \"Forecaster\" present'\n return None\n\n\ndef check_about_scaffold_label_list(html: str) -> str | None:\n \"\"\"About-section sentence enumerating scaffold tags as a feature.\"\"\"\n if re.search(\n r'\\[WEF\\],\\s*\\[EURASIA\\],\\s*\\[POLL\\],\\s*\\[WEB\\],\\s*\\[INFERENCE\\]',\n html,\n ):\n return 'about-section lists scaffold-source tags as feature'\n return None\n\n\ndef check_hero_scaffold_pill(html: str) -> str | None:\n \"\"\"Hero pill that surfaced scaffold markers as a UI element.\"\"\"\n if re.search(r'\\[WEF\\]\\s*\\[EURASIA\\]\\s*\\[POLL\\]\\s*\\[INFERENCE\\]', html):\n return 'hero pill displays scaffold markers'\n return None\n\n\ndef check_section_count_parity(html: str) -> str | None:\n \"\"\"EN and ES h2 sections should have the same count (±1 tolerance).\"\"\"\n en = len(re.findall(\n r']*class=\"[^\"]*\\blang-en\\b[^\"]*\"', html\n ))\n es = len(re.findall(\n r']*class=\"[^\"]*\\blang-es\\b[^\"]*\"', html\n ))\n if en == 0 and es == 0:\n # Maybe wrapped sections, not lang-class h2 — skip\n return None\n if abs(en - es) > 1:\n return f'EN/ES section count mismatch: EN={en}, ES={es}'\n return None\n\n\ndef check_unbalanced_tables(html: str) -> str | None:\n \"\"\"Catch broken tables (e.g. from the pseudo-pipe artefact).\"\"\"\n opens = html.count('')\n tr_o = html.count('')\n td_o = html.count('')\n issues = []\n if opens != closes:\n issues.append(f'table {opens}/{closes}')\n if tr_o != tr_c:\n issues.append(f'tr {tr_o}/{tr_c}')\n if td_o != td_c:\n issues.append(f'td {td_o}/{td_c}')\n return f'unbalanced HTML: {\"; \".join(issues)}' if issues else None\n\n\n# ── Structural-integrity helpers (Brief 7 prep priorities item 4) ──\n# Origin: BACKLOG §ESTA SEMANA #8 item 4 (pre-generation hardening); was\n# \"design-only; not yet ticketed.\" Activated 2026-05-12 PM via\n# BRIER-DISCIPLINE-SET-APPLICATION-01 autonomous task path.\n\n_H3_INSIDE_P_RE = re.compile(\n r']*>(?:(?!

)[\\s\\S]){0,2000}? that does NOT have inline color\n # style override. Anchors: hex bg starting #0/#1 (slate/navy range),\n # CSS var --slate, or class cover/site-footer/dark-bg.\n r'(?:background\\s*:\\s*(?:#[01][0-9a-fA-F]{5}\\b|var\\(\\s*--slate\\s*\\))|'\n r'<(?:div|section|footer)[^>]*class=\"[^\"]*\\b(?:cover|site-footer|dark-bg)\\b[^\"]*\")'\n r'[^>]*>(?:(?!)[\\s\\S]){0,3000}?'\n r']*style=\"[^\"]*color)',\n re.IGNORECASE,\n)\n\n\ndef _check_toc_sync_issues(html: str) -> list[str]:\n \"\"\"Return list of TOC anchor href targets that don't exist as id= in the document.\n\n Scoped to anchors that look TOC-shaped (path is fragment-only, target is\n section/§-style identifier). Filters out external links + self-canonical.\n \"\"\"\n # Collect all id= values in the document\n ids = set(re.findall(r'\\bid=\"([^\"]+)\"', html, re.IGNORECASE))\n # Collect href=\"#xxx\" fragments\n anchors = re.findall(r']*href=\"#([^\"]+)\"', html, re.IGNORECASE)\n # Filter to TOC-shape (alphanumeric + dash; skip pure-numeric footnote anchors\n # like #fn1 that may be defined elsewhere via different schemes)\n toc_shaped = [a for a in anchors\n if re.fullmatch(r'[a-zA-Z][\\w\\-§]{1,80}', a)\n and not a.startswith('fn')\n and not a.startswith('cite')]\n missing = [a for a in toc_shaped if a not in ids]\n # Dedupe preserving order\n seen: set[str] = set()\n out: list[str] = []\n for a in missing:\n if a not in seen:\n seen.add(a)\n out.append(a)\n return out\n\n\ndef _check_sections_inside_tab_full(html: str) -> list[str]:\n \"\"\"Detect
placed inside
.\n\n Mirror of test_sources_disclaimer_outside_tab_full in scripts/test-system.py\n but scoped to single-brief input. Surfaces during quality_check (before\n promote), not only via test battery.\n \"\"\"\n m = re.search(r'
]*>', html)\n if not m:\n return []\n pos = m.end()\n depth = 1\n end_of_tab_full = None\n for tag in re.finditer(r'<(/?)div\\b', html[pos:]):\n if tag.group(1) == '/':\n depth -= 1\n if depth == 0:\n end_of_tab_full = pos + tag.end()\n break\n else:\n depth += 1\n if end_of_tab_full is None:\n return []\n issues: list[str] = []\n for sec_class in ('sources', 'disclaimer'):\n sec_pos = html.find(f'
str | None:\n \"\"\"Detect 4 classes of structural malformation that render unpredictably or hide content.\n\n Origin: BACKLOG §ESTA SEMANA #8 (Brief 7 prep priorities) item 4 — Brief 6\n surfaced sources-inside-tab-full + dark-bg-strong invisibility patterns;\n Brief 5 surfaced h3-inside-p; TOC drift hit Brief 2 / 4 historically. This\n check consolidates the family as a single brief-level WARN.\n\n Failure modes:\n\n 1. h3-inside-p — block-level

nested inside an open

; HTML invalid\n per the content-model rules; renders unpredictably across browsers and\n RSS readers.\n\n 2. dark-bg-bare-strong — inside a dark-background container\n (style background:#0X/#1X, var(--slate), or class cover/site-footer/\n dark-bg) without an inline color override. The brief CSS template's\n `strong{color:var(--slate)}` rule (#1A1F2E) renders bold labels\n invisible against dark navy/slate backgrounds. First hit cluster:\n a424cea (Brief 5 revisions) + feeff61 (Brief 6 executive alert).\n\n 3. toc-sync-broken — TOC anchor href targets missing as id= in document.\n Drift cause: section retitled but TOC entry not updated, or\n vice-versa. Symptom: in-page navigation breaks silently.\n\n 4. sections-inside-tab-full —

placed\n inside
. CSS rule `.tab-content{display:none}` on\n inactive tab hides everything inside; readers on the default Brief\n tab never see Sources or the legal disclaimer. Twice-bitten on Spain\n Blackout 2026-04-27. Test-battery analog:\n test_sources_disclaimer_outside_tab_full.\n\n WARNING level — calibrate against Brief 7+8 emissions before promoting to\n BLOCKER per feedback_runner_calibrate_then_ratchet. Reports each issue\n distinctly so per-mode false-positive rates calibrate independently.\n \"\"\"\n issues: list[str] = []\n\n if _H3_INSIDE_P_RE.search(html):\n issues.append('h3-inside-p')\n\n if _DARK_BG_BARE_STRONG_RE.search(html):\n issues.append('dark-bg-bare-strong')\n\n toc_missing = _check_toc_sync_issues(html)\n if toc_missing:\n sample = toc_missing[:3]\n more = f' (+{len(toc_missing) - 3} more)' if len(toc_missing) > 3 else ''\n issues.append(f'toc-sync-broken: missing id= for {sample}{more}')\n\n tab_issues = _check_sections_inside_tab_full(html)\n if tab_issues:\n issues.append(f'sections-inside-tab-full: {\", \".join(tab_issues)}')\n\n if issues:\n return f'{len(issues)} structural integrity issue(s): {\" || \".join(issues)}'\n return None\n\n\n# ── Posterior-predictive runner checks (FORECASTING-DISCIPLINE-LESSONS-2026-05-12-01\n# sub-item 3): runner predicts brief properties that SHOULD appear under Tier A\n# rules, then measures actual brief output. Catches drift between prompt-encoded\n# discipline and emitted briefs. Per Gelman posterior-predictive-check pattern. ──\n\n# Tolerant of SD header style drift across briefs (audit 2026-05-13):\n# - Bare \"FORMAL PREDICTIONS\" (Briefs 1-2)\n# - \"10. FORMAL PREDICTIONS\" / \"10. PREDICCIONES FORMALES\" (Brief 3 numeric prefix)\n# - \"SECTION VII: FORMAL PREDICTIONS\" / \"SECCIÓN VII: PREDICCIONES FORMALES\" (Brief 5)\n# - \"SECTION VIII: FORMAL PREDICTION\" singular + \"SECCIÓN VIII: PREDICCIÓN FORMAL\" (Brief 6)\n# Excludes \"HISTORIAL DE PREDICCIONES\" (track record, semantically distinct).\n# Bug history: prior regex required plural English-only with optional \"§\" prefix\n# only; silently bypassed Brief 5 (SECTION VII prefix) + Brief 6 (singular bilingual)\n# + Brief 3 (numeric prefix). E3/E5/E9 posterior-predictive checks did not fire on\n# those briefs at promote-time. Filed as RUNNER-FORMAL-PREDICTIONS-REGEX-BILINGUAL-01.\n_FORMAL_PREDICTIONS_SECTION_RE = re.compile(\n r']*>\\s*'\n r'(?:§\\s*|'\n r'SECTION\\s+[IVX]+\\s*:?\\s*|'\n r'SECCI[ÓO]N\\s+[IVX]+\\s*:?\\s*|'\n r'\\d+\\.\\s*'\n r')?'\n r'(?:FORMAL\\s+PREDICTION(?:S)?|PREDICCI[ÓO]N(?:ES)?\\s+FORMAL(?:ES)?)'\n r'[^<]*

(.*?)(?= str | None:\n \"\"\"E3-cluster-ref (posterior-predictive): when a prediction includes an\n \"Independence:\" sentence, that sentence should reference the cluster taxonomy\n OR positively assert structural independence — not just be a generic boilerplate.\n\n Per Tier A E3 rule + data/prediction_clusters.yaml 5 clusters identified in\n audit §7. Catches \"Independence: this prediction is independent.\" class of\n empty assertion that doesn't engage the cluster framework.\n \"\"\"\n m = _FORMAL_PREDICTIONS_SECTION_RE.search(html)\n if not m:\n return None\n section = m.group(1)\n\n ind_re = re.compile(\n r'Independenc(?:e|ia)\\s*:\\s*([^.<]{5,400}\\.)',\n re.IGNORECASE,\n )\n inds = ind_re.findall(section)\n if not inds:\n return None\n\n issues = []\n for i, ind_text in enumerate(inds, 1):\n has_cluster_ref = bool(re.search(\n r'\\bcluster[-_]?\\d|cluster[-_]?id|cluster\\s+\\w+|cluster-\\d-[a-z-]+',\n ind_text,\n re.IGNORECASE,\n ))\n has_structural_ind = bool(re.search(\n r'structural(ly)?\\s+independ|independ\\w*\\s+from|no\\s+correlation|'\n r'sin\\s+correlación|independencia\\s+estructural|distinct\\s+driver|'\n r'unrelated\\s+upstream',\n ind_text,\n re.IGNORECASE,\n ))\n if not (has_cluster_ref or has_structural_ind):\n issues.append(i)\n\n if issues:\n return (\n f'E3 independence-cluster-ref: Independence sentence(s) at position(s) '\n f'{issues} (1-indexed within Formal Predictions section) lack explicit '\n f'cluster_id reference OR structural-independence assertion. Per Tier A '\n f'E3 + data/prediction_clusters.yaml taxonomy, Independence sentences '\n f'should engage the cluster framework, not state generic independence.'\n )\n return None\n\n\ndef check_prediction_e5_ternary_sum_100(html: str) -> str | None:\n \"\"\"E5-ternary-sum (posterior-predictive): TERNARY-SCENARIO label triggers\n expectation that 3 scenarios with explicit probabilities summing to 100% (±2%\n tolerance for rounding) appear in the prediction block.\n\n Per Tier A E5 rule. Catches \"TERNARY-SCENARIO: 40% / 35% / 20%\" (sums to 95%)\n or \"TERNARY-SCENARIO: 50% / 30%\" (only 2 probabilities present) emission errors.\n \"\"\"\n m = _FORMAL_PREDICTIONS_SECTION_RE.search(html)\n if not m:\n return None\n section = m.group(1)\n\n ternary_re = re.compile(r'TERNARY[-\\s]SCENARIO', re.IGNORECASE)\n ternary_matches = list(ternary_re.finditer(section))\n if not ternary_matches:\n return None\n\n issues = []\n for i, t_match in enumerate(ternary_matches, 1):\n # Window: 2000 chars after the label (covers most prediction block sizes)\n window = section[t_match.end():t_match.end() + 2000]\n # Find probabilities; constrain to 1-99 to avoid matching footnote numbers\n probs = re.findall(r'\\b(\\d{1,2})\\s*%', window)\n if len(probs) < 3:\n issues.append(f'ternary #{i}: only {len(probs)} probability tokens found within 2000-char window')\n else:\n three = [int(p) for p in probs[:3]]\n s = sum(three)\n if not (98 <= s <= 102):\n issues.append(f'ternary #{i}: probabilities {three}% sum to {s}% (expected 100%±2)')\n\n if issues:\n return (\n f'E5 ternary-sum-100: TERNARY-SCENARIO emission(s) violate sum-to-100 '\n f'expectation: {\"; \".join(issues)}. Per Tier A E5 rule, scenario-decomposed '\n f'predictions must have ≥2 mutually-exclusive scenarios with probabilities '\n f'summing to 100% (±2% rounding tolerance).'\n )\n return None\n\n\ndef check_prediction_e9_pos_threshold_numeric(html: str) -> str | None:\n \"\"\"E9-POS-numeric (posterior-predictive): POS-THRESHOLD / POS-EVENT-DATE labels\n trigger expectation that a numeric threshold (X% / $X / N units / over Y) appears\n in proximity to the label, not narrative magnitude (\"substantial\" / \"large\").\n\n Per Tier A E9 rule. Catches \"POS-THRESHOLD: substantial defence spending increase\"\n (no number) vs the expected \"POS-THRESHOLD: ≥$400B annual defence procurement\".\n \"\"\"\n m = _FORMAL_PREDICTIONS_SECTION_RE.search(html)\n if not m:\n return None\n section = m.group(1)\n\n pos_re = re.compile(\n r'POS[-\\s](?:THRESHOLD|EVENT[-\\s]DATE|EVENT)',\n re.IGNORECASE,\n )\n pos_matches = list(pos_re.finditer(section))\n if not pos_matches:\n return None\n\n numeric_threshold_re = re.compile(\n r'(?:≥|>=|>|at\\s+least|exceeds?|over|greater\\s+than|más\\s+de|al\\s+menos)\\s*'\n r'[\\$€£]?\\s*[\\d,]+(?:\\.\\d+)?\\s*(?:[%]|billion|million|trillion|bn|mn|tn|'\n r'\\b[a-zA-Z]{1,30}\\b)?',\n re.IGNORECASE,\n )\n issues = []\n for i, p_match in enumerate(pos_matches, 1):\n # Window: 1000 chars after label\n window = section[p_match.end():p_match.end() + 1000]\n if not numeric_threshold_re.search(window):\n label = section[p_match.start():p_match.end()].strip()\n issues.append(f'POS label #{i} (\"{label}\")')\n\n if issues:\n return (\n f'E9 POS-numeric: POS-event prediction(s) lack explicit numeric threshold '\n f'within 1000-char proximity to class label: {\"; \".join(issues)}. Per Tier A '\n f'E9 rule, POS-event predictions must specify quantitative threshold (≥ N / '\n f'over $X / exceeds N% / at least Y units), not narrative magnitude.'\n )\n return None\n\n\n# ── WARNINGS ─────────────────────────────────────────────────────────────────\n\ndef check_translation_body_parity(html: str) -> str | None:\n \"\"\"ES word count should be within reasonable ratio of EN (typically 1.0-1.5).\"\"\"\n en_blocks = re.findall(\n r']*class=\"[^\"]*\\blang-en\\b[^\"]*\"[^>]*>(.*?)
',\n html, re.DOTALL,\n )\n es_blocks = re.findall(\n r']*class=\"[^\"]*\\blang-es\\b[^\"]*\"[^>]*>(.*?)
',\n html, re.DOTALL,\n )\n en_text = ' '.join(re.sub(r'<[^>]+>', ' ', b) for b in en_blocks)\n es_text = ' '.join(re.sub(r'<[^>]+>', ' ', b) for b in es_blocks)\n en_w = len(en_text.split())\n es_w = len(es_text.split())\n if en_w == 0:\n return None\n ratio = es_w / en_w\n if ratio < 0.85:\n return (f'ES body shorter than EN: EN={en_w}w ES={es_w}w '\n f'(ratio={ratio:.2f}, expected 0.95-1.5)')\n if ratio > 1.7:\n return (f'ES body unexpectedly long: EN={en_w}w ES={es_w}w '\n f'(ratio={ratio:.2f}, expected 0.95-1.5)')\n return None\n\n\ndef check_meta_tags_present(html: str) -> str | None:\n \"\"\"OG / canonical / description must be in .\"\"\"\n missing = []\n if 'property=\"og:title\"' not in html:\n missing.append('og:title')\n if 'property=\"og:description\"' not in html:\n missing.append('og:description')\n if 'rel=\"canonical\"' not in html:\n missing.append('canonical')\n if not re.search(r' str | None:\n \"\"\"Local /briefs/SD_*.html links must point to existing files.\"\"\"\n missing = []\n seen = set()\n for m in re.finditer(r'href=[\"\\'](/briefs/SD_\\d+_\\d+_[\\w]+\\.html)[\"\\']',\n html):\n target = m.group(1)\n if target in seen:\n continue\n seen.add(target)\n local = os.path.join(briefs_dir, os.path.basename(target))\n if not os.path.exists(local):\n missing.append(target)\n return f'broken local links: {\", \".join(missing[:3])}' if missing else None\n\n\ndef check_minimum_word_count(html: str) -> str | None:\n \"\"\"Brief should be substantive — flag suspiciously short content.\"\"\"\n visible = re.sub(r'|', '',\n html, flags=re.DOTALL)\n text = re.sub(r'<[^>]+>', ' ', visible)\n words = len(text.split())\n if words < 1500:\n return f'low word count: {words}w (typical brief 1500+)'\n return None\n\n\n# Heuristic: significant numerical claims that should carry a citation.\n# The pattern catches: percentages above noise floor (>5%), dollar/euro\n# amounts in millions/billions, large round numbers, year-prefixed claims.\n# A \"citation marker\" is anything within 200 chars that looks like a\n# source attribution: bracketed source label, a hyperlink, a parenthetical\n# author/year, or a footnote anchor.\n\nNUMERICAL_CLAIM_RE = re.compile(\n r'(?:'\n # Percentages ≥5% (matches the prompt rule threshold; excludes 1-4%\n # which the editorial standard does not require to be sourced).\n # 5-9% or 10-99% or 100-999%.\n r'\\b(?:[5-9]|\\d{2,3})(?:[.,]\\d+)?\\s*%(?!s)'\n # Currency amounts ≥1 million (millions/billions/trillions in any form).\n r'|[€$£]\\s*\\d+(?:[.,]\\d+)?\\s*(?:billion|million|trillion|bn|mn|tn|millones|miles de millones)\\b'\n r'|\\b\\d+(?:[.,]\\d+)?\\s*(?:billion|million|trillion)\\s+(?:dollars|euros|USD|EUR)\\b'\n # Year ranges (2014-2025 etc.) — useful for evaluating event windows.\n r'|\\b(?:1[89]\\d{2}|20[0-4]\\d)\\b(?=\\s*[-–—]\\s*(?:1[89]\\d{2}|20[0-4]\\d))'\n r')',\n re.IGNORECASE,\n)\n\n# Source-organization names that count as \"cited in prose\" when they\n# appear within the proximity window. Editorial style favours\n# conversational citation (\"the IMF's April 2026 WEO identifies...\")\n# over bracket-formal (\"[IMF, WEO 2026]\"). The runner must accept both.\nSOURCE_ORG_NAMES = (\n r'IMF|FMI|NATO|OTAN|OECD|OCDE|WEF|World\\s+Economic\\s+Forum|'\n r'IEA|EIA|BIS|ECB|BCE|Federal\\s+Reserve|Fed|World\\s+Bank|Banco\\s+Mundial|'\n r'Eurostat|UN|United\\s+Nations|ONU|EU|European\\s+(?:Commission|Council|Parliament|Union)|'\n r'CE|Comisión\\s+Europea|Reuters|Bloomberg|Financial\\s+Times|FT|'\n r'New\\s+York\\s+Times|NYT|Washington\\s+Post|Wall\\s+Street\\s+Journal|WSJ|'\n r'Economist|Foreign\\s+Affairs|Foreign\\s+Policy|Politico|Le\\s+Monde|'\n r'El\\s+País|S&P|Moody\\'?s|Fitch|REE|CNMC|ENTSO-E|TSMC|ASML|'\n r'Tesla|Microsoft|Google|Apple|Meta|OpenAI|Anthropic|xAI|COSCO|'\n r'Pew|Eurasia\\s+Group|RAND|Brookings|Atlantic\\s+Council|CSIS|Chatham\\s+House|'\n r'CWA|Munich\\s+Security\\s+Conference|'\n # Defense + critical-minerals research institutions (added 2026-05-02 retrofit Brief 5)\n r'SIPRI|EDA|Agencia\\s+Europea\\s+de\\s+Defensa|IISS|MERICS|EIB|BEI|USGS|Lynas|EUR-Lex'\n)\n\nCITATION_NEAR_RE = re.compile(\n r'(?:'\n r'\\[[A-Z]{2,}[^\\]]{0,80}\\]' # [WEF Global Risks Report 2026]\n r'|\\([A-Z][a-zA-Z]+\\s+(?:et\\s+al\\.?,?\\s+)?\\d{4}\\)' # (Smith et al. 2024)\n r'|\\bhttps?://[^\\s<>\"]+' # bare URL\n r'|]*href=[\"\\']https?://' # hyperlink\n r'|\\b(?:source|fuente|según|per|via)\\s*:\\s*[A-Z]' # \"Source: NATO...\"\n r'|\\b(?:' + SOURCE_ORG_NAMES + r')\\b' # prose mention of known source org\n r')',\n re.IGNORECASE,\n)\n\n\ndef check_numerical_citation_contract(html: str) -> str | None:\n \"\"\"Heuristic: numerical claims should sit near a citation marker.\n\n Removes ', ' ', html, flags=re.DOTALL)\n cleaned = re.sub(r']*>.*?', ' ', cleaned, flags=re.DOTALL)\n # Remove elements that exist for UI summary, not editorial claim:\n # KPI cards (.kpi, .kpi-val, .kpi-label), score chips (.score),\n # and badges (.badge, .pill).\n cleaned = re.sub(\n r'<(?:span|div|td)[^>]*class=\"[^\"]*\\b(?:kpi|kpi-val|kpi-label|score|badge|pill|tag|sh|chart-title)\\b[^\"]*\"[^>]*>.*?',\n ' ', cleaned, flags=re.DOTALL,\n )\n text = re.sub(r'<[^>]+>', ' ', cleaned)\n text = re.sub(r'&[a-z]+;', ' ', text)\n text = re.sub(r'\\s+', ' ', text)\n\n uncited: list[str] = []\n for m in NUMERICAL_CLAIM_RE.finditer(text):\n if len(uncited) >= 12:\n break\n window_start = max(0, m.start() - 250)\n window_end = min(len(text), m.end() + 250)\n window = text[window_start:window_end]\n if not CITATION_NEAR_RE.search(window):\n ctx = text[max(0, m.start() - 50):min(len(text), m.end() + 50)]\n uncited.append(f'\"…{ctx.strip()}…\"')\n\n # Threshold: 8 strikes ≈ enough to suspect systemic uncited claims\n if len(uncited) >= 8:\n return (f'{len(uncited)}+ numerical claims without nearby citation; '\n f'sample: {\" | \".join(uncited[:3])}')\n return None\n\n\nURL_RE = re.compile(r'href=[\"\\'](https?://[^\"\\'<>]+)[\"\\']')\n\n\n# ── L5: critical-fact registry ───────────────────────────────────────────────\n# Loaded lazily on first call. Set FACTS_REGISTRY_PATH env var to override.\n\n_FACTS_CACHE: dict | None = None\n\n\ndef _load_facts_registry() -> dict:\n \"\"\"Load data/facts.yaml once. Returns {} if missing or yaml not installed.\"\"\"\n global _FACTS_CACHE\n if _FACTS_CACHE is not None:\n return _FACTS_CACHE\n try:\n import yaml\n except ImportError:\n _FACTS_CACHE = {}\n return _FACTS_CACHE\n path = os.environ.get(\n 'FACTS_REGISTRY_PATH',\n os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'data', 'facts.yaml'),\n )\n try:\n with open(path, encoding='utf-8') as f:\n data = yaml.safe_load(f) or {}\n except OSError:\n data = {}\n _FACTS_CACHE = data if isinstance(data, dict) else {}\n return _FACTS_CACHE\n\n\ndef _filter_registry_by_brand(registry: dict, brand: str | None) -> dict:\n \"\"\"Filter facts.yaml-style registry by brand field. Default behavior:\n\n - If brand is None: return all entries (back-compat — pre-FORK-PREP-GAP-1\n callers without brand context get full registry).\n - If brand is a specific value (e.g. 'shadow-dynamics' or 'clave'):\n return entries whose `brand` field matches OR equals 'both'. Entries\n with no `brand` field default to 'shadow-dynamics' (pre-2026-05-09\n schema implicit assumption preserved).\n\n FORK-PREP-GAP-1-DATA-BRAND-AWARE-01 (BACKLOG 2026-05-09): brand-aware\n schema unblocks first Clave brief by ensuring SD-specific facts\n (NATO 5%, IMF 60%, COSCO Hamburg 24.9%, etc.) don't fire on Clave\n Hispanic-context briefs and vice versa.\n \"\"\"\n if brand is None:\n return registry\n filtered = {}\n for fact_id, entry in registry.items():\n if not isinstance(entry, dict):\n continue\n entry_brand = entry.get('brand', 'shadow-dynamics')\n if entry_brand == brand or entry_brand == 'both':\n filtered[fact_id] = entry\n return filtered\n\n\ndef check_critical_facts(html: str, brand: str | None = None) -> str | None:\n \"\"\"Match HTML against the curated registry of recurring fact errors.\n\n Each registry entry has `forbidden: [{pattern, reason}, ...]`. A pattern\n match (case-insensitive) is a warning — not a blocker, since false\n positives are possible for narrow contexts. Use `[fact_id] reason …` in\n the detail so the failure log clusters cleanly for the retrospective.\n\n `brand` (optional) filters registry to entries matching the brief's brand\n (or `both`). When None, all entries apply (back-compat; pre-FORK-PREP-GAP-1\n callers preserved). Derived from brief filename by `_derive_brand_from_filename`.\n \"\"\"\n registry = _filter_registry_by_brand(_load_facts_registry(), brand)\n if not registry:\n return None\n hits: list[str] = []\n for fact_id, entry in registry.items():\n if not isinstance(entry, dict):\n continue\n for rule in entry.get('forbidden') or []:\n pat = rule.get('pattern')\n reason = rule.get('reason', '')\n context_negate = rule.get('context_negate')\n if not pat:\n continue\n try:\n m = re.search(pat, html, flags=re.IGNORECASE)\n if not m:\n continue\n if context_negate:\n win = 400\n s = max(0, m.start() - win)\n e = min(len(html), m.end() + win)\n if re.search(context_negate, html[s:e], flags=re.IGNORECASE):\n continue\n hits.append(f'[{fact_id}] {reason}')\n break # one hit per fact is enough\n except re.error:\n continue\n if hits:\n return f'{len(hits)} fact(s) flagged: {\" || \".join(hits[:3])}'\n return None\n\n\ndef check_magnitude_framing_consistency(html: str) -> str | None:\n \"\"\"Detect anti-canonical magnitude framings against facts.yaml::magnitude_flips.\n\n Catches errors like 'capacity runs 40% short of wartime' when canonical is\n 'capacity at 40% of wartime' — same number, opposite semantic. Per-claim\n Layer 4 (Tavily) verification cannot catch this because both framings verify\n against the same source numerically; the bug is cross-section coherence\n against a canonical magnitude framing recorded in data/facts.yaml.\n\n Schema: facts.yaml entries optionally carry a `magnitude_flips` list of\n `{pattern, reason}` regex rules — same structure as `forbidden` but\n semantically scoped to magnitude-flip errors (vs direct contradictions).\n\n Origin: Brief 6 Critical Minerals pre-promote audit 2026-05-08 caught the\n same fact (`ammunition_capacity_wartime`) framed canonically in 2 places\n and anti-canonically in 2 places within the same document; runner missed\n because the existing `forbidden` patterns target the contradiction\n direction ('fully adequate'), not the magnitude flip ('40% short').\n \"\"\"\n registry = _load_facts_registry()\n if not registry:\n return None\n hits: list[str] = []\n for fact_id, entry in registry.items():\n if not isinstance(entry, dict):\n continue\n for rule in entry.get('magnitude_flips') or []:\n pat = rule.get('pattern')\n reason = rule.get('reason', '')\n if not pat:\n continue\n try:\n if re.search(pat, html, flags=re.IGNORECASE):\n hits.append(f'[{fact_id}] {reason}')\n break # one hit per fact is enough\n except re.error:\n continue\n if hits:\n return f'{len(hits)} magnitude flip(s): {\" || \".join(hits[:3])}'\n return None\n\n\n# ── PATTERN RECOGNITION CITATIONS ────────────────────────────────────────────\n# Tracker: PATTERN-RECOGNITION-SYSTEM-01\n# Spec: EDITORIAL-PATTERN-XREF-01 (Forecaster prompt) + data/patterns.yaml\n# Loaded lazily; respects PATTERNS_REGISTRY_PATH env override.\n\n_PATTERNS_CACHE: set[str] | None = None\n\n\ndef _load_pattern_ids() -> set[str]:\n \"\"\"Return set of registered pattern_ids from data/patterns.yaml.\n Empty set if missing or yaml unavailable (gracefully no-ops the check).\"\"\"\n global _PATTERNS_CACHE\n if _PATTERNS_CACHE is not None:\n return _PATTERNS_CACHE\n try:\n import yaml\n except ImportError:\n _PATTERNS_CACHE = set()\n return _PATTERNS_CACHE\n path = os.environ.get(\n 'PATTERNS_REGISTRY_PATH',\n os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'data', 'patterns.yaml'),\n )\n try:\n with open(path, encoding='utf-8') as f:\n data = yaml.safe_load(f) or {}\n except OSError:\n data = {}\n ids: set[str] = set()\n for entry in (data.get('patterns') or []):\n pid = entry.get('pattern_id') if isinstance(entry, dict) else None\n if pid:\n ids.add(pid)\n _PATTERNS_CACHE = ids\n return _PATTERNS_CACHE\n\n\n_PATTERN_ID_RE = re.compile(r'\\b(PATTERN-[A-Z]+(?:-[A-Z]+)+)\\b')\n_BRIEF_FILE_RE = re.compile(r'\\b((?:SD|CL)_\\d{8}_\\d{4}_[A-Za-z_]+\\.html)\\b')\n\n\ndef check_pattern_citations(html: str, briefs_dir: str | None = None) -> str | None:\n \"\"\"Validate §12 PATTERN RECOGNITION integrity (post EDITORIAL-PATTERN-XREF-01).\n\n - Pattern_ids cited (PATTERN-NOUN-MECHANISM uppercase) must exist in\n data/patterns.yaml.\n - Brief filenames cited (SD_*.html / CL_*.html) must exist on disk.\n - Briefs that use prose-only references (no formal markers) pass trivially.\n\n WARNING level — false positives possible if a §12 reformulates a pattern\n without naming a pattern_id, or references a brief by title rather than\n filename. Editorial review remains the source of truth.\n \"\"\"\n m = re.search(r'(?is)PATTERN\\s+RECOGNITION.*?(?= str | None:\n \"\"\"Optionally HEAD each cited URL to detect 4xx/5xx and DNS failures.\n\n Skipped by default (network=False) so the runner stays offline-clean\n in CI. Enable per-call by passing `--check-urls` to the CLI or\n setting `network=True` in library calls.\n \"\"\"\n if not network:\n return None\n import urllib.request\n import urllib.error\n import socket\n\n urls = list(set(URL_RE.findall(html)))\n failures: list[str] = []\n for url in urls:\n try:\n req = urllib.request.Request(url, method='HEAD',\n headers={'User-Agent': 'sd-quality-check/1.0'})\n with urllib.request.urlopen(req, timeout=timeout) as resp:\n if resp.status >= 400:\n failures.append(f'{resp.status} {url}')\n except urllib.error.HTTPError as e:\n if e.code >= 400:\n failures.append(f'{e.code} {url}')\n except (urllib.error.URLError, socket.timeout, OSError) as e:\n failures.append(f'unreachable {url} ({type(e).__name__})')\n\n if failures:\n return f'{len(failures)} broken citation URL(s): {\"; \".join(failures[:3])}'\n return None\n\n\n# Footnote anchors introduced by EDITORIAL-FOOTNOTE-01.\n# Permissive on attribute order/class/whitespace.\nSUP_FN_RE = re.compile(\n r']*>\\s*]*\\bhref=[\"\\']#fn-(\\d+)[\"\\'][^>]*>',\n re.IGNORECASE,\n)\nLI_FN_ID_RE = re.compile(\n r']*\\bid=[\"\\']fn-(\\d+)[\"\\'][^>]*>',\n re.IGNORECASE,\n)\n\n\ndef check_footnote_integrity(html: str) -> str | None:\n \"\"\"Footnote anchor integrity (EDITORIAL-FOOTNOTE-01 F2).\n\n When a brief uses footnotes, every must resolve\n to a
  • , every
  • must be referenced ≥1 time,\n numbering must be contiguous 1..max, and ids must be unique.\n\n Brief with zero footnote markup → returns None (current corpus state).\n \"\"\"\n sup_refs = [int(m.group(1)) for m in SUP_FN_RE.finditer(html)]\n li_ids = [int(m.group(1)) for m in LI_FN_ID_RE.finditer(html)]\n\n if not sup_refs and not li_ids:\n return None\n\n issues: list[str] = []\n sup_set = set(sup_refs)\n li_set = set(li_ids)\n\n orphan_refs = sorted(sup_set - li_set)\n if orphan_refs:\n issues.append(f' ref(s) without
  • : {orphan_refs[:5]}')\n\n unreferenced = sorted(li_set - sup_set)\n if unreferenced:\n issues.append(f'
  • never referenced: {unreferenced[:5]}')\n\n if li_ids:\n max_n = max(li_ids)\n gaps = sorted(set(range(1, max_n + 1)) - li_set)\n if gaps:\n issues.append(f'numbering not contiguous, missing: {gaps[:5]}')\n\n if len(li_ids) != len(li_set):\n from collections import Counter\n dupes = sorted(n for n, c in Counter(li_ids).items() if c > 1)\n issues.append(f'duplicate
  • : {dupes[:5]}')\n\n return '; '.join(issues) if issues else None\n\n\n_DATE_NUM_WORDS = {\n 'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5,\n 'six': 6, 'seven': 7, 'eight': 8, 'nine': 9, 'ten': 10,\n 'eleven': 11, 'twelve': 12, 'thirteen': 13, 'fourteen': 14,\n 'fifteen': 15, 'sixteen': 16, 'seventeen': 17, 'eighteen': 18,\n 'nineteen': 19, 'twenty': 20, 'thirty': 30, 'forty': 40, 'fifty': 50,\n 'uno': 1, 'una': 1, 'dos': 2, 'tres': 3, 'cuatro': 4, 'cinco': 5,\n 'seis': 6, 'siete': 7, 'ocho': 8, 'nueve': 9, 'diez': 10,\n 'once': 11, 'doce': 12, 'trece': 13, 'catorce': 14, 'quince': 15,\n 'dieciséis': 16, 'diecisiete': 17, 'dieciocho': 18,\n 'diecinueve': 19, 'veinte': 20, 'treinta': 30, 'cuarenta': 40,\n 'cincuenta': 50,\n}\n_DATE_UNIT_DAYS = {\n 'day': 1, 'days': 1, 'día': 1, 'días': 1,\n 'week': 7, 'weeks': 7, 'semana': 7, 'semanas': 7,\n 'month': 30, 'months': 30, 'mes': 30, 'meses': 30,\n 'year': 365, 'years': 365, 'año': 365, 'años': 365,\n}\n_DATE_EN_MONTHS = ['january', 'february', 'march', 'april', 'may', 'june',\n 'july', 'august', 'september', 'october', 'november', 'december']\n_DATE_ES_MONTHS = ['enero', 'febrero', 'marzo', 'abril', 'mayo', 'junio',\n 'julio', 'agosto', 'septiembre', 'octubre', 'noviembre', 'diciembre']\n_DATE_MONTH_INDEX = {\n **{m: i + 1 for i, m in enumerate(_DATE_EN_MONTHS)},\n **{m: i + 1 for i, m in enumerate(_DATE_ES_MONTHS)},\n}\n_DATE_CLAIM_RE = re.compile(\n r'\\b(\\d{1,3}|' + '|'.join(sorted(_DATE_NUM_WORDS, key=len, reverse=True)) + r')\\s+'\n r'(' + '|'.join(sorted(_DATE_UNIT_DAYS, key=len, reverse=True)) + r')\\s+'\n r'(?:after|tras|después\\s+de)\\b',\n re.IGNORECASE,\n)\n# DATE-ARITH Phase 2 — duration-to-end-date pattern (Brief 6 origin 2026-05-08).\n# Catches \"N-unit [noun] running to / expires / expira / hasta DATE\" — explicit\n# duration tied to an explicit end date. The Phase 1 _DATE_CLAIM_RE catches\n# \"N units after [event]\" but skips when window dates span >1.2× claim\n# (treated as unrelated dates). For the running-to construction the dates\n# ARE the claim's anchors; both directions of mismatch matter.\n# Brief 6 case: \"6-month suspension running to 2026-11-10\" + \"October 2025\n# truce\" in window. claimed=180d, actual≈400d, ratio≈2.2.\n_DATE_RUNNING_TO_RE = re.compile(\n r'\\b(\\d{1,3})[\\s–—-]+' # N + (space|hyphen|en-dash|em-dash)\n r'(' + '|'.join(sorted(_DATE_UNIT_DAYS, key=len, reverse=True)) + r')\\b'\n r'(?:\\s+[\\wÀ-ſ-]+){0,4}?\\s+' # 0-4 filler words (suspension, pause, etc.)\n r'(?:running\\s+to|expires?(?:\\s+on)?|que\\s+expira(?:r[áa])?\\s+(?:el\\s+)?|'\n r'expira(?:r[áa])?\\s+(?:el\\s+)?|hasta\\s+(?:el\\s+)?|through\\s+|until\\s+(?:the\\s+)?|'\n r'ending\\s+(?:on\\s+)?|ends?\\s+(?:on\\s+)?|finaliz(?:a|ar[áa])\\s+(?:el\\s+)?)'\n r'\\s*(?:<[^>]+>\\s*)?(?:el\\s+)?' # optional inline HTML or article\n r'(\\d{4}-\\d{2}-\\d{2}|\\d{1,2}-\\d{1,2}-\\d{4})', # ISO or DD-MM-YYYY\n re.IGNORECASE,\n)\n# Month-year reference, e.g. \"October 2025\" / \"octubre de 2025\" / \"octubre 2025\"\n_DATE_MONTH_YEAR_RE = re.compile(\n r'\\b(' + '|'.join(_DATE_MONTH_INDEX.keys()) + r')\\s+(?:de\\s+)?(\\d{4})\\b',\n re.IGNORECASE,\n)\n_DATE_MD_Y_RE = re.compile(\n r'\\b(' + '|'.join(_DATE_EN_MONTHS + _DATE_ES_MONTHS) + r')\\s+(\\d{1,2}),?\\s+(\\d{4})\\b',\n re.IGNORECASE,\n)\n_DATE_DM_Y_RE = re.compile(\n r'\\b(\\d{1,2})\\s+(?:de\\s+)?(' + '|'.join(_DATE_EN_MONTHS + _DATE_ES_MONTHS) + r')\\s+(?:de\\s+)?(\\d{4})\\b',\n re.IGNORECASE,\n)\n\n\n_MDTOHTML_BLOCK_WRAP_RE = re.compile(\n r'

    \\s* str | None:\n \"\"\"Detect block-level HTML tags wrapped in

    by mdToHTML.\n\n The Forecaster's mdToHTML JS function in Format Output treats\n blank-line-separated tokens as paragraphs. When the model emits raw\n block-level HTML (

    ,
      ,