proof-citations 1.33.0__py3-none-any.whl

proof_citations/__init__.py

@@ -0,0 +1,10 @@
+"""proof-citations — fetch URLs and verify quoted text appears on the page."""
+
+__version__ = "1.33.0"
+
+from proof_citations.verify import (
+    verify_citation,
+    verify_all_citations,
+)
+
+__all__ = ["verify_citation", "verify_all_citations"]

proof_citations/cli.py

@@ -0,0 +1,67 @@
+"""Command-line entry point for proof-citations.
+
+    proof-citations verify --url URL --quote "QUOTE" --fact-id B1
+    proof-citations verify --facts facts.json
+"""
+
+import argparse
+import json
+import sys
+
+from proof_citations import verify_citation, verify_all_citations
+
+
+def _cmd_verify(args) -> int:
+    if args.facts:
+        with open(args.facts) as f:
+            facts = json.load(f)
+        results = verify_all_citations(facts)
+        # verify_all_citations returns a dict (per verify_citations.py).
+        # Emit as-is, and compute exit from statuses.
+        sys.stdout.write(
+            json.dumps(results, indent=2 if args.pretty else None) + "\n"
+        )
+        statuses = [r.get("status") for r in results.values()] \
+            if isinstance(results, dict) else []
+        return 0 if statuses and all(s == "verified" for s in statuses) else 1
+
+    if not args.url or not args.quote:
+        sys.stderr.write("error: --url and --quote are required (or --facts)\n")
+        return 2
+
+    result = verify_citation(args.url, args.quote, args.fact_id)
+    if args.json:
+        sys.stdout.write(
+            json.dumps(result, indent=2 if args.pretty else None, default=str)
+            + "\n"
+        )
+    else:
+        sys.stdout.write(f"{result.get('status')}: {args.url}\n")
+    return 0 if result.get("status") == "verified" else 1
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="proof-citations")
+    sub = parser.add_subparsers(dest="cmd", required=True)
+
+    verify = sub.add_parser("verify", help="Verify a quote appears at a URL.")
+    verify.add_argument("--url")
+    verify.add_argument("--quote")
+    verify.add_argument("--fact-id", default="B1",
+                        help="Identifier used in messages (default: B1).")
+    verify.add_argument("--facts", help="Path to JSON file of facts to verify.")
+    verify.add_argument("--json", action="store_true", help="Emit JSON.")
+    verify.add_argument("--pretty", action="store_true", help="Pretty-print JSON.")
+    verify.set_defaults(func=_cmd_verify)
+
+    return parser
+
+
+def main(argv=None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

proof_citations/data/academic_domains.json

@@ -0,0 +1,97 @@
+{
+    "_comment": "Academic/research TLDs and known scholarly publisher domains. Tier 4 (peer-reviewed/institutional).",
+    "tld_suffixes": [
+        ".edu",
+        ".edu.au",
+        ".edu.br",
+        ".edu.cn",
+        ".edu.hk",
+        ".edu.sg",
+        ".edu.tw",
+        ".edu.mx",
+        ".edu.ar",
+        ".edu.co",
+        ".edu.pe",
+        ".ac.uk",
+        ".ac.jp",
+        ".ac.kr",
+        ".ac.il",
+        ".ac.in",
+        ".ac.nz",
+        ".ac.za",
+        ".ac.id",
+        ".ac.th",
+        ".uni.edu",
+        ".edu.pl",
+        ".edu.tr"
+    ],
+    "known_domains": [
+        "nature.com",
+        "science.org",
+        "sciencemag.org",
+        "thelancet.com",
+        "bmj.com",
+        "nejm.org",
+        "cell.com",
+        "pnas.org",
+        "plos.org",
+        "plosone.org",
+        "plosmedicine.org",
+        "springer.com",
+        "link.springer.com",
+        "wiley.com",
+        "onlinelibrary.wiley.com",
+        "sciencedirect.com",
+        "elsevier.com",
+        "tandfonline.com",
+        "taylorandfrancis.com",
+        "oxfordacademic.com",
+        "academic.oup.com",
+        "cambridge.org",
+        "journals.sagepub.com",
+        "sagepub.com",
+        "jstor.org",
+        "arxiv.org",
+        "biorxiv.org",
+        "medrxiv.org",
+        "ssrn.com",
+        "pubmed.ncbi.nlm.nih.gov",
+        "ncbi.nlm.nih.gov",
+        "scholar.google.com",
+        "semanticscholar.org",
+        "researchgate.net",
+        "doi.org",
+        "crossref.org",
+        "cochranelibrary.com",
+        "annualreviews.org",
+        "frontiersin.org",
+        "mdpi.com",
+        "ieee.org",
+        "ieeexplore.ieee.org",
+        "acm.org",
+        "dl.acm.org",
+        "aps.org",
+        "aaas.org",
+        "royalsocietypublishing.org",
+        "worldscientific.com",
+        "degruyter.com",
+        "karger.com",
+        "biomedcentral.com",
+        "iopscience.iop.org",
+        "aanda.org",
+        "journals.aps.org",
+        "physrev.org",
+        "aip.org",
+        "scitation.org",
+        "projecteuclid.org",
+        "msp.org",
+        "ams.org",
+        "zbmath.org"
+    ],
+    "doi_prefixes": [
+        "https://doi.org/",
+        "http://doi.org/",
+        "https://dx.doi.org/",
+        "http://dx.doi.org/"
+    ]
+}

proof_citations/data/government_tlds.json

@@ -0,0 +1,114 @@
+{
+    "_comment": "Government and intergovernmental TLDs and domain suffixes. Tier 5 (primary official source).",
+    "_comment_un_agencies": "UN subsidiary organs and specialized agencies. un.int excluded — subdomain matching would promote all *.un.int hosts (including member-state mission sites) to tier 5.",
+    "tld_suffixes": [
+        ".gov",
+        ".gov.uk",
+        ".gov.au",
+        ".gov.ca",
+        ".gov.nz",
+        ".gov.za",
+        ".gov.in",
+        ".gov.sg",
+        ".gov.ie",
+        ".gov.il",
+        ".gc.ca",
+        ".gouv.fr",
+        ".gouv.qc.ca",
+        ".gob.mx",
+        ".gob.es",
+        ".gob.ar",
+        ".gob.cl",
+        ".gob.pe",
+        ".gobierno.ar",
+        ".go.jp",
+        ".go.kr",
+        ".go.id",
+        ".go.th",
+        ".gov.br",
+        ".gov.cn",
+        ".gov.tw",
+        ".gov.ph",
+        ".gov.my",
+        ".gov.eg",
+        ".gov.ng",
+        ".gov.gh",
+        ".gov.ke",
+        ".govt.nz",
+        ".bund.de",
+        ".admin.ch",
+        ".overheid.nl",
+        ".regeringen.se",
+        ".mil",
+        ".mil.br"
+    ],
+    "known_domains": [
+        "europa.eu",
+        "un.org",
+        "who.int",
+        "worldbank.org",
+        "imf.org",
+        "oecd.org",
+        "nato.int",
+        "wto.org",
+        "iaea.org",
+        "unicef.org",
+        "unhcr.org",
+        "icj-cij.org",
+        "icc-cpi.int",
+        "ilo.org",
+        "fao.org",
+        "wipo.int",
+        "itu.int",
+        "unep.org",
+        "undp.org",
+        "unfccc.int",
+        "ipcc.ch",
+        "data.worldbank.org",
+        "data.un.org",
+        "stats.oecd.org",
+        "federalreserve.gov",
+        "bls.gov",
+        "census.gov",
+        "cdc.gov",
+        "nih.gov",
+        "nasa.gov",
+        "noaa.gov",
+        "epa.gov",
+        "fda.gov",
+        "sec.gov",
+        "treasury.gov",
+        "state.gov",
+        "justice.gov",
+        "uscourts.gov",
+        "congress.gov",
+        "whitehouse.gov",
+        "ons.gov.uk",
+        "bankofengland.co.uk",
+        "abs.gov.au",
+        "rba.gov.au",
+        "statcan.gc.ca",
+        "bankofcanada.ca",
+        "destatis.de",
+        "bundesbank.de",
+        "insee.fr",
+        "banque-france.fr",
+        "istat.it",
+        "stats.govt.nz",
+        "ecb.europa.eu",
+        "eurostat.ec.europa.eu",
+        "unrwa.org",
+        "ungeneva.org",
+        "wfp.org",
+        "unesco.org",
+        "unodc.org",
+        "unhabitat.org",
+        "unwomen.org",
+        "unaids.org",
+        "unido.org",
+        "unctad.org",
+        "unops.org",
+        "reliefweb.int",
+        "ohchr.org"
+    ]
+}

proof_citations/data/major_news.json

@@ -0,0 +1,69 @@
+{
+    "_comment": "Major news organizations with established editorial standards. Tier 3 (major news). Based on Wikipedia's Perennial Sources list (generally reliable).",
+    "_comment_newer_outlets": "Established newer outlets with editorial standards, added 2026-04.",
+    "known_domains": [
+        "apnews.com",
+        "reuters.com",
+        "bbc.com",
+        "bbc.co.uk",
+        "nytimes.com",
+        "washingtonpost.com",
+        "wsj.com",
+        "economist.com",
+        "ft.com",
+        "theguardian.com",
+        "aljazeera.com",
+        "npr.org",
+        "pbs.org",
+        "abcnews.go.com",
+        "cbsnews.com",
+        "nbcnews.com",
+        "cnn.com",
+        "usatoday.com",
+        "latimes.com",
+        "chicagotribune.com",
+        "bostonglobe.com",
+        "politico.com",
+        "thehill.com",
+        "bloomberg.com",
+        "cnbc.com",
+        "time.com",
+        "newsweek.com",
+        "theatlantic.com",
+        "newyorker.com",
+        "propublica.org",
+        "theintercept.com",
+        "foreignaffairs.com",
+        "foreignpolicy.com",
+        "deseret.com",
+        "dw.com",
+        "france24.com",
+        "scmp.com",
+        "abc.net.au",
+        "cbc.ca",
+        "rte.ie",
+        "irishtimes.com",
+        "smh.com.au",
+        "nzherald.co.nz",
+        "japantimes.co.jp",
+        "haaretz.com",
+        "jpost.com",
+        "timesofisrael.com",
+        "hindustantimes.com",
+        "thehindu.com",
+        "scientificamerican.com",
+        "newscientist.com",
+        "nationalgeographic.com",
+        "smithsonianmag.com",
+        "popsci.com",
+        "livescience.com",
+        "sciencenews.org",
+        "theconversation.com",
+        "semafor.com",
+        "axios.com",
+        "themarkup.org",
+        "restofworld.org",
+        "defector.com",
+        "theinformation.com"
+    ]
+}

proof_citations/data/reference_domains.json

@@ -0,0 +1,44 @@
+{
+    "_comment": "Established reference sources, encyclopedias, major data aggregators, and known-good government data republishers. Tier 3 (established reference).",
+    "known_domains": [
+        "en.wikipedia.org",
+        "wikipedia.org",
+        "britannica.com",
+        "merriam-webster.com",
+        "dictionary.com",
+        "encyclopedia.com",
+        "plato.stanford.edu",
+        "investopedia.com",
+        "webmd.com",
+        "mayoclinic.org",
+        "clevelandclinic.org",
+        "statista.com",
+        "tradingeconomics.com",
+        "macrotrends.net",
+        "ourworldindata.org",
+        "gapminder.org",
+        "wolframalpha.com",
+        "mathworld.wolfram.com",
+        "khanacademy.org",
+        "coursera.org",
+        "rateinflation.com",
+        "inflationdata.com",
+        "measuringworth.com",
+        "usinflationcalculator.com",
+        "multpl.com",
+        "fred.stlouisfed.org",
+        "minneapolisfed.org",
+        "bea.gov",
+        "numbeo.com",
+        "countryeconomy.com",
+        "officialdata.org",
+        "brainfacts.org",
+        "simplypsychology.org",
+        "physicsworld.com",
+        "sciencebasedmedicine.org",
+        "amnh.org",
+        "burkemuseum.org",
+        "snopes.com",
+        "factcheck.org"
+    ]
+}

proof_citations/data/unreliable_domains.json

@@ -0,0 +1,54 @@
+{
+    "_comment": "Known unreliable, fake news, conspiracy, or junk science domains. Tier 1 (flagged unreliable). Sourced from OpenSources, Iffy.news, and Wikipedia Perennial Sources (deprecated/generally unreliable). This is intentionally a conservative list — only domains with broad consensus on unreliability.",
+    "known_domains": [
+        "infowars.com",
+        "naturalnews.com",
+        "beforeitsnews.com",
+        "globalresearch.ca",
+        "zerohedge.com",
+        "breitbart.com",
+        "dailycaller.com",
+        "thegatewaypundit.com",
+        "oann.com",
+        "newsmax.com",
+        "rt.com",
+        "sputniknews.com",
+        "tass.com",
+        "presstv.ir",
+        "globalresearch.ca",
+        "collective-evolution.com",
+        "davidicke.com",
+        "mercola.com",
+        "greenmedinfo.com",
+        "activistpost.com",
+        "lewrockwell.com",
+        "worldtruth.tv",
+        "yournewswire.com",
+        "neonnettle.com",
+        "newspunch.com",
+        "theepochtimes.com",
+        "dailywire.com",
+        "occupydemocrats.com",
+        "bipartisanreport.com",
+        "palmerreport.com",
+        "addictinginfo.com",
+        "conservativetribune.com",
+        "freedomdaily.com",
+        "worldnewsdailyreport.com",
+        "empirenews.net",
+        "nationalreport.net",
+        "huzlers.com",
+        "theonion.com",
+        "babylonbee.com",
+        "clickhole.com",
+        "borowitz.com",
+        "waterfordwhispersnews.com"
+    ],
+    "satire_domains": [
+        "theonion.com",
+        "babylonbee.com",
+        "clickhole.com",
+        "borowitz.com",
+        "waterfordwhispersnews.com"
+    ]
+}

proof_citations/fetch.py

@@ -0,0 +1,214 @@
+"""
+fetch.py — HTTP fetching with fallback chain for proof-engine.
+
+Handles: live fetch -> snapshot -> Wayback Machine fallback.
+Also handles PDF text extraction and GitHub raw README fallback.
+
+Extracted from verify_citations.py to separate transport from matching logic.
+"""
+
+import os
+import re
+
+try:
+    import requests
+except ImportError:
+    requests = None
+
+
+# ---------------------------------------------------------------------------
+# PDF text extraction
+# ---------------------------------------------------------------------------
+
+def extract_pdf_text(content: bytes) -> str | None:
+    """Extract text from PDF bytes. Tries pdfplumber first, then PyPDF2.
+
+    Returns None if no PDF library is available or if extraction fails.
+    """
+    try:
+        import pdfplumber
+        import io
+        with pdfplumber.open(io.BytesIO(content)) as pdf:
+            return "\n".join(page.extract_text() or "" for page in pdf.pages)
+    except ImportError:
+        pass
+    except Exception:
+        pass
+    try:
+        import PyPDF2
+        import io
+        reader = PyPDF2.PdfReader(io.BytesIO(content))
+        return "\n".join(page.extract_text() or "" for page in reader.pages)
+    except ImportError:
+        pass
+    except Exception:
+        pass
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Wayback Machine fallback
+# ---------------------------------------------------------------------------
+
+def try_wayback(url: str, timeout: int = 15) -> str | None:
+    """Try fetching a URL from the Wayback Machine. Returns page text or None."""
+    if requests is None:
+        return None
+    wayback_url = f"https://web.archive.org/web/{url}"
+    try:
+        resp = requests.get(wayback_url, timeout=timeout,
+                            headers={"User-Agent": "proof-engine/1.0"},
+                            allow_redirects=True)
+        resp.raise_for_status()
+        return resp.text
+    except requests.exceptions.RequestException:
+        return None
+
+
+# ---------------------------------------------------------------------------
+# GitHub raw README fallback
+# ---------------------------------------------------------------------------
+
+_GITHUB_REPO_RE = re.compile(r'^https?://github\.com/([^/]+)/([^/]+)/?$')
+
+
+_README_CANDIDATES = ["README.md", "README.rst", "README.txt", "README", "readme.md"]
+
+
+def try_github_raw(url: str, timeout: int = 15) -> str | None:
+    """Try fetching a GitHub repo's raw README. Returns text or None.
+
+    Only applies to bare repo URLs (github.com/owner/repo). URLs with
+    file paths are not rewritten. GitHub renders repo pages via JavaScript,
+    so requests.get() gets a React shell instead of the README content.
+
+    Tries multiple README filenames (README.md, README.rst, README.txt,
+    README, readme.md) since repos vary in naming convention.
+    """
+    if requests is None:
+        return None
+    m = _GITHUB_REPO_RE.match(url)
+    if not m:
+        return None
+    owner, repo = m.group(1), m.group(2)
+    for readme_name in _README_CANDIDATES:
+        raw_url = f"https://raw.githubusercontent.com/{owner}/{repo}/HEAD/{readme_name}"
+        try:
+            resp = requests.get(raw_url, timeout=timeout,
+                                headers={"User-Agent": "proof-engine/1.0"},
+                                allow_redirects=True)
+            resp.raise_for_status()
+            return resp.text
+        except requests.exceptions.RequestException:
+            continue
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Page fetching with fallback chain
+# ---------------------------------------------------------------------------
+
+def fetch_page(url: str, timeout: int = 15, snapshot: str = None,
+               wayback_fallback: bool = False,
+               skip_live_fetch: bool = False,
+               snapshot_file: str = None) -> tuple[str | None, str, str | None]:
+    """Fetch page text using the standard fallback chain.
+
+    Args:
+        url: The URL to fetch.
+        timeout: Fetch timeout in seconds.
+        snapshot: Pre-fetched page text for offline verification (inline).
+        wayback_fallback: If True, try Wayback Machine as last resort.
+        skip_live_fetch: If True, skip live HTTP fetch (e.g., when requests
+            is unavailable in the calling module).
+        snapshot_file: Path to a local file containing pre-fetched page text.
+            Used for paywalled content that cannot be embedded inline. Inline
+            snapshot takes precedence over snapshot_file.
+
+    Returns:
+        (page_text, fetch_mode, error_message)
+        - page_text: The page text, or None if all methods failed
+        - fetch_mode: "live", "snapshot", "wayback", "github_raw", or "fetch_failed"
+        - error_message: Error description if failed, else None
+    """
+    # --- 1. Try live fetch ---
+    fetch_error_msg = None
+    if requests is not None and not skip_live_fetch:
+        try:
+            resp = requests.get(
+                url,
+                timeout=timeout,
+                headers={"User-Agent": "proof-engine/1.0"},
+                allow_redirects=True,
+            )
+            resp.raise_for_status()
+
+            content_type = resp.headers.get("Content-Type", "")
+            is_pdf = "application/pdf" in content_type or url.lower().endswith(".pdf")
+
+            if is_pdf:
+                pdf_text = extract_pdf_text(resp.content)
+                if pdf_text is None:
+                    fetch_error_msg = "PDF detected but no extraction library available (pip install pdfplumber)"
+                else:
+                    return pdf_text, "live", None
+            else:
+                page_text = resp.text
+                # --- 1.5. GitHub raw README fallback ---
+                # Only try if live fetch returned an empty/JS-shell page (< 500 chars of
+                # visible text after tag stripping) for a bare GitHub repo URL.
+                # This preserves quotes about repo metadata that appears on the rendered
+                # github.com page, falling back to raw README only when the live page
+                # didn't contain useful content.
+                if len(re.sub(r'<[^>]+>', '', page_text).strip()) < 500:
+                    github_text = try_github_raw(url, timeout)
+                    if github_text is not None:
+                        return github_text, "github_raw", None
+                return page_text, "live", None
+
+        except requests.exceptions.Timeout:
+            fetch_error_msg = f"Timeout after {timeout}s on {url}"
+        except requests.exceptions.ConnectionError as e:
+            fetch_error_msg = f"Connection error on {url}: {e}"
+        except requests.exceptions.HTTPError:
+            fetch_error_msg = f"HTTP {resp.status_code} on {url}"
+        except requests.exceptions.RequestException as e:
+            fetch_error_msg = f"{e}"
+    else:
+        fetch_error_msg = "requests package not installed — skipping live fetch"
+
+    # --- 1.5b. GitHub raw README fallback (when live fetch failed entirely) ---
+    if 'github.com' in url:
+        github_text = try_github_raw(url, timeout)
+        if github_text is not None:
+            return github_text, "github_raw", None
+
+    # --- 2. Try snapshot fallback ---
+    if snapshot:
+        return snapshot, "snapshot", None
+
+    # --- 2b. Try snapshot_file fallback ---
+    if snapshot_file:
+        if os.path.isfile(snapshot_file):
+            try:
+                with open(snapshot_file, "r", encoding="utf-8") as f:
+                    return f.read(), "snapshot", None
+            except (OSError, UnicodeDecodeError) as exc:
+                read_err = f"snapshot_file '{snapshot_file}' could not be read: {exc}"
+                fetch_error_msg = f"{fetch_error_msg}; {read_err}" if fetch_error_msg else read_err
+        else:
+            # Record the missing file but continue to next fallback
+            snapshot_file_msg = f"snapshot_file '{snapshot_file}' not found (paywalled content stored locally only)"
+            if fetch_error_msg:
+                fetch_error_msg = f"{fetch_error_msg}; {snapshot_file_msg}"
+            else:
+                fetch_error_msg = snapshot_file_msg
+
+    # --- 3. Try Wayback Machine ---
+    if wayback_fallback:
+        wayback_text = try_wayback(url, timeout)
+        if wayback_text is not None:
+            return wayback_text, "wayback", None
+
+    # --- 4. All methods exhausted ---
+    return None, "fetch_failed", fetch_error_msg