gather-engine 1.5.0__py3-none-any.whl

gather/__init__.py

@@ -0,0 +1,31 @@
+"""Gather: an accountable research-intake organ.
+
+The one place network access, third-party tools, and credentials are allowed to live,
+isolated behind source adapters, so the rest of the constellation stays clean. Every
+ingested item carries a provenance receipt, and a gather run emits a witnessed digest
+that index, refine, and the crucible consume. A peer organ: it composes through the
+seam, it does not absorb or get absorbed.
+
+The stable surface is re-exported here (``from gather import make_item, Corpus, gather_run``);
+adapters and the network/credentials edges are imported from their submodules when needed.
+"""
+
+from gather.derive import NullSynthesizer, Synthesizer, derive, synthesize_item
+from gather.digest import Digest, digest, digest_of_receipts, verify_digest
+from gather.item import Item, Provenance, content_hash, make_item
+from gather.provenance import NullProvenanceProvider, ProvenanceProvider
+from gather.recall import Query, recall, recall_audited
+from gather.run import RunRecord, gather_run, verify_record
+from gather.scope import filter_scope, in_scope
+from gather.source import Catalog, Source
+from gather.store import Corpus
+
+__version__ = "1.5.0"
+
+__all__ = [
+    "Catalog", "Corpus", "Digest", "Item", "NullProvenanceProvider", "NullSynthesizer",
+    "Provenance", "ProvenanceProvider", "Query", "RunRecord", "Source", "Synthesizer",
+    "content_hash", "derive", "digest", "digest_of_receipts", "filter_scope", "gather_run",
+    "in_scope", "make_item", "recall", "recall_audited", "synthesize_item", "verify_digest",
+    "verify_record", "__version__",
+]

gather/api.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import json
+import time
+from typing import Any
+
+from gather.credentials import require_secret
+from gather.item import Item, make_item
+from gather.net import decode_body, http_get
+
+
+def _records(payload: Any, items_key: str | None) -> list[dict]:
+    """The records in an API payload: the array at ``items_key``, or the top-level list, or the
+    single object wrapped in a list."""
+    if items_key is not None:
+        data = payload.get(items_key, []) if isinstance(payload, dict) else []
+    else:
+        data = payload
+    if isinstance(data, dict):
+        return [data]
+    if isinstance(data, list):
+        return [r for r in data if isinstance(r, dict)]
+    return []
+
+
+def parse_api(
+    payload: str | bytes,
+    url: str,
+    *,
+    fetched_at: float,
+    items_key: str | None = None,
+    id_key: str = "id",
+    title_key: str = "title",
+    text_key: str | None = None,
+    method: str = "api-get",
+) -> list[Item]:
+    """Turn a JSON API response into one Item per record. Pure: no network.
+
+    Each record becomes an Item whose text is the record's ``text_key`` field if given, else the
+    record serialized as canonical JSON (so the receipt fingerprints the exact record). A non-string
+    ``text_key`` field is itself serialized as canonical JSON, never a Python repr, so the receipt
+    always fingerprints JSON. The id and title are read from ``id_key`` / ``title_key`` when present;
+    id uniqueness across records is the API's responsibility. ``ref`` is the request URL. No
+    credential is ever placed in an Item or its receipt. Raises ValueError on malformed JSON.
+    """
+    try:
+        data = json.loads(payload)
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"not valid API JSON: {exc}") from exc
+    items: list[Item] = []
+    for i, rec in enumerate(_records(data, items_key)):
+        if text_key:
+            field = rec.get(text_key, "")
+            text = field if isinstance(field, str) else json.dumps(field, sort_keys=True, ensure_ascii=False)
+        else:
+            text = json.dumps(rec, sort_keys=True, ensure_ascii=False)
+        rid = str(rec.get(id_key, i))
+        items.append(
+            make_item(
+                kind="record", id=rid, title=str(rec.get(title_key, "")), text=text,
+                source="api", ref=url, method=method, fetched_at=fetched_at,
+            )
+        )
+    return items
+
+
+class ApiSource:
+    """An authenticated JSON-API adapter: the worked example of the credentials-isolation pattern.
+
+    The impure edge that needs a secret. The token is read from the environment by name through
+    require_secret (never from source, a config, or the URL), sent in an Authorization header (so
+    it never reaches the URL, the receipt, or the disk), and the records come back through the pure
+    parse_api. Point another authenticated source at this shape: env in, header out, secret never
+    witnessed. fetch(target) takes the full request URL; needs the named env var set and network.
+    """
+
+    name = "api"
+
+    def __init__(
+        self,
+        *,
+        clock=time.time,
+        auth_env: str = "GATHER_API_TOKEN",
+        auth_scheme: str = "Bearer",
+        items_key: str | None = None,
+        id_key: str = "id",
+        title_key: str = "title",
+        text_key: str | None = None,
+        timeout: float = 20.0,
+    ) -> None:
+        self._clock = clock
+        self._auth_env = auth_env
+        self._auth_scheme = auth_scheme
+        self._items_key = items_key
+        self._id_key = id_key
+        self._title_key = title_key
+        self._text_key = text_key
+        self._timeout = timeout
+
+    def fetch(self, target: str) -> list[Item]:
+        token = require_secret(self._auth_env)
+        if token in target:
+            # the URL is witnessed in the receipt's ref; the secret must travel in the header only
+            raise ValueError("the credential must not appear in the URL (ref is witnessed); it is sent as a header")
+        body, ctype = http_get(
+            target, timeout=self._timeout,
+            headers={"Authorization": f"{self._auth_scheme} {token}"},
+        )
+        return parse_api(
+            decode_body(body, ctype), target, fetched_at=float(self._clock()),
+            items_key=self._items_key, id_key=self._id_key,
+            title_key=self._title_key, text_key=self._text_key,
+        )

gather/arxiv.py

@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+import re
+import time
+import urllib.parse
+import xml.etree.ElementTree as ET
+
+from gather.item import Item, make_item
+from gather.net import http_get
+
+ARXIV_API = "https://export.arxiv.org/api/query"
+_ARXIV_ID = re.compile(r"^\d{4}\.\d{4,5}(v\d+)?$|^[a-z-]+(\.[A-Za-z-]+)?/\d{7}(v\d+)?$", re.IGNORECASE)
+
+
+def _local(tag: str) -> str:
+    return tag.rsplit("}", 1)[-1]
+
+
+def _arxiv_id(id_url: str) -> str:
+    """The bare arXiv id from an abs URL: http://arxiv.org/abs/2301.12345v2 -> 2301.12345v2."""
+    if "/abs/" in id_url:
+        return id_url.split("/abs/", 1)[1].rstrip("/")
+    return id_url.rsplit("/", 1)[-1]
+
+
+def is_arxiv_id(target: str) -> bool:
+    """True if ``target`` is a bare arXiv id (new-style ``2301.12345`` or old ``hep-th/9901001``)
+    rather than a free-text query. The single classifier deciding id-fetch vs search."""
+    return bool(_ARXIV_ID.match(target.strip()))
+
+
+def arxiv_query_url(target: str, *, max_results: int = 10) -> str:
+    """Build the arXiv API URL for a target. Pure and deterministic.
+
+    A bare arXiv id (``2301.12345``, optionally versioned, or an old ``cs.AI/0601001``) is
+    fetched by id; anything else is treated as a free-text search over all fields, returning
+    up to ``max_results`` by relevance. Every value is urlencoded, so a query cannot break out
+    of the query string.
+    """
+    target = target.strip()
+    if is_arxiv_id(target):
+        params = {"id_list": target, "max_results": "1"}
+    else:
+        params = {
+            "search_query": f"all:{target}", "start": "0",
+            "max_results": str(max_results), "sortBy": "relevance",
+        }
+    return f"{ARXIV_API}?{urllib.parse.urlencode(params)}"
+
+
+def parse_arxiv(xml: str | bytes, *, fetched_at: float, method: str = "arxiv-api") -> list[Item]:
+    """Parse an arXiv API Atom response into one paper Item per entry. Pure: no network.
+
+    Each Item's text is the ABSTRACT, not the full paper (the API returns abstracts); the PDF
+    link is recorded in ``meta`` for the separate full-text adapter, so an abstract is never
+    mistaken for the paper. Authors, categories (which include the primary), primary category,
+    publication date, and DOI are carried in ``meta``. An entry without an id is skipped (real
+    arXiv always sends one). Raises ValueError on malformed XML.
+    """
+    try:
+        root = ET.fromstring(xml)
+    except ET.ParseError as exc:
+        raise ValueError(f"not valid arXiv API XML: {exc}") from exc
+
+    items: list[Item] = []
+    for entry in (e for e in root if _local(e.tag) == "entry"):
+        id_url = title = abstract = published = primary = doi = pdf = ""
+        authors: list[str] = []
+        cats: list[str] = []
+        for ch in entry:
+            n = _local(ch.tag)
+            if n == "id":
+                id_url = (ch.text or "").strip()
+            elif n == "title":
+                title = " ".join((ch.text or "").split())
+            elif n == "summary":
+                abstract = " ".join("".join(ch.itertext()).split())
+            elif n == "published":
+                published = (ch.text or "").strip()
+            elif n == "author":
+                authors.extend(s.text.strip() for s in ch if _local(s.tag) == "name" and s.text)
+            elif n == "primary_category":
+                primary = ch.get("term") or primary
+            elif n == "category":
+                term = ch.get("term")
+                if term:
+                    cats.append(term)
+            elif n == "link" and (ch.get("title") == "pdf" or ch.get("type") == "application/pdf"):
+                pdf = ch.get("href") or pdf
+            elif n == "doi":
+                doi = (ch.text or "").strip()
+        if not id_url:
+            continue  # an entry with no id has no identity; real arXiv always sends one
+        meta: dict[str, object] = {}
+        for key, val in (("authors", authors), ("published", published), ("primary_category", primary),
+                         ("categories", cats), ("pdf", pdf), ("doi", doi)):
+            if val:
+                meta[key] = val
+        items.append(
+            make_item(
+                kind="paper", id=_arxiv_id(id_url), title=title, text=abstract,
+                source="arxiv", ref=id_url, method=method,
+                fetched_at=fetched_at, meta=meta,
+            )
+        )
+    return items
+
+
+class ArxivSource:
+    """arXiv paper intake via the public arXiv API. The isolated impure edge; parsing is pure.
+
+    fetch(target) takes an arXiv id or a free-text query and returns paper Items carrying the
+    abstract and metadata. Needs network. The full text behind the PDF link is a separate
+    adapter (see gather.pdf); this returns abstracts, and the receipt's method says so.
+    """
+
+    name = "arxiv"
+
+    def __init__(self, *, clock=time.time, timeout: float = 20.0, max_results: int = 10) -> None:
+        self._clock = clock
+        self._timeout = timeout
+        self._max_results = max_results
+
+    def fetch(self, target: str) -> list[Item]:
+        by_id = is_arxiv_id(target)
+        url = arxiv_query_url(target, max_results=self._max_results)
+        body, _ = http_get(url, timeout=self._timeout)
+        # the receipt records HOW the paper was found: a direct id lookup, or a relevance search
+        method = "arxiv-api-id" if by_id else "arxiv-api-search"
+        return parse_arxiv(body, fetched_at=float(self._clock()), method=method)

gather/browser.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import subprocess
+import time
+
+from gather.item import Item, make_item
+from gather.net import validate_public_http_url
+from gather.web import html_to_title_text
+
+
+def parse_browser(html: str, url: str, *, fetched_at: float, method: str = "browser-extract") -> Item:
+    """Turn rendered DOM HTML into one webpage Item. Pure: no subprocess. Reuses the web text
+    extractor, but the method is ``browser-extract`` (JavaScript WAS run), not ``http-get``, so a
+    JS-rendered page is honestly distinguished from a raw fetch."""
+    title, text = html_to_title_text(html)
+    return make_item(
+        kind="webpage", id=url, title=title or url, text=text,
+        source="browser", ref=url, method=method, fetched_at=fetched_at,
+    )
+
+
+class BrowserSource:
+    """JavaScript-walled page intake via a headless browser. The isolated external-tool edge for
+    pages the static `web` adapter can only see the shell of.
+
+    Shells out to a headless Chromium-family browser (an external tool, not a Python dependency)
+    to dump the rendered DOM, then extracts text with the pure parser. The receipt's
+    ``browser-extract`` method records that JavaScript was executed to produce this text.
+
+    Two safety boundaries an operator must understand, because this edge is more powerful and more
+    exposed than the http edge:
+
+    - The scheme + private-host guard is applied only to the INITIAL navigation. Once the page
+      loads, a real browser follows its own redirects and loads sub-resources (fetch/XHR, iframes,
+      images) with NO host filtering, so a hostile or open-redirecting page can still reach an
+      internal address (an SSRF the receipt would then attest as gathered). The DNS-rebinding
+      window is wider here than for the http edge. Do not point this at untrusted URLs in an
+      environment with reachable internal services.
+    - The Chromium sandbox is left ON by default. ``no_sandbox=True`` disables it (sometimes needed
+      to run as root in a container) and is a real hardening downgrade while executing untrusted
+      JavaScript; use it only when you must, and prefer running as a non-root user instead.
+
+    fetch() needs the browser on PATH and network.
+    """
+
+    name = "browser"
+
+    def __init__(self, *, clock=time.time, browser: str = "chromium", timeout: float = 60.0,
+                 virtual_time_ms: int = 8000, no_sandbox: bool = False) -> None:
+        self._clock = clock
+        self._browser = browser
+        self._timeout = timeout
+        self._virtual_time_ms = virtual_time_ms
+        self._no_sandbox = no_sandbox
+
+    def fetch(self, target: str) -> list[Item]:
+        url = validate_public_http_url(target)  # guards the INITIAL navigation only (see class doc)
+        cmd = [self._browser, "--headless=new", "--disable-gpu"]
+        if self._no_sandbox:
+            cmd.append("--no-sandbox")
+        cmd += [f"--virtual-time-budget={int(self._virtual_time_ms)}", "--dump-dom", url]
+        proc = subprocess.run(cmd, capture_output=True, timeout=self._timeout)
+        if proc.returncode != 0:
+            raise RuntimeError(f"browser failed: {proc.stderr.decode('utf-8', 'replace').strip()[:200]}")
+        html = proc.stdout.decode("utf-8", "replace")
+        return [parse_browser(html, url, fetched_at=float(self._clock()))]

gather/cli.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+import argparse
+import sys
+
+from gather import __version__
+from gather.commands import (
+    cmd_api,
+    cmd_arxiv,
+    cmd_browser,
+    cmd_docs,
+    cmd_feed,
+    cmd_ocr,
+    cmd_parse,
+    cmd_pdf,
+    cmd_run,
+    cmd_transcribe,
+    cmd_video,
+    cmd_web,
+)
+from gather.corpus_cmd import cmd_corpus
+
+
+def _add_common(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--scope", default=None, help="comma-separated scope terms; keep items mentioning any")
+    p.add_argument("--json", action="store_true", help="emit the catalog and digest as JSON")
+    p.add_argument("--store", default=None, metavar="DIR", help="persist gathered items into a corpus at DIR")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="gather", description="Gather: accountable research intake.")
+    parser.add_argument("--version", action="version", version=f"gather {__version__}")
+    sub = parser.add_subparsers(dest="command")
+
+    parse = sub.add_parser("parse", help="parse a saved yt-dlp info.json (+ optional .vtt), offline, no network")
+    parse.add_argument("info", help="path to a yt-dlp info.json")
+    parse.add_argument("--vtt", default=None, help="path to a .vtt captions file")
+    parse.add_argument("--auto-captions", action="store_true",
+                       help="captions are machine-generated: collapse rolling-window growth, stamp auto-caption")
+    _add_common(parse)
+    parse.set_defaults(func=cmd_parse)
+
+    video = sub.add_parser("video", help="fetch a video via yt-dlp (needs yt-dlp on PATH and network)")
+    video.add_argument("url")
+    video.add_argument("--comments", action="store_true", help="also gather comments")
+    _add_common(video)
+    video.set_defaults(func=cmd_video)
+
+    web = sub.add_parser("web", help="fetch a static web page via http(s) and extract readable text")
+    web.add_argument("url")
+    _add_common(web)
+    web.set_defaults(func=cmd_web)
+
+    feed = sub.add_parser("feed", help="fetch an RSS or Atom feed via http(s)")
+    feed.add_argument("url")
+    _add_common(feed)
+    feed.set_defaults(func=cmd_feed)
+
+    docs = sub.add_parser("docs", help="read a local text file or a directory of them, offline")
+    docs.add_argument("path")
+    _add_common(docs)
+    docs.set_defaults(func=cmd_docs)
+
+    arxiv = sub.add_parser("arxiv", help="fetch papers from the arXiv API by id or free-text query")
+    arxiv.add_argument("query", help="an arXiv id (2301.12345) or a search query")
+    arxiv.add_argument("--max-results", type=int, default=10, help="max results for a search query")
+    _add_common(arxiv)
+    arxiv.set_defaults(func=cmd_arxiv)
+
+    pdf = sub.add_parser("pdf", help="extract text from a local PDF (needs pdftotext on PATH)")
+    pdf.add_argument("path")
+    _add_common(pdf)
+    pdf.set_defaults(func=cmd_pdf)
+
+    api = sub.add_parser("api", help="fetch a JSON API with a bearer token from the environment")
+    api.add_argument("url")
+    api.add_argument("--auth-env", default="GATHER_API_TOKEN", help="env var holding the bearer token")
+    api.add_argument("--items-key", default=None, help="key of the records array in the JSON response")
+    api.add_argument("--text-key", default=None, help="record field to use as item text (else the whole record)")
+    api.add_argument("--id-key", default="id", help="record field to use as item id")
+    api.add_argument("--title-key", default="title", help="record field to use as item title")
+    _add_common(api)
+    api.set_defaults(func=cmd_api)
+
+    browser = sub.add_parser("browser", help="fetch a JS-rendered page via a headless browser (needs chromium on PATH)")
+    browser.add_argument("url")
+    browser.add_argument("--browser", default="chromium", help="headless browser binary")
+    browser.add_argument("--no-sandbox", action="store_true",
+                         help="disable the Chromium sandbox (only if running as root in a container; a downgrade)")
+    _add_common(browser)
+    browser.set_defaults(func=cmd_browser)
+
+    ocr = sub.add_parser("ocr", help="recognize text in a local image via tesseract (needs tesseract on PATH)")
+    ocr.add_argument("path")
+    ocr.add_argument("--lang", default="eng", help="tesseract language code")
+    _add_common(ocr)
+    ocr.set_defaults(func=cmd_ocr)
+
+    transcribe = sub.add_parser("transcribe", help="transcribe a local audio file via whisper (needs whisper on PATH)")
+    transcribe.add_argument("path")
+    transcribe.add_argument("--model", default="base", help="whisper model name")
+    _add_common(transcribe)
+    transcribe.set_defaults(func=cmd_transcribe)
+
+    run = sub.add_parser("run", help="run a multi-source gather session from a JSON config")
+    run.add_argument("config",
+                     help="JSON config: {jobs:[{source,target}], scope, store, "
+                          "synthesize | synthesizer:[cmd...], synth_prompt}")
+    run.add_argument("--json", action="store_true", help="emit the witnessed run record as JSON")
+    run.set_defaults(func=cmd_run)
+
+    corpus = sub.add_parser("corpus", help="inspect a stored corpus: list/verify/digest/runs/search/stats/prune")
+    corpus.add_argument("action", choices=["list", "verify", "digest", "runs", "search", "stats", "prune"])
+    corpus.add_argument("dir", help="the corpus directory (created by --store)")
+    corpus.add_argument("--json", action="store_true", help="emit as JSON")
+    corpus.add_argument("--verify", action="store_true", help="with runs: re-check each record's seal")
+    corpus.add_argument("--apply", action="store_true", help="with prune: actually delete orphan objects")
+    corpus.add_argument("--terms", default=None,
+                        help="with search: scope keywords, case-insensitive substrings of title+body (any match)")
+    corpus.add_argument("--source", default=None,
+                        help="with search: keep items from any of these sources (comma-sep, OR within)")
+    corpus.add_argument("--kind", default=None, help="with search: keep items of any of these kinds (comma-sep)")
+    corpus.add_argument("--method", default=None, help="with search: keep items of any of these methods (comma-sep)")
+    corpus.add_argument("--limit", type=int, default=None, help="with search: cap the matches (<=0 means none)")
+    corpus.set_defaults(func=cmd_corpus)
+
+    return parser
+
+
+def main(argv=None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    func = getattr(args, "func", None)
+    if func is None:
+        parser.print_help()
+        return 1
+    return func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())