descriptron-core 2.1.2__tar.gz → 2.1.3__tar.gz

{descriptron_core-2.1.2 → descriptron_core-2.1.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: descriptron-core
-Version: 2.1.2
+Version: 2.1.3
 Summary: Evidence-tiered species descriptions, keys and audits from phenomic matrices
 Project-URL: Homepage, https://github.com/alexrvandam/Descriptron
 Project-URL: Repository, https://github.com/alexrvandam/Descriptron

descriptron_core-2.1.3/pyproject.toml

@@ -0,0 +1,82 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "descriptron-core"
+version = "2.1.3"
+description = "Evidence-tiered species descriptions, keys and audits from phenomic matrices"
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "Apache-2.0" }
+authors = [{ name = "Alex R. Van Dam" }]
+keywords = ["taxonomy", "morphometrics", "phenomics", "species description", "COCO"]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "Topic :: Scientific/Engineering :: Bio-Informatics",
+  "Topic :: Scientific/Engineering :: Image Processing",
+  "Intended Audience :: Science/Research",
+  "License :: OSI Approved :: Apache Software License",
+  "Operating System :: OS Independent",
+]
+
+# Derived from the imports of the 62 programs in this package, not from memory:
+# `python -m descriptron_core.cli --list` names them. No torch, no CUDA, no
+# compiler — this installs in about a minute on Linux, macOS and Windows, and
+# with the deposited COCO files it reproduces every analysis in the methods paper.
+dependencies = [
+  "numpy>=1.23",
+  "pandas>=1.5",
+  "scipy>=1.9",
+  "matplotlib>=3.6",
+  "scikit-learn>=1.2",
+  "scikit-image>=0.20",
+  "opencv-python-headless>=4.7",   # headless: the analysis half never opens a window
+  "pillow>=9",
+  "pycocotools>=2.0.6",
+  "seaborn>=0.12",
+  "statsmodels>=0.14",
+  "scikit-posthocs>=0.7",
+  "umap-learn>=0.5",
+  "networkx>=3",
+  "pyefd>=1.6",
+  "biopython>=1.81",
+  "PyYAML>=6",
+  "lxml>=4.9",
+  "openpyxl>=3.1",
+  "python-docx>=1.0",              # treatment and monograph .docx export
+]
+
+[project.optional-dependencies]
+# Only needed to *write* treatments. The audit, key, matrix, novelty scoring and
+# delimitation are all deterministic and need no model at all.
+llm = ["anthropic>=0.40"]
+# literature retrieval (the RAG in BioRAG): BioSysLit treatments from Zenodo and your
+# own PDFs. Keyword retrieval needs only the first two; sentence-embedding retrieval
+# (`index --embeddings`) also needs sentence-transformers + faiss-cpu, which pull in torch.
+rag = ["requests>=2.28", "pdfplumber>=0.10"]
+rag-embeddings = ["requests>=2.28", "pdfplumber>=0.10", "sentence-transformers>=2.2", "faiss-cpu>=1.7"]
+dev = ["pytest>=7", "build>=1", "twine>=5"]
+
+[project.scripts]
+descriptron        = "descriptron_core.cli:main"
+biorag-pipeline    = "descriptron_core.cli:run_pipeline"
+biorag-key         = "descriptron_core.cli:run_key_builder"
+biorag-audit       = "descriptron_core.cli:run_confabulation_checker"
+biorag-novelty     = "descriptron_core.cli:run_novelty_score"
+biorag-calibrate   = "descriptron_core.cli:run_calibrate"
+descriptron-keys   = "descriptron_core.cli:run_keys"
+
+[project.urls]
+Homepage = "https://github.com/alexrvandam/Descriptron"
+Repository = "https://github.com/alexrvandam/Descriptron"
+Issues = "https://github.com/alexrvandam/Descriptron/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/descriptron_core"]
+
+# tools/ and data/ sit inside the package directory, so `packages` already ships
+# them; a force-include here would add every file a second time and the build
+# fails with "a second file is being added to the wheel archive at the same path".
+[tool.hatch.build]
+artifacts = ["src/descriptron_core/tools/**", "src/descriptron_core/data/**"]

{descriptron_core-2.1.2 → descriptron_core-2.1.3}/src/descriptron_core/tools/biorag_llm_backend.py

@@ -82,6 +82,11 @@ class LLMResponse:
 def _log_call(log_path: Optional[str], record: Dict):
     if not log_path:
         return
+    try:                                   # when the call was made, for matching a call to a result
+        from datetime import datetime, timezone
+        record = {"time_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"), **record}
+    except Exception:
+        pass
     with _LOG_LOCK:
         Path(log_path).parent.mkdir(parents=True, exist_ok=True)
         with open(log_path, "a") as f:

descriptron_core-2.1.3/src/descriptron_core/tools/biorag_provenance_v1.py

@@ -0,0 +1,230 @@
+#!/usr/bin/env python3
+"""
+biorag_provenance_v1.py — stamp every output with what produced it
+==================================================================
+
+A number in a manuscript is only checkable if a reader can get from it back to
+the code and the inputs that made it. `manuscript_numbers_v18.tsv` already maps
+each number to the report file that supplied it; this closes the remaining link,
+from the report back to the script, the commit and the exact inputs.
+
+Three lines in a script:
+
+    from biorag_provenance_v1 import stamp, write_json
+    ...
+    write_json(out_dir / "report.json", payload)        # adds "_provenance"
+
+or, for anything that is not JSON (a CSV, a figure, a directory of tables):
+
+    stamp(out_dir, inputs=[args.matrix, args.coco])     # writes provenance.json
+
+What is recorded: the script and its SHA-256, the git commit and whether the tree
+was dirty, the full command line, the interpreter and platform, the run time in
+UTC, and a SHA-256 for every named input file. Hashing the inputs is what makes
+the record worth anything — "produced by version X from these exact files" is a
+claim a sceptic can test, "produced by version X" is not.
+
+Deliberately: no network, no imports beyond the standard library, and every
+failure is swallowed. A provenance stamp must never be the reason an analysis
+dies.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import platform
+import subprocess
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+__all__ = ["provenance", "stamp", "write_json", "sha256"]
+
+_MAX_HASH_BYTES = 2_000_000_000     # 2 GB: beyond this, record the size only
+
+
+def sha256(path) -> str | None:
+    """SHA-256 of a file, or None if it cannot be read. Never raises."""
+    try:
+        p = Path(path)
+        if not p.is_file():
+            return None
+        if p.stat().st_size > _MAX_HASH_BYTES:
+            return f"(not hashed: {p.stat().st_size} bytes)"
+        h = hashlib.sha256()
+        with open(p, "rb") as fh:
+            for chunk in iter(lambda: fh.read(1 << 20), b""):
+                h.update(chunk)
+        return h.hexdigest()
+    except Exception:
+        return None
+
+
+def _git(*args, cwd=None):
+    try:
+        out = subprocess.run(["git", *args], cwd=cwd, capture_output=True,
+                             text=True, timeout=10)
+        return out.stdout.strip() if out.returncode == 0 else None
+    except Exception:
+        return None
+
+
+def provenance(inputs=None, script=None, extra=None) -> dict:
+    """The record itself. Safe to call from anywhere; never raises."""
+    try:
+        script_path = Path(script or sys.argv[0]).resolve()
+    except Exception:
+        script_path = Path(sys.argv[0] if sys.argv else "unknown")
+    repo = script_path.parent if script_path.exists() else None
+
+    # Only report a commit if THIS FILE is tracked by that repository. The
+    # Descriptron scripts currently sit untracked inside a clone of
+    # facebookresearch/segment-anything-2, so a naive `git rev-parse HEAD` returns
+    # SAM2's commit — a hash that says nothing about the code that produced the
+    # output, and is worse than no hash because it looks authoritative.
+    commit = dirty = tracked = None
+    if repo is not None:
+        tracked = bool(_git("ls-files", "--error-unmatch", str(script_path), cwd=repo))
+        if tracked:
+            commit = _git("rev-parse", "HEAD", cwd=repo)
+            dirty = bool(_git("status", "--porcelain", "--", str(script_path), cwd=repo))
+
+    rec = {
+        "script": script_path.name,
+        "script_path": str(script_path),
+        "script_sha256": sha256(script_path),
+        "git_commit": commit,
+        "git_dirty": dirty,
+        # a run from an untracked tree cannot be pinned to a version; say so in
+        # the record rather than leaving a reader to infer it
+        "git_tracked": tracked,
+        "git_note": (None if commit else
+                     "this script is NOT tracked by any git repository, so the output "
+                     "cannot be tied to a version of the code"),
+        "command": " ".join(sys.argv),
+        "argv": list(sys.argv),
+        "cwd": os.getcwd(),
+        "python": sys.version.split()[0],
+        "platform": platform.platform(),
+        "hostname": platform.node(),
+        "run_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+        "inputs": {},
+    }
+    for item in (inputs or []):
+        if item is None:
+            continue
+        try:
+            rec["inputs"][str(item)] = sha256(item)
+        except Exception:
+            rec["inputs"][str(item)] = None
+    if extra:
+        rec["extra"] = extra
+    return rec
+
+
+def stamp(out_dir, inputs=None, script=None, extra=None, name="provenance.json"):
+    """Write a provenance record beside an output that is not itself JSON."""
+    try:
+        d = Path(out_dir)
+        d.mkdir(parents=True, exist_ok=True)
+        rec = provenance(inputs=inputs, script=script, extra=extra)
+        (d / name).write_text(json.dumps(rec, indent=2))
+        return d / name
+    except Exception:
+        return None
+
+
+def write_json(path, payload, inputs=None, script=None, extra=None, indent=2):
+    """json.dump with a `_provenance` key added. Drop-in for an existing dump."""
+    try:
+        rec = provenance(inputs=inputs, script=script, extra=extra)
+        if isinstance(payload, dict):
+            payload = {**payload, "_provenance": rec}
+        else:
+            payload = {"data": payload, "_provenance": rec}
+    except Exception:
+        pass
+    p = Path(path)
+    p.parent.mkdir(parents=True, exist_ok=True)
+    p.write_text(json.dumps(payload, indent=indent, default=str))
+    return p
+
+
+def verify(record_path) -> dict:
+    """
+    Re-check a stamped record: does the script still hash the same, do the inputs
+    still hash the same? This is what a sceptical reader runs.
+    """
+    rec = json.loads(Path(record_path).read_text())
+    rec = rec.get("_provenance", rec)
+    out = {"script": None, "inputs": {}, "ok": True}
+    now = sha256(rec.get("script_path"))
+    out["script"] = ("unchanged" if now and now == rec.get("script_sha256")
+                     else "CHANGED or missing")
+    if out["script"] != "unchanged":
+        out["ok"] = False
+    for path, want in (rec.get("inputs") or {}).items():
+        got = sha256(path)
+        same = (got is not None and got == want)
+        out["inputs"][path] = "unchanged" if same else "CHANGED or missing"
+        if not same:
+            out["ok"] = False
+    # records written by run_full_pipeline_v2 also list what the step wrote
+    if isinstance(rec.get("outputs"), dict):
+        out["outputs"] = {}
+        for path, want in rec["outputs"].items():
+            if path.startswith("("):
+                continue
+            got = sha256(path)
+            same = (got is not None and got == want)
+            out["outputs"][path] = "unchanged" if same else "CHANGED or missing"
+            if not same:
+                out["ok"] = False
+    return out
+
+
+def which(result_file, search_dir) -> list:
+    """Which pipeline step wrote this file? Hash it and look for the hash among the outputs of
+    every provenance record under search_dir. A match means this exact content was written by
+    that step; no match means the file was changed afterwards or made outside the pipeline."""
+    want = sha256(result_file)
+    hits = []
+    if want is None:
+        return hits
+    for rp in sorted(Path(search_dir).rglob("*provenance*.json")):
+        try:
+            rec = json.loads(rp.read_text())
+        except Exception:
+            continue
+        rec = rec.get("_provenance", rec)
+        outs = rec.get("outputs") or {}
+        if isinstance(outs, dict) and want in outs.values():
+            hits.append({"record": str(rp),
+                         "step": (rec.get("extra") or {}).get("pipeline_step"),
+                         "script": rec.get("script"), "script_sha256": rec.get("script_sha256"),
+                         "git_commit": rec.get("git_commit"), "run_utc": rec.get("run_utc"),
+                         "command": rec.get("command")})
+    return hits
+
+
+if __name__ == "__main__":
+    import argparse
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--verify", help="a provenance.json (or a report with _provenance) to re-check")
+    ap.add_argument("--show", action="store_true", help="print a record for this invocation")
+    ap.add_argument("--which", help="a result file: find the provenance record of the step that wrote it")
+    ap.add_argument("--search", default=".", help="folder to search for provenance records (with --which)")
+    a = ap.parse_args()
+    if a.verify:
+        result = verify(a.verify)
+        print(json.dumps(result, indent=2))
+        sys.exit(0 if result["ok"] else 1)
+    if a.which:
+        hits = which(a.which, a.search)
+        print(json.dumps(hits, indent=2) if hits else
+              "no provenance record lists this exact file content (changed since, or not made by the pipeline)")
+        sys.exit(0 if hits else 1)
+    if a.show:
+        print(json.dumps(provenance(), indent=2))

{descriptron_core-2.1.2 → descriptron_core-2.1.3}/src/descriptron_core/tools/run_full_pipeline_v2.py

@@ -164,6 +164,118 @@ def _has_keypoints(coco_json_path: str) -> bool:
     return False
 
 
+_OUT_FLAGS = ("--output_dir", "--output-dir", "--out_dir", "--out-dir", "--output_base", "--outdir")
+_MAX_OUTPUT_FILES = 2000        # hash at most this many files written by one step
+
+
+def _dir_manifest(d: Path) -> Dict:
+    """A cheap fingerprint of an input folder (e.g. thousands of images): names, sizes and
+    modification times of its files, hashed together. Contents are not read."""
+    import hashlib
+    h, n = hashlib.sha256(), 0
+    for p in sorted(d.rglob("*")):
+        if p.is_file():
+            st = p.stat()
+            h.update(f"{p.relative_to(d)}\t{st.st_size}\t{int(st.st_mtime)}\n".encode())
+            n += 1
+    return {"files": n, "manifest_sha256": h.hexdigest()}
+
+
+def _prov_import():
+    """biorag_provenance_v1 sits beside this script (working tree, pip package, Docker)."""
+    if str(SCRIPT_DIR) not in sys.path:
+        sys.path.insert(0, str(SCRIPT_DIR))
+    import biorag_provenance_v1
+    return biorag_provenance_v1
+
+
+def _step_pre_state(cmd: List[str]) -> Dict:
+    """Before a step runs: its script and every file or folder named on its command line."""
+    try:
+        sha256 = _prov_import().sha256
+        script = next((c for c in cmd if str(c).endswith(".py")), None)
+        files, folders = {}, {}
+        skip_next = False
+        for c in cmd[1:]:
+            if skip_next:                        # the step's output folder is not an input
+                skip_next = False
+                continue
+            if str(c) in _OUT_FLAGS:
+                skip_next = True
+                continue
+            p = Path(str(c))
+            if str(c).startswith("-") or str(c) == script or len(str(c)) > 4096:
+                continue
+            if p.is_file():
+                files[str(p)] = sha256(p)
+            elif p.is_dir():
+                folders[str(p)] = _dir_manifest(p)
+        return {"script": script, "files": files, "folders": folders}
+    except Exception as e:                                   # never the reason a step fails
+        return {"error": f"{type(e).__name__}: {e}"}
+
+
+def _stamp_step(cmd: List[str], step_name: str, log_dir: Path, pre: Dict,
+                t0: float, elapsed: float, returncode: int) -> None:
+    """Write <log_dir>/<step>.provenance.json (and provenance_<step>.json in the step's output
+    folder): the step script and its SHA-256, git commit if tracked, the command, a SHA-256 of
+    every input file named on the command line (taken BEFORE the step ran; files the step then
+    overwrote are listed under both inputs and outputs), a manifest of every input folder, and a
+    SHA-256 of every file the step wrote. Check later with
+    `descriptron biorag_provenance_v1 --verify <file>` (or `python biorag_provenance_v1.py ...` from the source tree). Never raises."""
+    try:
+        _pv = _prov_import()
+        provenance, sha256 = _pv.provenance, _pv.sha256
+        extra = {"pipeline_step": step_name, "exit_code": returncode,
+                 "seconds": round(elapsed, 1), "python_bin": cmd[0] if cmd else None,
+                 "input_folders": pre.get("folders", {}),
+                 "pipeline_script": Path(__file__).name,
+                 "pipeline_script_sha256": sha256(Path(__file__).resolve())}
+        try:
+            from importlib.metadata import version
+            extra["descriptron_packages"] = {}
+            for dist in ("descriptron", "descriptron-core", "descriptron-vision"):
+                try:
+                    extra["descriptron_packages"][dist] = version(dist)
+                except Exception:
+                    pass
+        except Exception:
+            pass
+        rec = provenance(inputs=list(pre.get("files", {})), script=pre.get("script"), extra=extra)
+        rec["command"] = " ".join(str(c) for c in cmd)            # the step's command, not the pipeline's
+        rec["argv"] = [str(c) for c in cmd]
+        rec["inputs"] = pre.get("files", {})                      # hashes as they were before the run
+        if pre.get("error"):
+            rec["pre_state_error"] = pre["error"]
+        out_dir = None
+        for i, c in enumerate(cmd[:-1]):
+            if str(c) in _OUT_FLAGS:
+                out_dir = Path(str(cmd[i + 1]))
+                break
+        outputs = {}
+        if out_dir is not None and out_dir.is_dir():
+            for p in sorted(out_dir.rglob("*")):
+                if len(outputs) >= _MAX_OUTPUT_FILES:
+                    outputs["(truncated)"] = f"more than {_MAX_OUTPUT_FILES} files written"
+                    break
+                # the pipeline's own bookkeeping (its running log, the step logs and every
+                # provenance record) keeps changing after the step and is not the step's output
+                if (p.is_file() and p.stat().st_mtime >= t0 - 1
+                        and not p.name.startswith("provenance_")
+                        and not p.name.endswith(".provenance.json")
+                        and p.name != "pipeline_log.txt"
+                        and log_dir.resolve() not in p.resolve().parents):
+                    outputs[str(p)] = sha256(p)
+        rec["outputs"] = outputs
+        rec["output_dir"] = str(out_dir) if out_dir else None
+        text = json.dumps(rec, indent=2, default=str)
+        (log_dir / f"{step_name}.provenance.json").write_text(text)
+        if out_dir is not None and out_dir.is_dir():
+            (out_dir / f"provenance_{step_name}.json").write_text(text)
+    except Exception as e:
+        logger.warning(f"  (provenance record not written: {type(e).__name__}: {e})")
+
+
 def _run_step(cmd: List[str], step_name: str, log_dir: Path,
               dry_run: bool = False) -> bool:
     cmd_str = " \\\n  ".join(cmd)
@@ -173,6 +285,7 @@ def _run_step(cmd: List[str], step_name: str, log_dir: Path,
         logger.info(f"  [DRY RUN] Skipping execution")
         return True
 
+    pre = _step_pre_state(cmd)          # provenance: hash the inputs before the step can change them
     log_file = log_dir / f"{step_name}.log"
     with open(log_file, "w") as lf:
         lf.write(f"# {step_name}\n# {datetime.now().isoformat()}\n")
@@ -185,6 +298,8 @@ def _run_step(cmd: List[str], step_name: str, log_dir: Path,
         )
         elapsed = time.time() - t0
 
+    _stamp_step(cmd, step_name, log_dir, pre, t0, elapsed, result.returncode)
+
     if result.returncode != 0:
         logger.error(f"  FAILED (exit code {result.returncode}, {elapsed:.0f}s)")
         logger.error(f"  See log: {log_file}")
@@ -1538,7 +1653,7 @@ def step_char_gate(cfg: Dict, python: str, log_dir: Path) -> bool:
     # only what it was told to. "llm" is available and reads better, but measured on the Diaphorina
     # treatments it also deleted statements about characters that had passed, in 47% of its edits —
     # invisible to both the subsequence check and the numeric audit. See BIORAG_V2_README.md §9d.
-    engine = cfg.get("gate_engine", "python")
+    engine = cfg.get("gate_engine") or "python"      # argparse stores None when --gate_engine is not given
     if engine == "llm" and cfg.get("llm_backend") == "none":
         logger.info("  no LLM backend configured — editing by rule instead (--engine python)")
         engine = "python"