descriptron-core 2.1.0__tar.gz → 2.1.2__tar.gz

descriptron_core-2.1.2/NOTICE

@@ -0,0 +1,59 @@
+Descriptron
+Copyright 2024-2026 Alex R. Van Dam (ORCID 0000-0002-1966-0338)
+
+This product includes software developed by Alex R. Van Dam
+https://github.com/alexrvandam/Descriptron
+
+Licensed under the Apache License, Version 2.0 (see LICENSE). Under Section
+4(d) of that licence, this NOTICE file must be included with any
+redistribution of this software or of any work derived from it.
+
+REQUIRED CITATION
+-----------------
+Anyone who uses Descriptron, or any software derived from it (including
+descriptron-core, descriptron-vision, descriptron-gui and descriptron-mcp), in
+work that is published, presented or distributed must cite:
+
+  1. The software (Descriptron v2), by the DOI of the version used:
+     Van Dam, A. R. (2026). Descriptron: morphology-driven species descriptions,
+     keys and delimitation for dark taxa (Version 2.1.1). Zenodo.
+     https://doi.org/10.5281/zenodo.22958109
+     (all versions: https://doi.org/10.5281/zenodo.17077224;
+      repository: https://github.com/alexrvandam/Descriptron)
+
+  2. The first Descriptron paper:
+     Van Dam, A. R. & Štarhová Serbina, L. (2026). Descriptron: Artificial
+     intelligence for automating taxonomic species descriptions with a
+     user-friendly software package. Systematic Entomology, 51(1), e70005.
+     https://doi.org/10.1111/syen.70005
+
+  3. Once it is published, the Descriptron v2 paper as well. Its reference
+     will be added here and in CITATION.cff on publication.
+
+BibTeX:
+
+@software{vandam_2026_descriptron_v211,
+  author    = {Van Dam, Alex R.},
+  title     = {Descriptron: morphology-driven species descriptions, keys and
+               delimitation for dark taxa},
+  version   = {v2.1.1},
+  year      = {2026},
+  publisher = {Zenodo},
+  doi       = {10.5281/zenodo.22958109},
+  url       = {https://doi.org/10.5281/zenodo.22958109}
+}
+
+@article{vandam2026descriptron,
+  author  = {Van Dam, Alex R. and Štarhová Serbina, Liliya},
+  title   = {Descriptron: Artificial intelligence for automating taxonomic species
+             descriptions with a user-friendly software package},
+  journal = {Systematic Entomology},
+  volume  = {51},
+  number  = {1},
+  pages   = {e70005},
+  year    = {2026},
+  doi     = {10.1111/syen.70005}
+}
+
+The same information is in CITATION.cff, which GitHub shows as
+"Cite this repository".

{descriptron_core-2.1.0 → descriptron_core-2.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: descriptron-core
-Version: 2.1.0
+Version: 2.1.2
 Summary: Evidence-tiered species descriptions, keys and audits from phenomic matrices
 Project-URL: Homepage, https://github.com/alexrvandam/Descriptron
 Project-URL: Repository, https://github.com/alexrvandam/Descriptron
@@ -9,6 +9,7 @@ Author: Alex R. Van Dam
 License: Apache-2.0
 License-File: LICENSE
 License-File: NOTICE
+License-File: NOTICE.bak_pre_212_20260929_1100
 Keywords: COCO,morphometrics,phenomics,species description,taxonomy
 Classifier: Intended Audience :: Science/Research
 Classifier: License :: OSI Approved :: Apache Software License

descriptron_core-2.1.2/pyproject.toml

@@ -0,0 +1,82 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "descriptron-core"
+version = "2.1.2"
+description = "Evidence-tiered species descriptions, keys and audits from phenomic matrices"
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "Apache-2.0" }
+authors = [{ name = "Alex R. Van Dam" }]
+keywords = ["taxonomy", "morphometrics", "phenomics", "species description", "COCO"]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "Topic :: Scientific/Engineering :: Bio-Informatics",
+  "Topic :: Scientific/Engineering :: Image Processing",
+  "Intended Audience :: Science/Research",
+  "License :: OSI Approved :: Apache Software License",
+  "Operating System :: OS Independent",
+]
+
+# Derived from the imports of the 62 programs in this package, not from memory:
+# `python -m descriptron_core.cli --list` names them. No torch, no CUDA, no
+# compiler — this installs in about a minute on Linux, macOS and Windows, and
+# with the deposited COCO files it reproduces every analysis in the methods paper.
+dependencies = [
+  "numpy>=1.23",
+  "pandas>=1.5",
+  "scipy>=1.9",
+  "matplotlib>=3.6",
+  "scikit-learn>=1.2",
+  "scikit-image>=0.20",
+  "opencv-python-headless>=4.7",   # headless: the analysis half never opens a window
+  "pillow>=9",
+  "pycocotools>=2.0.6",
+  "seaborn>=0.12",
+  "statsmodels>=0.14",
+  "scikit-posthocs>=0.7",
+  "umap-learn>=0.5",
+  "networkx>=3",
+  "pyefd>=1.6",
+  "biopython>=1.81",
+  "PyYAML>=6",
+  "lxml>=4.9",
+  "openpyxl>=3.1",
+  "python-docx>=1.0",              # treatment and monograph .docx export
+]
+
+[project.optional-dependencies]
+# Only needed to *write* treatments. The audit, key, matrix, novelty scoring and
+# delimitation are all deterministic and need no model at all.
+llm = ["anthropic>=0.40"]
+# literature retrieval (the RAG in BioRAG): BioSysLit treatments from Zenodo and your
+# own PDFs. Keyword retrieval needs only the first two; sentence-embedding retrieval
+# (`index --embeddings`) also needs sentence-transformers + faiss-cpu, which pull in torch.
+rag = ["requests>=2.28", "pdfplumber>=0.10"]
+rag-embeddings = ["requests>=2.28", "pdfplumber>=0.10", "sentence-transformers>=2.2", "faiss-cpu>=1.7"]
+dev = ["pytest>=7", "build>=1", "twine>=5"]
+
+[project.scripts]
+descriptron        = "descriptron_core.cli:main"
+biorag-pipeline    = "descriptron_core.cli:run_pipeline"
+biorag-key         = "descriptron_core.cli:run_key_builder"
+biorag-audit       = "descriptron_core.cli:run_confabulation_checker"
+biorag-novelty     = "descriptron_core.cli:run_novelty_score"
+biorag-calibrate   = "descriptron_core.cli:run_calibrate"
+descriptron-keys   = "descriptron_core.cli:run_keys"
+
+[project.urls]
+Homepage = "https://github.com/alexrvandam/Descriptron"
+Repository = "https://github.com/alexrvandam/Descriptron"
+Issues = "https://github.com/alexrvandam/Descriptron/issues"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/descriptron_core"]
+
+# tools/ and data/ sit inside the package directory, so `packages` already ships
+# them; a force-include here would add every file a second time and the build
+# fails with "a second file is being added to the wheel archive at the same path".
+[tool.hatch.build]
+artifacts = ["src/descriptron_core/tools/**", "src/descriptron_core/data/**"]

{descriptron_core-2.1.0 → descriptron_core-2.1.2}/src/descriptron_core/tools/descriptron_shape_stats.py

@@ -618,33 +618,52 @@ def main(argv=None):
 
     def safe_name(c):
         return re.sub(r"\W+", "_", c)
+
+    # specimens with a value for every variable of a model: a blank group (an unidentified specimen) or a blank
+    # factor must not become a level of its own. The GPA above still uses every specimen; unidentified ones are
+    # what the assignment is for.
+    def complete(cols):
+        return [i for i in range(len(meta))
+                if all(str(data[c][i]).strip().lower() not in ("", "na", "nan", "none") for c in cols if c in data)]
+
+    def sub(idx):
+        return {c: [v[i] for i in idx] for c, v in data.items()}
+
+    def left_out(idx):
+        n = len(meta) - len(idx)
+        return [f"{n} specimen(s) without a value for a model variable left out of this analysis."] if n else []
     if "anova" in a.analyses:
         default = " + ".join([c for c in [group_col] + factors + conts if c])
         f = a.formula or default
         if f:
-            design = Design(f, data, kinds)
-            tab = procrustes_anova(Y, design, a.iterations, rng)
+            used = [c for c in kinds if c != "logCS" and re.search(r"(?<![\w])" + re.escape(c) + r"(?![\w])", f)]
+            idx = complete(used)
+            design = Design(f, sub(idx), kinds)
+            tab = procrustes_anova(Y[idx], design, a.iterations, rng)
             write_rows(out / "procrustes_anova.csv", tab)
-            report += ["## Procrustes ANOVA (RRPP, type I SS)", f"shape ~ {f}", _md(tab), ""]
+            report += ["## Procrustes ANOVA (RRPP, type I SS)", f"shape ~ {f}", *left_out(idx), _md(tab), ""]
     if "allometry" in a.analyses and group_col:
-        design = Design(f"logCS*{group_col}", data, kinds)
-        tab = procrustes_anova(Y, design, a.iterations, rng)
+        idx = complete([group_col])
+        design = Design(f"logCS*{group_col}", sub(idx), kinds)
+        tab = procrustes_anova(Y[idx], design, a.iterations, rng)
         write_rows(out / "allometry_homogeneity_of_slopes.csv", tab)
         inter = next(r for r in tab if r["term"] == f"logCS:{group_col}")
         report += ["## Allometry: homogeneity of slopes", f"shape ~ logCS * {group_col}; the interaction tests whether groups "
-                   f"differ in allometric slope (P = {inter['P']:.3g}).", _md(tab), ""]
+                   f"differ in allometric slope (P = {inter['P']:.3g}).", *left_out(idx), _md(tab), ""]
     if "trajectory" in a.analyses and group_col and (a.trajectory_levels or factors):
         lev = a.trajectory_levels or factors[0]
-        tr = trajectory(Y, data[group_col], [m.get(lev, "") for m in meta], a.iterations, rng)
+        idx = [i for i in complete([group_col]) if str(meta[i].get(lev, "")).strip()]
+        tr = trajectory(Y[idx], [data[group_col][i] for i in idx], [meta[i].get(lev, "") for i in idx], a.iterations, rng)
         if tr:
             write_rows(out / "trajectory_pairwise.csv", tr["pairs"])
             report += ["## Phenotypic trajectories", f"{group_col} across levels of {lev}: {tr['levels']}", _md(tr["pairs"]), ""]
         else:
             report += ["## Phenotypic trajectories", f"skipped: needs >= 2 groups each observed at every level of {lev}", ""]
     if "disparity" in a.analyses and group_col:
-        pv, prs = disparity(Y, data[group_col], a.iterations, rng)
+        idx = complete([group_col])
+        pv, prs = disparity(Y[idx], [data[group_col][i] for i in idx], a.iterations, rng)
         write_rows(out / "disparity_by_group.csv", pv); write_rows(out / "disparity_pairwise.csv", prs)
-        report += ["## Morphological disparity (Procrustes variance)", _md(pv), _md(prs), ""]
+        report += ["## Morphological disparity (Procrustes variance)", *left_out(idx), _md(pv), _md(prs), ""]
     if "pls" in a.analyses and conts:
         E = np.column_stack([np.array(data[c], float) for c in conts]); E = (E - E.mean(0)) / np.where(E.std(0) > 0, E.std(0), 1)
         r = two_block_pls(Y, E, a.iterations, rng)

descriptron_core-2.1.2/src/descriptron_core/tools/validate_centerline_lengths.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""Centre-line length on shapes of known length (straight bar, arcs of increasing bend, an S-curve), at several
+widths, against the straight measurement a length-along-the-main-axis would give. Writes a CSV and prints a summary;
+the numbers quoted in the paper and README come from this script.
+
+  python validate_centerline_lengths.py --out_csv centerline_validation.csv
+"""
+import argparse
+import csv
+import sys
+from pathlib import Path
+
+import cv2
+import numpy as np
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import descriptron_centerline as dc  # noqa: E402
+
+
+def mask(pts, width, size=1000):
+    m = np.zeros((size, size), np.uint8)
+    cv2.polylines(m, [np.round(pts).astype(np.int32)], False, 1, width)
+    return m.astype(bool)
+
+
+def arclen(p):
+    return float(np.hypot(*np.diff(p, axis=0).T).sum())
+
+
+def axis_extent(m):
+    ys, xs = np.nonzero(m)
+    P = np.stack([xs, ys], 1).astype(float); P -= P.mean(0)
+    pr = P @ np.linalg.svd(P, full_matrices=False)[2][0]
+    return float(pr.max() - pr.min())
+
+
+T = np.linspace(0, 1, 4000)
+SHAPES = {"straight": np.stack([200 + 346.4 * T, 300 + 200 * T], 1)}
+for deg in (60, 90, 120):
+    a = np.radians(-90 - deg / 2 + deg * T)
+    SHAPES[f"arc_{deg}deg"] = np.stack([500 + 300 * np.cos(a), 700 + 300 * np.sin(a)], 1)
+SHAPES["s_curve"] = np.stack([150 + 700 * T, 500 + 120 * np.sin(2 * np.pi * T)], 1)
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--out_csv", required=True)
+    a = ap.parse_args()
+    rows = []
+    for name, pts in SHAPES.items():
+        for w in (6, 10, 20, 30):
+            m = mask(pts, w)
+            true = arclen(pts) + w                     # round caps add w/2 at each end
+            r = dc.centerline(m)
+            rows.append(dict(shape=name, width_px=w, true_length_px=round(true, 1),
+                             centerline_px=round(r["length_px"], 1),
+                             centerline_err_pct=round(100 * (r["length_px"] - true) / true, 2),
+                             straight_axis_px=round(axis_extent(m), 1),
+                             straight_err_pct=round(100 * (axis_extent(m) - true) / true, 2)))
+    with open(a.out_csv, "w", newline="") as fh:
+        wr = csv.DictWriter(fh, fieldnames=list(rows[0])); wr.writeheader(); wr.writerows(rows)
+    ce = np.abs([r["centerline_err_pct"] for r in rows])
+    print(f"centre line: |error| median {np.median(ce):.2f}%, max {ce.max():.2f}% over {len(rows)} shape x width cases")
+    for name in SHAPES:
+        s = [r["straight_err_pct"] for r in rows if r["shape"] == name]
+        c = [r["centerline_err_pct"] for r in rows if r["shape"] == name]
+        print(f"  {name:12s} straight axis {min(s):+.1f} to {max(s):+.1f}%   centre line {min(c):+.2f} to {max(c):+.2f}%")
+
+
+if __name__ == "__main__":
+    main()

descriptron_core-2.1.2/src/descriptron_core/tools/validate_texture_glcm.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""Checks of Descriptron's texture features (texture_phenomics_homology.py).
+
+1. Masked GLCM against scikit-image: pixels outside the mask get an extra 'outside' grey level, scikit-image builds
+   the symmetric GLCM, and that level's row and column are dropped; what remains is exactly the 'both pixels in the
+   mask' matrix Descriptron computes. (scikit-image names the two diagonals the other way round; Descriptron averages
+   the four directions, so its features are unaffected.)
+2. Grey-level invariance: the same image quantised to 8 and 32 levels; ratio of the feature values (1 = invariant),
+   for Descriptron's invariant Haralick formulas and the standard ones.
+
+  python validate_texture_glcm.py --image_glob "<dir>/*.png" --texture_script <texture_phenomics_homology.py> --out_csv out.csv
+"""
+import argparse
+import csv
+import glob
+import importlib.util
+
+import cv2
+import numpy as np
+from skimage.feature import graycomatrix
+
+ANGLES = (0, np.pi / 4, np.pi / 2, 3 * np.pi / 4)
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--image_glob", required=True); ap.add_argument("--texture_script", required=True)
+    ap.add_argument("--n_images", type=int, default=15); ap.add_argument("--out_csv", required=True)
+    a = ap.parse_args()
+    spec = importlib.util.spec_from_file_location("tx", a.texture_script)
+    tx = importlib.util.module_from_spec(spec); spec.loader.exec_module(tx)
+    imgs = sorted(glob.glob(a.image_glob))[: a.n_images]
+    rows = []
+    worst, n = 0.0, 0
+    for f in imgs:
+        g = cv2.imread(f, 0); q = (g // 16).astype(np.uint8)
+        h, w = q.shape; yy, xx = np.mgrid[:h, :w]
+        for s in range(3):
+            cy, cx, ry, rx = h * (0.3 + 0.2 * s), w * (0.3 + 0.2 * s), h * 0.25, w * 0.2
+            m = ((yy - cy) / ry) ** 2 + ((xx - cx) / rx) ** 2 < 1
+            for ang in ANGLES:
+                ours = tx._masked_glcm_single(q, m, 3, ang, levels=16)
+                sk = {np.pi / 4: 3 * np.pi / 4, 3 * np.pi / 4: np.pi / 4}.get(ang, ang)
+                R = graycomatrix(np.where(m, q, 16).astype(np.uint8), [3], [sk], levels=17,
+                                 symmetric=True, normed=False)[:16, :16, 0, 0].astype(float)
+                R /= R.sum()
+                worst = max(worst, float(np.abs(ours - R).max())); n += 1
+        full = np.ones(g.shape, bool); gf = g.astype(float); vals = {}
+        for L in (8, 32):
+            ql = np.clip((gf - gf.min()) / (gf.max() - gf.min()) * (L - 1), 0, L - 1).astype(np.uint8)
+            G = np.mean([tx._masked_glcm_single(ql, full, 3, an, levels=L) for an in ANGLES], axis=0)
+            I, J = np.meshgrid(np.arange(L), np.arange(L), indexing="ij")
+            vals[L] = (tx._haralick_invariant(G, levels=L),
+                       [float((G * (I - J) ** 2).sum()), float((G / (1 + (I - J) ** 2)).sum()), float((G ** 2).sum())])
+        for k, name in enumerate(("contrast", "homogeneity", "energy")):
+            rows.append(dict(image=f, feature=name, invariant_ratio_32_vs_8=vals[32][0][k] / vals[8][0][k],
+                             standard_ratio_32_vs_8=vals[32][1][k] / vals[8][1][k]))
+    with open(a.out_csv, "w", newline="") as fh:
+        wr = csv.DictWriter(fh, fieldnames=list(rows[0])); wr.writeheader(); wr.writerows(rows)
+    print(f"masked GLCM vs scikit-image: max |difference| {worst:.2e} over {n} matrices")
+    for name in ("contrast", "homogeneity", "energy"):
+        r = [x for x in rows if x["feature"] == name]
+        print(f"{name:12s} ratio 32/8 levels, median: invariant {np.median([x['invariant_ratio_32_vs_8'] for x in r]):.2f}"
+              f"   standard {np.median([x['standard_ratio_32_vs_8'] for x in r]):.2f}")
+
+
+if __name__ == "__main__":
+    main()