descriptron-core 2.7.8__tar.gz → 2.8.0__tar.gz

{descriptron_core-2.7.8 → descriptron_core-2.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: descriptron-core
-Version: 2.7.8
+Version: 2.8.0
 Summary: Evidence-tiered species descriptions, keys and audits from phenomic matrices
 Project-URL: Homepage, https://github.com/alexrvandam/Descriptron
 Project-URL: Repository, https://github.com/alexrvandam/Descriptron

{descriptron_core-2.7.8 → descriptron_core-2.8.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "descriptron-core"
-version = "2.7.8"
+version = "2.8.0"
 description = "Evidence-tiered species descriptions, keys and audits from phenomic matrices"
 readme = "README.md"
 requires-python = ">=3.9"

descriptron_core-2.8.0/src/descriptron_core/tools/annotation_scale_transfer_v1.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+"""Annotate huge images on smaller copies, then put the annotations back on the originals (17 Oct 2026; general, any taxon).
+
+  shrink    --images DIR [--list FILE] --out DIR [--max_side 2700 | --scale 0.5] [--quality 95]
+            writes a JPG copy of every image (aspect kept) + scale_map.tsv (small file -> original path and both sizes)
+  to_small  --coco FULL.json --scale_map scale_map.tsv --out SMALL.json
+            existing full-resolution annotations (e.g. predictions to correct) -> coordinates of the small copies
+  to_full   --coco SMALL.json(.rlejson) --scale_map scale_map.tsv --out FULL.json
+            annotations made on the small copies -> original image names, sizes and coordinates; writes FULL.json (polygons)
+            AND FULL.rlejson (RLE, keeps holes), as the GUI does
+
+Masks are scaled as CONTOURS (outer + holes, sub-pixel scaled, refilled at the target size), not by nearest-neighbour upsampling,
+so edges stay smooth. Keypoints, keypoints_clicked, bbox, area, line_points and line_thickness are scaled; every other field is copied.
+A mask made at scale s can only be as precise as 1/s original pixels: at 0.5 that is about 2 px on the original."""
+import argparse, csv, json, os, sys, numpy as np, cv2
+from PIL import Image
+Image.MAX_IMAGE_PIXELS = None
+EXT = ('.tif', '.tiff', '.png', '.jpg', '.jpeg', '.bmp')
+
+
+def shrink(a):
+    os.makedirs(a.out, exist_ok=True)
+    files = [l.strip() for l in open(a.list) if l.strip()] if a.list else sorted(f for f in os.listdir(a.images) if f.lower().endswith(EXT) and not f.startswith('._') and '_mask' not in f)
+    rows = []
+    for f in files:
+        p = f if os.path.isabs(f) else os.path.join(a.images, f); im = Image.open(p); W, H = im.size
+        s = a.scale if a.scale else min(1.0, a.max_side / max(W, H)); w, h = max(1, round(W * s)), max(1, round(H * s))
+        small = os.path.splitext(os.path.basename(p))[0] + '.jpg'
+        im.convert('RGB').resize((w, h), Image.LANCZOS).save(os.path.join(a.out, small), quality=a.quality)
+        rows.append((small, os.path.abspath(p), W, H, w, h)); print(f'{small}: {W}x{H} -> {w}x{h}', flush=True)
+    with open(os.path.join(a.out, 'scale_map.tsv'), 'w') as fh:
+        fh.write('small_file\toriginal_path\torig_width\torig_height\tsmall_width\tsmall_height\n')
+        for r in rows: fh.write('\t'.join(map(str, r)) + '\n')
+    print(len(rows), 'images ->', a.out)
+
+
+def load_map(p):
+    M = {}
+    for r in csv.DictReader(open(p), delimiter='\t'):
+        d = dict(small=r['small_file'], orig=os.path.basename(r['original_path']), W=int(r['orig_width']), H=int(r['orig_height']), w=int(r['small_width']), h=int(r['small_height']))
+        for k in (d['small'], d['orig'], os.path.splitext(d['small'])[0], os.path.splitext(d['orig'])[0]): M[k] = d
+    return M
+
+
+def seg_to_mask(seg, h, w):
+    from pycocotools import mask as mu
+    if isinstance(seg, dict):
+        r = dict(seg)
+        if isinstance(r['counts'], list): r = mu.frPyObjects(r, h, w)
+        elif isinstance(r['counts'], str): r['counts'] = r['counts'].encode('ascii')
+        return mu.decode(r).astype(np.uint8)
+    m = np.zeros((h, w), np.uint8)
+    polys = [np.asarray(p, np.float64).reshape(-1, 2) for p in seg if len(p) >= 6]
+    if polys: cv2.fillPoly(m, [np.round(p).astype(np.int32) for p in polys], 1)
+    return m
+
+
+def scale_mask(m, sx, sy, W, H):
+    """contours (with holes) scaled about pixel centres, refilled at W x H with 8x sub-pixel precision"""
+    cs, hier = cv2.findContours(m.astype(np.uint8), cv2.RETR_CCOMP, cv2.CHAIN_APPROX_NONE); out = np.zeros((H, W), np.uint8)
+    if not cs: return out
+    SH = 3; f = 1 << SH
+    def sc(c): q = c.reshape(-1, 2).astype(np.float64); q[:, 0] = (q[:, 0] + 0.5) * sx - 0.5; q[:, 1] = (q[:, 1] + 0.5) * sy - 0.5; return np.round(q * f).astype(np.int32)
+    for i, c in enumerate(cs):
+        if hier[0][i][3] < 0: cv2.fillPoly(out, [sc(c)], 1, lineType=cv2.LINE_8, shift=SH)
+    for i, c in enumerate(cs):
+        if hier[0][i][3] >= 0: cv2.fillPoly(out, [sc(c)], 0, lineType=cv2.LINE_8, shift=SH)
+    return out
+
+
+def rle(m):
+    from pycocotools import mask as mu
+    r = mu.encode(np.asfortranarray(m.astype(np.uint8))); r['counts'] = r['counts'].decode('ascii'); return r
+
+
+def polys(m):
+    cs, _ = cv2.findContours(m.astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
+    return [c.reshape(-1).astype(float).tolist() for c in cs if len(c) >= 3]
+
+
+def scale_kp(v, sx, sy):
+    if isinstance(v, str):
+        try: v = json.loads(v)
+        except Exception: return v
+    if not isinstance(v, list): return v
+    if v and isinstance(v[0], (list, tuple)): return [[p[0] * sx, p[1] * sy] + list(p[2:]) for p in v]
+    out = list(v)
+    for i in range(0, len(out) - 2, 3): out[i] = out[i] * sx; out[i + 1] = out[i + 1] * sy
+    return out
+
+
+def convert(a, direction):
+    M = load_map(a.scale_map); d = json.load(open(a.coco)); img_new = {}; key = {}
+    for im in d['images']:
+        m = M.get(os.path.basename(im['file_name'])) or M.get(os.path.splitext(os.path.basename(im['file_name']))[0])
+        if m is None: sys.exit(f"image {im['file_name']} not in {a.scale_map}")
+        new = dict(im)
+        if direction == 'to_full': new.update(file_name=m['orig'], width=m['W'], height=m['H']); sx, sy, tw, th = m['W'] / m['w'], m['H'] / m['h'], m['W'], m['H']; sh, sw = im.get('height', m['h']), im.get('width', m['w'])
+        else: new.update(file_name=m['small'], width=m['w'], height=m['h']); sx, sy, tw, th = m['w'] / m['W'], m['h'] / m['H'], m['w'], m['h']; sh, sw = im.get('height', m['H']), im.get('width', m['W'])
+        if isinstance(im['id'], str) and os.path.splitext(os.path.basename(im['file_name']))[0] in im['id']:
+            new['id'] = im['id'].replace(os.path.splitext(os.path.basename(im['file_name']))[0], os.path.splitext(new['file_name'])[0])
+        key[im['id']] = (new['id'], sx, sy, tw, th, sh, sw); img_new[im['id']] = new
+    P = {k: v for k, v in d.items() if k not in ('images', 'annotations')}; R = dict(P)
+    P['images'] = R['images'] = list(img_new.values()); P['annotations'] = []; R['annotations'] = []
+    for an in d['annotations']:
+        iid, sx, sy, tw, th, sh, sw = key[an['image_id']]; b = dict(an, image_id=iid)
+        for k in ('keypoints', 'keypoints_clicked', 'line_points'):
+            if k in b: b[k] = scale_kp(b[k], sx, sy)
+        if 'line_thickness' in b and isinstance(b['line_thickness'], (int, float)): b['line_thickness'] = max(1, round(b['line_thickness'] * (sx + sy) / 2))
+        if isinstance(b.get('bbox'), list) and len(b['bbox']) == 4: x, y, w, h = b['bbox']; b['bbox'] = [x * sx, y * sy, w * sx, h * sy]
+        seg = an.get('segmentation')
+        if seg:
+            m = scale_mask(seg_to_mask(seg, sh, sw), sx, sy, tw, th)
+            b['area'] = float(m.sum());
+            if m.any(): b['bbox'] = [float(v) for v in cv2.boundingRect(m)]
+            P['annotations'].append(dict(b, segmentation=polys(m))); R['annotations'].append(dict(b, segmentation=rle(m)))
+        else:
+            if isinstance(b.get('area'), (int, float)) and b['area'] > 1: b['area'] = b['area'] * sx * sy
+            P['annotations'].append(b); R['annotations'].append(b)
+    base = os.path.splitext(a.out)[0]
+    json.dump(P, open(base + '.json', 'w')); json.dump(R, open(base + '.rlejson', 'w'))
+    print(f"{len(d['images'])} images, {len(d['annotations'])} annotations -> {base}.json (polygons) + {base}.rlejson (RLE)")
+
+
+if __name__ == '__main__':
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter); sp = ap.add_subparsers(dest='cmd', required=True)
+    s = sp.add_parser('shrink'); s.add_argument('--images', required=True); s.add_argument('--list', default=''); s.add_argument('--out', required=True)
+    s.add_argument('--max_side', type=int, default=2700); s.add_argument('--scale', type=float, default=0); s.add_argument('--quality', type=int, default=95)
+    for c in ('to_small', 'to_full'):
+        s = sp.add_parser(c); s.add_argument('--coco', required=True); s.add_argument('--scale_map', required=True); s.add_argument('--out', required=True)
+    a = ap.parse_args()
+    shrink(a) if a.cmd == 'shrink' else convert(a, a.cmd)

{descriptron_core-2.7.8 → descriptron_core-2.8.0}/src/descriptron_core/tools/biorag_confabulation_checker_v2.py

@@ -769,7 +769,17 @@ def verify_measurement(c: Claim, code: str, ev: Evidence, cits, issues: List[Dic
         elif matching and not ctx_fid:
             cand = [f for f in matching if f.split(".", 1)[0] in mentioned] or matching
             target = cand[0]
-            if len(cand) > 1 and len({f.split('.', 1)[1] for f in cand}) > 1:
+            # 2.7.9: a single value in a key-couplet context that is a key threshold is that threshold, even when
+            # it happens to equal some other measurement of the species (e.g. 'L* > 58 ... key couplet 20b')
+            key_thr = None
+            if hi is None and c.get("key_context") and ev.key:
+                for r_ in ev.key["couplets"]:
+                    for ch_ in r_["characters"]:
+                        if matches(lo, ch_.get("threshold")):
+                            key_thr = (ev.canon_fid(ch_["feature_id"]), r_["number"])
+            if key_thr is not None:
+                target, note = key_thr[0], f"key couplet {key_thr[1]} threshold"
+            elif len(cand) > 1 and len({f.split('.', 1)[1] for f in cand}) > 1:
                 status, etype, note = "review", "unresolved_attribution", f"value fits {', '.join(cand[:3])}"
             else:
                 note = "uncited; matched by value"

{descriptron_core-2.7.8 → descriptron_core-2.8.0}/src/descriptron_core/tools/biorag_description_refiner_v1.py

@@ -689,6 +689,16 @@ def codes_to_check(profile: Dict) -> List[str]:
     return out
 
 
+def deterministic_diagnosis(args, code):
+    """2.7.9: the diagnosis from biorag_diagnosis_builder_v1 (--deterministic_diagnosis diagnoses.json), or None"""
+    path = getattr(args, "deterministic_diagnosis", None)
+    if not path:
+        return None
+    if getattr(args, "_det_cache", None) is None:
+        args._det_cache = json.loads(Path(path).read_text())
+    return args._det_cache.get(code)
+
+
 def refine_one(code, ev, client, prompts, args, out_dir, log, cev=None):
     sp_dir = out_dir / code
     sp_dir.mkdir(parents=True, exist_ok=True)
@@ -786,6 +796,13 @@ def refine_one(code, ev, client, prompts, args, out_dir, log, cev=None):
         history.append({"attempt": "merged into the existing treatment", "problems": probs})
     cleaned = clean_treatment(treat, cev) if treat else {}
     removed = strip_bad_sentences(treat, allowed, codes_to_check(ev.profile)) if probs and treat else []
+    det = deterministic_diagnosis(args, code) if treat else None
+    if det:                                              # 2.7.9: the diagnosis written by fixed rules, verbatim
+        treat["diagnosis"] = det["text"]
+        treat["diagnosis_source"] = "deterministic (biorag_diagnosis_builder_v1): key path + lineage-specific characters"
+        treat["citations"] = (treat.get("citations") or []) + \
+            [c for c in det.get("citations", []) if c not in (treat.get("citations") or [])]
+        allowed["t1"] = list(allowed.get("t1", [])) + [float(x) for x in re.findall(r"\d+(?:\.\d+)?", det["text"])]
     final = (validate(treat, allowed, codes_to_check(ev.profile), sections) +
              independent_problems(code, treat, sheet, cev)) if treat else probs
     treat["_validation"] = {"attempts": history, "final_problems": final, "removed_sentences": removed,
@@ -839,6 +856,9 @@ def main():
     ap.add_argument("--added_by", default=None,
                     help="Who is adding those sections, e.g. 'Smith, 2031'. Recorded per section so "
                          "a reader can tell the original description from what was added later")
+    ap.add_argument("--deterministic_diagnosis", default=None,
+                    help="diagnoses.json from biorag_diagnosis_builder_v1: its text replaces the model's diagnosis "
+                         "verbatim (key path + lineage-specific characters, written by fixed rules)")
     ap.add_argument("--repair_existing", action="store_true",
                     help="Audit existing treatments and send only the flagged problems back to the model "
                          "(the previous JSON is kept as <code>_treatment_before_repair_<date>.json)")

descriptron_core-2.8.0/src/descriptron_core/tools/biorag_diagnosis_builder_v1.py

@@ -0,0 +1,243 @@
+#!/usr/bin/env python3
+"""
+biorag_diagnosis_builder_v1.py - a deterministic diagnosis per species, from the key and the matrix (no model)
+==========================================================================================================
+
+The model-written diagnosis (biorag_description_refiner_v1) chooses which of the supplied true statements to lead
+with, so two runs can stress different characters. This program writes the diagnosis itself, by fixed rules, so the
+same matrix and key always give the same text:
+
+  1. KEY CHARACTERS - the couplets on the species' own path through the identification key (biorag_key_builder_v1),
+     in key order; for each couplet its primary character, worded as the key words it, but with THIS species'
+     observed range and n (a key couplet describes a group of species).
+  2. LINEAGE-SPECIFIC CHARACTERS
+     - categorical states and presence / absence recorded by the taxonomist (GUI descriptive characters, Record
+       absent, Lost keypoints): a state recorded in no other species, or in at most --max_shared others (named);
+     - meristic characters (counts): the species' range does not overlap that of at least half of the other species
+       scored (n >= 2 in both), with the species named, as in the comparative statements;
+     - measurements whose range overlaps no other species scored (n >= 2 in both): unique values.
+  Every number is the species' own specimen range from specimen_matrix_long.csv; every species list is written by
+  the same function as the comparative statements, so the independent audit (biorag_confabulation_checker_v2) can
+  re-derive each claim.
+
+  python biorag_diagnosis_builder_v1.py --matrix_dir compiled_key_tier --key_tree key_v2/key_tree.json \\
+      --taxon_profile profile.yaml --out_dir diagnoses/
+
+Writes diagnoses.json ({species: {text, items: [{source, feature_id, couplet, text}], words}}), diagnoses.md and
+diagnoses.tsv. biorag_description_refiner_v1 --deterministic_diagnosis diagnoses.json uses the text verbatim in
+place of the model's diagnosis (run_full_pipeline_v2 --deterministic_diagnosis does both).
+"""
+import argparse
+import copy
+import json
+import re
+import sys
+import types
+from pathlib import Path
+
+import numpy as np
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import biorag_feature_policy as pol                       # noqa: E402
+import biorag_description_refiner_v1 as rf                # noqa: E402
+import biorag_key_builder_v1 as kb                        # noqa: E402
+
+VERSION = "1.0"
+
+
+class _NS(types.SimpleNamespace):
+    def __getattr__(self, k):
+        return None
+
+
+def species_values(ev, code, fid):
+    v = ev.long.loc[(ev.long["species"] == code) & (ev.long["feature_id"] == fid), "value"].dropna()
+    return v.to_numpy(dtype=float)
+
+
+def key_items(ev, code, max_group=6):
+    """the primary character of each couplet on the species' path whose side holds at most max_group species (the
+    couplets that separate it from its nearest species; the same rule as the data sheet), with its own range"""
+    items = []
+    if not ev.key:
+        return items
+    max_group = min(max_group, max(1, len(ev.species) // 4))   # scales with the genus: 6 of 29, 1 of 4
+    for r in sorted(ev.key["couplets"], key=lambda r: r["number"]):
+        for side in ("A", "B"):
+            if code not in r[f"{side}_species"] or (r[f"{side}_goto"] != code and
+                                                     len(r[f"{side}_species"]) > max_group):
+                continue
+            chars = sorted(r["characters"], key=lambda c: c.get("role") != "primary")
+            for c in chars:                              # the first character this species was scored for
+                vals = species_values(ev, code, c["feature_id"])
+                if not len(vals):
+                    continue
+                c2 = copy.deepcopy(c)
+                c2[f"{side}_range"] = [float(vals.min()), float(vals.max())]
+                c2[f"{side}_n"] = int(len(vals))
+                thr = c["threshold"]
+                ok = (vals <= thr) if c[f"{side}_operator"] == "<=" else (vals > thr)
+                c2["perfect_separation"] = bool(ok.all())
+                c2.pop(f"{side}_colour_term", None)          # the group's colour word; not this species'
+                txt = kb.template_lead([c2], side)
+                where = f"key couplet {r['number']}{side.lower()}"
+                txt = txt[:-1] + f"; {where})" if txt.endswith(")") else f"{txt} ({where})"
+                m = re.search(r"observed ([^;)]+)", txt)
+                cit = [{"value": m.group(1).strip(), "id": c["feature_id"]}] if m else []
+                items.append({"source": "key", "couplet": f"{r['number']}{side.lower()}",
+                              "feature_id": c["feature_id"], "text": txt, "citations": cit})
+                break
+    return items
+
+
+def categorical_items(ev, code, max_shared):
+    from biorag_coded_states_from_coco_v1 import recorded_states
+    rec = recorded_states(ev.fdict, ev.long)
+    mine = rec.get(code) or {}
+    items = []
+    for (cat, ch), r in sorted(mine.items()):
+        scored = [sp for sp in rec if sp != code and (cat, ch) in rec[sp]]
+        if len(scored) < 2:
+            continue
+        term = pol.structure_info(cat, ev.profile).get("term", cat.replace("_", " "))
+        for st, k in sorted(r["states"].items(), key=lambda kv: -kv[1]):
+            with_ = sorted([sp for sp in scored if st in rec[sp][(cat, ch)]["states"]], key=rf.natural_key)
+            if len(with_) > max_shared or len(with_) >= len(scored) / 2:   # specific: few, and under half
+                continue
+            what = f"{term} {st}" if ch == "presence" else f"{term} {r['label']} {st}"
+            count = f"{k} of {r['n']} specimens"
+            if not with_:
+                tail = f"recorded in no other species, {len(scored)} scored"
+            else:
+                tail = "otherwise recorded only in " + rf.join_names([rf.short_name(ev, x) for x in with_])
+            fid = f"{cat}.coded_{pol_slug(ch)}__{pol_slug(st)}"
+            items.append({"source": "categorical", "feature_id": fid, "category": cat, "character": ch,
+                          "unique": not with_, "text": f"{what} ({count}; {tail})"})
+    items.sort(key=lambda x: (not x["unique"], x["text"]))
+    return items
+
+
+def pol_slug(s):
+    from biorag_coded_states_from_coco_v1 import slug
+    return slug(s)
+
+
+def range_items(ev, code, family_filter, min_share, label_source):
+    """non-overlapping specimen ranges (n >= 2 in both), as in the comparative statements"""
+    fd = ev.fdict
+    s_all = ev.summary[ev.summary["n"] >= 2]
+    out = []
+    for fid, grp in s_all.groupby("feature_id"):
+        if fid not in fd.index or not family_filter(fd.loc[fid]):
+            continue
+        me = grp[grp["species"] == code]
+        if me.empty:
+            continue
+        me = me.iloc[0]
+        others = [x for x in grp["species"] if x != code]
+        if len(others) < 2:
+            continue
+        lower = sorted(grp[(grp["species"] != code) & (grp["max"] < me["min"])]["species"], key=rf.natural_key)
+        higher = sorted(grp[(grp["species"] != code) & (grp["min"] > me["max"])]["species"], key=rf.natural_key)
+        sep = len(lower) + len(higher)
+        if sep < max(1, int(np.ceil(min_share * len(others)))):
+            continue
+        meta = fd.loc[fid]
+        unit = meta["unit"] if isinstance(meta["unit"], str) else ""
+        rng = rf.fmt_range(me, unit)[0].split(" (")[0]
+        count = meta.get("family") == "meristic"
+        up, down = ("more", "fewer") if count else ("greater", "smaller")
+        if meta.get("family") == "colour":
+            up, down = "paler (higher L*)", "darker (lower L*)"
+        # the data sheet's own wording (species_list_phrase), so the audit re-derives the same statement
+        clauses = [f"{word} than in {rf.species_list_phrase(ev, group, others)}"
+                   for group, word in ((lower, up), (higher, down)) if group]
+        term = meta["label"]
+        out.append({"source": label_source, "feature_id": fid, "separated": sep, "of": len(others),
+                    "text": f"{term} {rng} (n = {int(me['n'])}): " + "; ".join(clauses),
+                    "citations": [{"value": rng, "id": fid}]})
+    out.sort(key=lambda x: (-x["separated"] / max(x["of"], 1), x["feature_id"]))
+    return out
+
+
+def build(ev, code, max_shared=2, meristic_share=0.5, max_unique_measurements=3, key_max_group=6):
+    keys = key_items(ev, code, key_max_group)
+    cats = categorical_items(ev, code, max_shared)
+    # a categorical state already used by the key is not repeated: its 'recorded in no other species' note is
+    # appended to the key item instead
+    key_cc = {}
+    for it in keys:
+        if it["feature_id"] in ev.fdict.index and ev.fdict.loc[it["feature_id"]].get("family") == "coded_state":
+            key_cc[(ev.fdict.loc[it["feature_id"]]["category"], ev.fdict.loc[it["feature_id"]].get("character"))] = it
+    rest = []
+    for c in cats:
+        k = key_cc.get((c["category"], c["character"]))
+        if k is not None:
+            tail = c["text"][c["text"].rfind("; ") + 2:-1]
+            k["text"] = k["text"][:-1] + f"; {tail})"
+        else:
+            rest.append(c)
+    used = {it["feature_id"] for it in keys}
+    meris = [m for m in range_items(ev, code, lambda m: m.get("family") == "meristic", meristic_share, "meristic")
+             if m["feature_id"] not in used]
+    uniq = [m for m in range_items(ev, code, lambda m: m.get("tier") == pol.TIER_KEY and
+                                   m.get("family") not in ("meristic", "coded_state"), 1.0, "unique_measurement")
+            if m["feature_id"] not in used][:max_unique_measurements]
+    n_other = len(ev.species) - 1
+    name = ev.display(code)
+    parts = []
+    if keys:
+        parts.append(f"{name} is separated from its nearest species in the key by: "
+                     + "; ".join(it["text"] for it in keys) + ".")
+    lin = rest + meris + uniq
+    if lin:                                              # one sentence per character (lists contain ';')
+        parts.append(("Further diagnostic characters: " if keys else
+                      f"{name} differs from the other {n_other} species examined by the following characters: ")
+                     + " ".join(f"{it['text'][0].upper()}{it['text'][1:]}." for it in lin))
+    text = " ".join(parts) or f"No character separates {name} from the other species in the current matrix."
+    cits = [c for it in keys + lin for c in it.get("citations", [])]
+    return {"text": text, "items": keys + lin, "citations": cits, "words": len(text.split())}
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--matrix_dir", required=True)
+    ap.add_argument("--key_tree", required=True, help="key_tree.json from biorag_key_builder_v1")
+    ap.add_argument("--taxon_profile", required=True)
+    ap.add_argument("--species", nargs="*", default=None, help="only these species codes (default: all)")
+    ap.add_argument("--max_shared", type=int, default=2,
+                    help="a categorical state counts as lineage-specific when recorded in at most this many other "
+                         "species (named); 0 = only states recorded in no other species (default 2)")
+    ap.add_argument("--meristic_share", type=float, default=0.5,
+                    help="a count is reported when its range does not overlap this share of the other species (0.5)")
+    ap.add_argument("--max_unique_measurements", type=int, default=3)
+    ap.add_argument("--key_max_group", type=int, default=6,
+                    help="use a key couplet when the species' side holds at most this many species (default 6)")
+    ap.add_argument("--out_dir", required=True)
+    a = ap.parse_args()
+    profile = pol.load_taxon_profile(a.taxon_profile)
+    ev = rf.Evidence(_NS(matrix_dir=a.matrix_dir, key_tree=a.key_tree), profile)
+    out = Path(a.out_dir); out.mkdir(parents=True, exist_ok=True)
+    res = {}
+    for code in (a.species or ev.species):
+        res[code] = build(ev, code, a.max_shared, a.meristic_share, a.max_unique_measurements, a.key_max_group)
+    meta = {"version": VERSION, "matrix_dir": str(a.matrix_dir), "key_tree": str(a.key_tree),
+            "max_shared": a.max_shared, "meristic_share": a.meristic_share}
+    json.dump({"_meta": meta, **res}, open(out / "diagnoses.json", "w"), indent=1, ensure_ascii=False)
+    with open(out / "diagnoses.md", "w") as f:
+        f.write("# Diagnoses (deterministic: key path + lineage-specific characters)\n\n")
+        for code, r in res.items():
+            f.write(f"**{ev.display(code)}.** {r['text']}\n\n")
+    with open(out / "diagnoses.tsv", "w") as f:
+        f.write("species\tsource\tcouplet\tfeature_id\ttext\n")
+        for code, r in res.items():
+            for it in r["items"]:
+                f.write(f"{code}\t{it['source']}\t{it.get('couplet', '')}\t{it['feature_id']}\t{it['text']}\n")
+    n_k = sum(any(i["source"] == "key" for i in r["items"]) for r in res.values())
+    n_l = sum(any(i["source"] != "key" for i in r["items"]) for r in res.values())
+    print(f"{len(res)} diagnoses: {n_k} with key characters, {n_l} with lineage-specific characters; median "
+          f"{int(np.median([r['words'] for r in res.values()]))} words -> {out / 'diagnoses.md'}")
+
+
+if __name__ == "__main__":
+    main()

{descriptron_core-2.7.8 → descriptron_core-2.8.0}/src/descriptron_core/tools/color_phenomics_homology_v2_1.py

@@ -607,7 +607,8 @@ def combine_pattern_and_hue(X_pattern, X_hue, hue_weight=1.0):
 ###############################################################################
 
 def process_specimen(fn_gpa, specimen_contour, mean_shape, grid_info,
-                     image_dir, mask_lookup, ctrl_step=5, color_bins=8):
+                     image_dir, mask_lookup, ctrl_step=5, color_bins=8,
+                     occluder_lookup=None, coverage_out=None):
     """Process one specimen: TPS warp grid, extract per-cell color."""
 
     # resolve image path
@@ -632,6 +633,16 @@ def process_specimen(fn_gpa, specimen_contour, mean_shape, grid_info,
         logging.warning(f"  No mask for {orig_fn}")
         return None
 
+    # 2026-10-12 (opt-in): occluders removed -> features from visible pixels only; see homology_grid_ext.py
+    _expected = None
+    if occluder_lookup is not None:
+        from homology_grid_ext import visible_and_expected
+        fg_mask, _expected = visible_and_expected(fg_mask, occluder_lookup.get(orig_fn), decode_mask)
+        if fg_mask.sum() < 100:
+            logging.warning(f"  Too little visible foreground for {orig_fn}")
+            return None
+    _cov = []
+
     fg_bool = fg_mask > 0
 
     # specimen-wide L*a*b* means for relative color
@@ -693,8 +704,12 @@ def process_specimen(fn_gpa, specimen_contour, mean_shape, grid_info,
         for c in range(n_cols):
             if not cell_inside[r, c]:
                 cell_features.append([np.nan] * n_feats)
+                _cov.append(np.nan)
                 continue
             cm = build_cell_mask(warped_ints, r, c, h, w, fg_mask)
+            if _expected is not None:
+                _e = build_cell_mask(warped_ints, r, c, h, w, _expected).sum()
+                _cov.append(cm.sum() / _e if _e else np.nan)
             feats = extract_cell_color_features(img_lab, cm,
                                                  specimen_lab_means,
                                                  n_color_bins=color_bins)
@@ -704,6 +719,8 @@ def process_specimen(fn_gpa, specimen_contour, mean_shape, grid_info,
                 cell_features.append(feats)
 
     fv = np.array(cell_features).flatten()
+    if coverage_out is not None and _expected is not None:
+        coverage_out[fn_gpa] = np.array(_cov, dtype=float)
     # v2: 6-tuple now — trailing hue_feats is the specimen-level block.
     return (fn_gpa, fv, img, fg_mask, warped_ints, hue_feats)
 
@@ -824,7 +841,18 @@ def parse_args():
     p.add_argument('--workers', type=int, default=0)
     p.add_argument('--n_clusters', type=int, default=0,
                    help='Number of clusters (0 = auto via silhouette score)')
-    p.add_argument('--viz_specimens', type=int, default=5)
+    p.add_argument('--viz_specimens', type=int, default=5,
+                   help='Number of per-specimen grid overlays to save; -1 = every specimen')
+    # 2026-10-12 opt-in additions (homology_grid_ext.py); defaults keep the previous behaviour
+    p.add_argument('--grid_ext', type=int, default=0,
+                   help='Extend the grid by N cells on every side and keep cells that overlap the outline '
+                        '(>= --min_cell_overlap) instead of requiring the cell centre inside. 0 = previous grid')
+    p.add_argument('--min_cell_overlap', type=float, default=0.02,
+                   help='With --grid_ext: minimum fraction of a cell inside the consensus outline (default 0.02)')
+    p.add_argument('--occluder_categories', type=str, default='',
+                   help='Comma-separated COCO categories that hide the structure (e.g. scape,antenna,leg), or "auto" '
+                        '(categories flagged as occluders in the COCO file, GUI v86). They are removed from the mask '
+                        '(visible pixels only) and a per-cell visible fraction is written. Nested parts are never used')
     p.add_argument('--umap_n_neighbors', type=int, default=15)
     p.add_argument('--color_bins', type=int, default=8,
                    help='Number of bins for within-cell color entropy '
@@ -877,8 +905,19 @@ def main():
                  f"{len(mask_lookup)} masks available")
 
     # --- Build grid ---
-    grid_info = create_grid_inside_contour(
-        mean_shape, n_rows=args.grid_rows, n_cols=args.grid_cols)
+    if args.grid_ext > 0:
+        from homology_grid_ext import create_grid_extended
+        grid_info = create_grid_extended(mean_shape, n_rows=args.grid_rows, n_cols=args.grid_cols,
+                                         ext=args.grid_ext, min_overlap=args.min_cell_overlap)
+    else:
+        grid_info = create_grid_inside_contour(
+            mean_shape, n_rows=args.grid_rows, n_cols=args.grid_cols)
+    occluder_lookup, coverage = None, None
+    if args.occluder_categories.strip():
+        from homology_grid_ext import build_occluder_lookup
+        occluder_lookup = build_occluder_lookup(args.json, args.occluder_categories.split(','),
+                                                target=cat_name)  # never the structure itself or its nested parts
+        coverage = {}
 
     ctrl_step = args.landmark_step
     n_ctrl = len(range(0, mean_shape.shape[0], ctrl_step))
@@ -898,7 +937,8 @@ def main():
         return process_specimen(fn, contour, mean_shape, grid_info,
                                 args.image_dir, mask_lookup,
                                 ctrl_step=ctrl_step,
-                                color_bins=color_bins)
+                                color_bins=color_bins,
+                                occluder_lookup=occluder_lookup, coverage_out=coverage)
 
     items = list(specimens.items())
     results = []
@@ -928,9 +968,14 @@ def main():
     viz_dir = os.path.join(args.output_dir, 'grid_overlays')
     os.makedirs(viz_dir, exist_ok=True)
     # v2: 6-tuple — use *_rest to stay robust if tuple ever grows.
-    for fn, fv, img, mask, wi, *_rest in results[:args.viz_specimens]:
+    n_viz = len(results) if args.viz_specimens < 0 else args.viz_specimens
+    for fn, fv, img, mask, wi, *_rest in results[:n_viz]:
         viz_warped_grid_on_specimen(fn, img, mask, wi, grid_info, viz_dir, cat_name)
-    logging.info(f"Saved {min(len(results), args.viz_specimens)} grid overlays")
+    logging.info(f"Saved {min(len(results), n_viz)} grid overlays")
+    if coverage is not None:
+        from homology_grid_ext import write_coverage_csv
+        write_coverage_csv(os.path.join(args.output_dir, f"color_homology_visible_fraction_{cat_name}.csv"), coverage, grid_info)
+        logging.info(f"Saved per-cell visible fraction for {len(coverage)} specimens")
 
     # --- Assemble feature matrix ---
     fns = [r[0] for r in results]