wormscore 0.6.1__py3-none-any.whl

wormscore/__init__.py

@@ -0,0 +1,3 @@
+"""wormscore: blind scoring and fluorescence measurement of neurons in confocal stacks."""
+
+__version__ = "0.6.1"

wormscore/cli.py

@@ -0,0 +1,81 @@
+"""Command line: wormscore <command> <project folder> ..."""
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from . import __version__
+
+
+def main(argv=None):
+    p = argparse.ArgumentParser(prog="wormscore", description="Blind scoring and measurement of neurons in confocal stacks.")
+    p.add_argument("--version", action="version", version=f"wormscore {__version__}")
+    sub = p.add_subparsers(dest="cmd", required=True)
+
+    s = sub.add_parser("init", help="create a project folder with a config.json to edit")
+    s.add_argument("project")
+    s.add_argument("--name", default="My project")
+    s = sub.add_parser("demo", help="create a demo project with simulated worms")
+    s.add_argument("project")
+    s.add_argument("--n", type=int, default=10, help="simulated worms per genotype")
+    s.add_argument("--real", action="store_true",
+                   help="download 4 real worm heads (about 150 MB, CC BY 4.0) instead of simulating")
+    s = sub.add_parser("copy", help="copy a data folder and verify every file by SHA-256")
+    s.add_argument("source")
+    s.add_argument("destination")
+    s = sub.add_parser("inventory", help="list every file and image in the project's sources")
+    s.add_argument("project")
+    s = sub.add_parser("prepare", help="build the blind image set, key and automatic measurements")
+    s.add_argument("project")
+    s.add_argument("--reassign-codes", action="store_true", help="new codes (only allowed before any scoring)")
+    s.add_argument("--add-new-images", dest="add", action="store_true",
+                   help="code only images not yet in the key; existing codes are kept")
+    s = sub.add_parser("app", help="open the scoring app")
+    s.add_argument("project")
+    s.add_argument("--port", type=int, default=8765)
+    s.add_argument("--no-browser", action="store_true")
+    s = sub.add_parser("unblind", help="join scores, measurements and the key (after scoring)")
+    s.add_argument("project")
+    s.add_argument("--i-have-finished-scoring", dest="finished", action="store_true")
+    s.add_argument("--anyway", action="store_true", help="unblind below the planned minimum number of scores")
+
+    a = p.parse_args(argv)
+    try:
+        if a.cmd == "init":
+            from .config import template
+            proj = Path(a.project)
+            proj.mkdir(parents=True, exist_ok=True)
+            cfg = proj / "config.json"
+            if cfg.exists():
+                sys.exit(f"{cfg} already exists; not overwriting.")
+            cfg.write_text(json.dumps(template(a.name), indent=2), encoding="utf-8")
+            print(f"Created {cfg}. Edit 'sources' to point at your images, then run 'wormscore inventory {proj}'.")
+        elif a.cmd == "demo":
+            if a.real:
+                from .realdemo import make_real_demo
+                make_real_demo(a.project)
+            else:
+                from .demo import make_demo
+                make_demo(a.project, n_per_group=a.n)
+            print(f"Next: wormscore inventory {a.project} && wormscore prepare {a.project} && wormscore app {a.project}")
+        elif a.cmd == "copy":
+            from .copydata import copy_verified
+            copy_verified(a.source, a.destination)
+        elif a.cmd == "inventory":
+            from .pipeline import inventory
+            inventory(a.project)
+        elif a.cmd == "prepare":
+            from .pipeline import prepare
+            prepare(a.project, reassign=a.reassign_codes, add=a.add)
+        elif a.cmd == "app":
+            from .server import serve
+            serve(a.project, port=a.port, open_browser=not a.no_browser)
+        elif a.cmd == "unblind":
+            from .unblind import unblind
+            unblind(a.project, finished=a.finished, anyway=a.anyway)
+    except FileNotFoundError as exc:
+        sys.exit(str(exc))
+
+
+if __name__ == "__main__":
+    main()

wormscore/config.py

@@ -0,0 +1,265 @@
+"""Project configuration.
+
+A project is a folder holding config.json plus everything wormscore writes
+(outputs/, blind/, key/, scoring/, log.md). Every analysis parameter lives in
+config.json; anything left out falls back to DEFAULTS below. Each run copies the
+full, merged configuration into log.md so the analysis can be repeated.
+"""
+import copy
+import fnmatch
+import json
+from pathlib import Path
+
+DEFAULTS = {
+    "project_name": "Untitled project",
+
+    # Folders of images. path = where the files are read from (ideally a read-only copy);
+    # original_path = where they first came from, recorded in the key for traceability.
+    # genotypes: first matching pattern (on the path relative to the source folder) wins.
+    # exclude (optional): [{"match": pattern, "reason": text}]; matching images are left out of
+    # the blind set and listed in outputs/excluded.csv with the reason.
+    # note (optional): a label such as "collected by others" shown with an asterisk next to
+    # this source in the results tables.
+    "sources": [],
+
+    # Which channel is the reporter. Identified from metadata only, never guessed.
+    "marker": {
+        "label": "GFP",
+        "excitation_nm": [488],
+        "emission_band_start_nm": [490, 530],
+        "dye_names": ["EGFP", "GFP", "Leica/EGFP"],
+        "channel_name_prefixes": ["GFP", "EGFP"],
+        "leica_lut": "Green",
+    },
+    "brightfield_channel_names": ["BF", "Brightfield", "DIC", "Trans", "T-PMT", "T PMT", "TL"],
+
+    "inclusion": {
+        "min_objective_magnification": 40,
+        "min_z_extent_um": 9.0,
+        "max_time_points": 1,
+        "same_field_tolerance_um": 5.0,
+    },
+    # For byte-identical copies of an image, keep the one whose path contains the first
+    # matching text here; otherwise the shortest path.
+    "duplicate_preference": [],
+
+    "display": {
+        "display_um_per_px": 0.20,
+        "frame_px": 940,
+        "brightfield_contrast_percentiles": [1, 99],
+    },
+    # Highest value each microscope can record, by system name. Missing systems use the
+    # bit depth from the file metadata (255 for 8-bit, 65535 for 16-bit).
+    "saturation_value": {},
+
+    "measurement": {
+        "median_filter_px": 3,
+        "smooth_um": 1.5,
+        "min_separation_um": 6.0,
+        "spot_ring_um": [6.0, 10.0],
+        "max_radius_um": 8.0,
+        "min_object_area_um2": 4.0,
+        "n_candidates": 4,
+        "background_ring_gap_um": 2.0,
+        "background_ring_width_um": 4.0,
+        "streak_jump_fraction": 0.04,
+        "other_channel_qc": True,
+    },
+
+    "scoring": {
+        "neurons": ["Neuron 1", "Neuron 2"],
+        "calls": ["normal", "dim", "missing"],
+        "defect_calls": ["dim", "missing"],
+        "sides": ["left", "right"],
+        "count_question": {"label": "Marker-positive cell bodies seen", "options": ["0", "1", "2", "3+"]},
+        "neuron_questions": [],
+        "unusable_reasons": [
+            "wrong region of the animal",
+            "out of focus or moving",
+            "wrong stage",
+            "looks like a different marker",
+            "damaged animal",
+            "other (see notes)",
+        ],
+        "rules": [
+            "Normal: cell body clearly visible at the expected brightness.",
+            "Dim: visible but weaker than controls imaged at the same settings.",
+            "Missing: no cell body in any plane.",
+            "Score each neuron separately. The animal is the unit for statistics.",
+        ],
+    },
+
+    # reference_genotype: the control that each other genotype is tested against (Fisher's
+    # exact, within each microscope). Empty: every pair of genotypes is tested.
+    # question_outcomes (optional): tables built from the per-neuron questions, for example
+    #   {"name": "process defect", "questions": ["q1", "q2"], "defect_answers": ["no"],
+    #    "unclear_answers": ["can't tell"], "unclear": "exclude_neuron", "not_visible_calls": ["missing"]}
+    # A neuron is defective if any listed question has a defect answer. Neurons whose call is in
+    # not_visible_calls are not assessed; animals with no assessed neuron are left out.
+    # unclear: "exclude_neuron" drops a neuron with an unclear answer, "exclude_animal" drops the animal.
+    # interaction (optional): {"single_a": genotype, "single_b": genotype, "double": genotype,
+    #   "model": "product" | "additive" | "min", "outcome": "calls" or a question_outcomes name}.
+    # Chosen before scoring; prepare records it in log.md and unblinding tests only that model.
+    # secondary_defect_calls (optional): a second "affected" definition (for example dim or
+    #   missing), reported with 95% intervals within each microscope only; tested (Fisher's
+    #   exact, within one microscope) only if test_secondary is true.
+    # allow_cross_microscope (optional): measures ("calls" or a question_outcomes name) whose
+    #   tests may compare genotypes imaged on different microscopes; animals of each genotype
+    #   are then pooled over microscopes and every table says which microscopes contributed.
+    # alternative: "two-sided" (default), "greater" or "less" for the binomial and Fisher tests.
+    #   One-sided Fisher tests need comparisons, which give the direction (genotype_a vs _b).
+    # comparisons (optional): the planned [genotype_a, genotype_b] pairs for Fisher's exact
+    #   tests; Bonferroni counts only these.
+    # These settings and the interaction test form the analysis plan: prepare records it in
+    # log.md, and unblinding runs no test if it changed after the first score was saved.
+    # cross_microscope_reason and methods_notes (optional) only add text to methods_text.md.
+    "unblinding": {"min_scored_worms": 30, "reference_genotype": "", "question_outcomes": [],
+                   "interaction": {}, "secondary_defect_calls": [], "test_secondary": False,
+                   "allow_cross_microscope": [], "alternative": "two-sided", "comparisons": [],
+                   "cross_microscope_reason": "", "methods_notes": []},
+
+    # Used only for the Methods text written at unblinding. Left empty, it says [name].
+    # imager_knew_genotype: "yes", "no" or "" (unknown).
+    "blinding": {"coded_by": "", "scored_by": "", "imager_knew_genotype": ""},
+}
+
+
+def merge(base, over):
+    out = copy.deepcopy(base)
+    for k, v in over.items():
+        if isinstance(v, dict) and isinstance(out.get(k), dict):
+            out[k] = merge(out[k], v)
+        else:
+            out[k] = v
+    return out
+
+
+def load(project):
+    path = Path(project) / "config.json"
+    if not path.exists():
+        raise FileNotFoundError(f"No config.json in {project}. Run 'wormscore init {project}' first.")
+    with open(path, encoding="utf-8") as fh:
+        return merge(DEFAULTS, json.load(fh))
+
+
+def matches(pattern, rel_path):
+    rel = rel_path.replace("\\", "/")
+    return fnmatch.fnmatch(rel, pattern) or fnmatch.fnmatch(Path(rel).name, pattern)
+
+
+def genotype_for(source, rel_path):
+    for rule in source.get("genotypes", []):
+        if matches(rule["match"], rel_path):
+            return rule["genotype"]
+    return source.get("default_genotype", "UNKNOWN")
+
+
+def exclude_rule_for(cfg, source_name, rel_path):
+    """The config exclude rule matching this image, or None."""
+    for src in cfg["sources"]:
+        if src.get("name") == source_name:
+            for rule in src.get("exclude", []):
+                if matches(rule["match"], rel_path):
+                    return rule
+    return None
+
+
+INTERACTION_MODELS = ("product", "additive", "min")
+
+
+def interaction(cfg):
+    """The interaction test settings with defaults filled in, or {} if none is configured."""
+    ia = cfg["unblinding"]["interaction"]
+    return {"model": "product", "outcome": "calls", "expected_direction": "", **ia} if ia else {}
+
+
+ALTERNATIVES = ("two-sided", "greater", "less")
+PLAN_KEYS = ("secondary_defect_calls", "test_secondary", "allow_cross_microscope", "alternative", "comparisons",
+             "question_outcomes")
+
+
+def analysis_plan(cfg):
+    """Everything that decides which tests run, or {} if only defaults are used (then nothing
+    is recorded or locked)."""
+    un = cfg["unblinding"]
+    plan = {k: un[k] for k in PLAN_KEYS} | {"interaction": interaction(cfg),
+                                            "defect_calls": cfg["scoring"]["defect_calls"]}
+    used = un["secondary_defect_calls"] or un["allow_cross_microscope"] or un["comparisons"] \
+        or un["alternative"] != "two-sided" or un["interaction"] or un["question_outcomes"]
+    return plan if used else {}
+
+
+def check(cfg):
+    """Stop with a plain message if the optional results settings do not fit the scoring setup."""
+    un, sc = cfg["unblinding"], cfg["scoring"]
+    questions = {q["id"]: q for q in sc["neuron_questions"]}
+    names = []
+    for oc in un["question_outcomes"]:
+        name = oc.get("name", "")
+        if not name or name in names or name == "calls":
+            raise SystemExit(f"question_outcomes: each entry needs a unique 'name' other than 'calls' (got {name!r}).")
+        names.append(name)
+        if not oc.get("questions") or not oc.get("defect_answers"):
+            raise SystemExit(f"question_outcomes '{name}': set 'questions' and 'defect_answers'.")
+        for qid in oc["questions"]:
+            if qid not in questions:
+                raise SystemExit(f"question_outcomes '{name}': no neuron question with id {qid!r} in scoring.neuron_questions.")
+            unknown = set(oc["defect_answers"]) | set(oc.get("unclear_answers", []))
+            unknown -= set(questions[qid]["options"])
+            if unknown:
+                raise SystemExit(f"question_outcomes '{name}': {sorted(unknown)} are not options of question {qid!r}.")
+        if oc.get("unclear", "exclude_neuron") not in ("exclude_neuron", "exclude_animal"):
+            raise SystemExit(f"question_outcomes '{name}': 'unclear' must be exclude_neuron or exclude_animal.")
+    ia = un["interaction"]
+    if ia:
+        for k in ("single_a", "single_b", "double"):
+            if not ia.get(k):
+                raise SystemExit(f"interaction: set '{k}' to a genotype name.")
+        if ia.get("model", "product") not in INTERACTION_MODELS:
+            raise SystemExit(f"interaction: model must be one of {', '.join(INTERACTION_MODELS)}.")
+        if ia.get("outcome", "calls") not in ["calls"] + names:
+            raise SystemExit("interaction: outcome must be 'calls' or a question_outcomes name.")
+        if ia.get("expected_direction", "") not in ("", "higher", "lower"):
+            raise SystemExit("interaction: expected_direction must be 'higher', 'lower' or left out.")
+    for key in ("defect_calls",):
+        if set(sc[key]) - set(sc["calls"]):
+            raise SystemExit(f"scoring.{key}: {sorted(set(sc[key]) - set(sc['calls']))} are not in scoring.calls.")
+    if set(un["secondary_defect_calls"]) - set(sc["calls"]):
+        raise SystemExit("secondary_defect_calls: "
+                         f"{sorted(set(un['secondary_defect_calls']) - set(sc['calls']))} are not in scoring.calls.")
+    for m in un["allow_cross_microscope"]:
+        if m == "secondary":
+            raise SystemExit("allow_cross_microscope: the secondary measure is compared only within one microscope.")
+        if m not in ["calls"] + names:
+            raise SystemExit(f"allow_cross_microscope: {m!r} is not 'calls' or a question_outcomes name.")
+    if un["alternative"] not in ALTERNATIVES:
+        raise SystemExit(f"alternative must be one of {', '.join(ALTERNATIVES)}.")
+    if any(not isinstance(c, (list, tuple)) or len(c) != 2 for c in un["comparisons"]):
+        raise SystemExit("comparisons: each entry must be a pair [genotype_a, genotype_b].")
+    if un["alternative"] != "two-sided" and not un["comparisons"]:
+        raise SystemExit("A one-sided alternative needs 'comparisons', so each test has a direction "
+                         "(genotype_a vs genotype_b).")
+
+
+def template(name="My project"):
+    return {
+        "project_name": name,
+        "sources": [
+            {
+                "name": "my images",
+                "path": "C:/path/to/read-only/copy/of/images",
+                "original_path": "where the files first came from (for the record)",
+                "genotypes": [
+                    {"match": "*control*", "genotype": "control"},
+                    {"match": "*mutant*", "genotype": "mutant"},
+                ],
+                "default_genotype": "UNKNOWN",
+            }
+        ],
+        "marker": DEFAULTS["marker"],
+        "scoring": {
+            "neurons": DEFAULTS["scoring"]["neurons"],
+            "calls": DEFAULTS["scoring"]["calls"],
+            "defect_calls": DEFAULTS["scoring"]["defect_calls"],
+        },
+    }

wormscore/copydata.py

@@ -0,0 +1,33 @@
+"""Copy a data folder and prove the copy is exact.
+
+Every file (hidden ones included) is copied with its timestamp, then compared with the
+original by SHA-256. Copies are set read-only. Nothing in the source is changed.
+"""
+import shutil
+import stat
+from pathlib import Path
+
+from .readers import sha256
+
+
+def copy_verified(src, dst, log=print):
+    src, dst = Path(src), Path(dst)
+    if not src.is_dir():
+        raise SystemExit(f"{src} is not a folder.")
+    files = sorted(p for p in src.rglob("*") if p.is_file())
+    bad = []
+    for n, p in enumerate(files, start=1):
+        target = dst / p.relative_to(src)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        if target.exists():
+            target.chmod(stat.S_IWRITE | stat.S_IREAD)
+        shutil.copy2(p, target)
+        if sha256(p) != sha256(target):
+            bad.append(str(p))
+        target.chmod(stat.S_IREAD)
+        if n % 25 == 0 or n == len(files):
+            log(f"copied and checked {n}/{len(files)}")
+    if bad:
+        raise SystemExit(f"{len(bad)} files did not match after copying: {bad[:5]}")
+    log(f"All {len(files)} files copied to {dst} and verified by SHA-256. Copies are read-only.")
+    return len(files)

wormscore/demo.py

@@ -0,0 +1,96 @@
+"""A demo project with simulated worms, so anyone can try wormscore without real data.
+
+Each simulated image is an OME-TIFF z-stack of a worm head with two marker-positive
+cell bodies (each with a faint dendrite), gut-like autofluorescence, camera offset and
+noise, plus a brightfield channel. Controls have two normal cells; mutants have each
+cell normal, dim or missing at random. The truth is written to demo_truth.csv so the
+measurements can be checked after unblinding. No real data is involved.
+"""
+import csv
+import json
+from pathlib import Path
+
+import numpy as np
+import tifffile
+from scipy import ndimage
+
+PX_UM, Z_UM, NZ, SIZE = 0.25, 1.0, 16, 384
+LEVELS = {"normal": 3000.0, "dim": 700.0, "missing": 0.0}
+
+
+def _worm(rng, cells):
+    """One simulated head: returns (marker stack, brightfield stack) as uint16 (z, y, x)."""
+    yy, xx = np.mgrid[:SIZE, :SIZE].astype(float)
+    angle = rng.uniform(0, np.pi)
+    c, s = np.cos(angle), np.sin(angle)
+    u = ((xx - SIZE / 2) * c + (yy - SIZE / 2) * s) * PX_UM     # um along the body (nose at negative u)
+    v = (-(xx - SIZE / 2) * s + (yy - SIZE / 2) * c) * PX_UM    # um across the body
+    half_width = np.clip(8 + 0.15 * (u + 48), 6, 22)            # head widens away from the nose
+    body = (np.abs(v) < half_width) & (u > -45)
+    zz = np.arange(NZ)[:, None, None].astype(float)
+
+    marker = np.full((NZ, SIZE, SIZE), 100.0)                   # camera offset
+    gut = (rng.random((SIZE, SIZE)) < 0.004) & body & (u > 15)
+    gut_img = np.zeros((SIZE, SIZE))
+    gut_img[gut] = rng.uniform(150, 400, gut.sum())
+    gut_img = ndimage.gaussian_filter(gut_img, 1.5) * 9
+    marker += gut_img[None] * np.exp(-((zz - NZ / 2) ** 2) / 30)
+    marker += body[None] * 60.0                                  # tissue autofluorescence
+
+    for k, level in enumerate(cells):
+        cu = rng.uniform(-6, 2)                                  # um along the body
+        cv = (-1) ** k * rng.uniform(3, 7)                       # um either side of the midline
+        cz = rng.uniform(5, NZ - 6)
+        r2 = (u - cu) ** 2 + (v - cv) ** 2
+        soma = np.exp(-r2 / (2 * 1.4 ** 2))[None] * np.exp(-((zz - cz) ** 2) / (2 * 1.6 ** 2))
+        marker += LEVELS[level] * soma
+        if level != "missing":                                   # faint dendrite toward the nose
+            along = (u < cu - 2) & (u > -44) & (np.abs(v - cv * (u + 44) / (cu + 44)) < 0.4)
+            marker += 0.15 * LEVELS[level] * along[None] * np.exp(-((zz - cz) ** 2) / 8)
+
+    marker = rng.poisson(np.clip(marker, 0, None)).astype(np.float64)
+    bf = 1800 + 600 * body + rng.normal(0, 60, (SIZE, SIZE))
+    bf = np.repeat(bf[None], NZ, axis=0) + rng.normal(0, 30, (NZ, SIZE, SIZE))
+    return np.clip(marker, 0, 65535).astype(np.uint16), np.clip(bf, 0, 65535).astype(np.uint16)
+
+
+def make_demo(project, n_per_group=10, seed=7, log=print):
+    project = Path(project)
+    data = project / "data"
+    data.mkdir(parents=True, exist_ok=True)
+    rng = np.random.default_rng(seed)
+    truth = []
+    for group in ("control", "mutant"):
+        for i in range(1, n_per_group + 1):
+            if group == "control":
+                cells = ["normal", "normal"]
+            else:
+                cells = list(rng.choice(["normal", "dim", "missing"], size=2, p=[0.4, 0.35, 0.25]))
+            marker, bf = _worm(rng, cells)
+            name = f"demo_{group}_{i:02d}.ome.tif"
+            tifffile.imwrite(data / name, np.stack([marker, bf]), photometric="minisblack", ome=True,
+                             metadata={"axes": "CZYX", "PhysicalSizeX": PX_UM, "PhysicalSizeY": PX_UM,
+                                       "PhysicalSizeZ": Z_UM, "Channel": {"Name": ["GFP", "Brightfield"]}})
+            truth.append({"file": name, "genotype": group, "cell_1": cells[0], "cell_2": cells[1]})
+    (data / "notes.txt").write_text("Simulated data made by 'wormscore demo'. Not real images.\n")
+    with open(project / "demo_truth.csv", "w", newline="") as fh:
+        w = csv.DictWriter(fh, fieldnames=list(truth[0]))
+        w.writeheader()
+        w.writerows(truth)
+    cfg = {
+        "project_name": "Demo (simulated worms)",
+        "sources": [{"name": "simulated", "path": str(data), "original_path": str(data),
+                     "genotypes": [{"match": "*control*", "genotype": "control"},
+                                   {"match": "*mutant*", "genotype": "mutant"}]}],
+        "inclusion": {"min_z_extent_um": 9.0},
+        "scoring": {
+            "neurons": ["Neuron 1", "Neuron 2"],
+            "neuron_questions": [{"id": "dendrite", "label": "dendrite visible?", "options": ["yes", "no", "can't tell"]}],
+            "rules": ["Simulated data. Normal: bright cell body. Dim: faint but visible. Missing: nothing there.",
+                      "Score each neuron separately; the animal is the unit for statistics."],
+        },
+        "unblinding": {"min_scored_worms": 5},
+    }
+    (project / "config.json").write_text(json.dumps(cfg, indent=2), encoding="utf-8")
+    log(f"Demo project with {2 * n_per_group} simulated worms written to {project}")
+    return project