omicsmeta 0.1.0__py3-none-any.whl

omicsmeta/__init__.py

@@ -0,0 +1,7 @@
+"""Tools for harmonizing public omics metadata."""
+
+from omicsmeta._version import __version__
+from omicsmeta.core.harmonizer import HarmonizationResult, Harmonizer
+
+__all__ = ["HarmonizationResult", "Harmonizer", "__version__"]
+

omicsmeta/_version.py

@@ -0,0 +1,2 @@
+__version__ = "0.1.0"
+

omicsmeta/benchmark.py

@@ -0,0 +1,199 @@
+"""Benchmark helpers for known-answer metadata harmonization fixtures."""
+
+from __future__ import annotations
+
+import csv
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+from omicsmeta.core.harmonizer import Harmonizer
+
+
+@dataclass(frozen=True)
+class BenchmarkMetrics:
+    """Precision/recall/F1 summary for ontology mappings."""
+
+    true_positive: int
+    false_positive: int
+    false_negative: int
+    precision: float
+    recall: float
+    f1: float
+
+    def as_dict(self) -> dict[str, object]:
+        return {
+            "true_positive": self.true_positive,
+            "false_positive": self.false_positive,
+            "false_negative": self.false_negative,
+            "precision": self.precision,
+            "recall": self.recall,
+            "f1": self.f1,
+        }
+
+
+MappingKey = tuple[str, str, str]
+
+
+def benchmark_file(
+    input_path: str | Path,
+    truth_path: str | Path,
+    *,
+    input_type: str = "tabular",
+    confidence_threshold: float = 0.70,
+) -> dict[str, object]:
+    """Run harmonization and compare accepted direct mappings to truth."""
+
+    result = Harmonizer(confidence_threshold=confidence_threshold).from_file(str(input_path), file_type=input_type)
+    observed = _observed_mappings(result.harmonized)
+    expected = _truth_mappings(truth_path)
+    overall = _metrics(observed, expected)
+    fields = sorted({field for _, field, _ in observed | expected})
+    by_field = {
+        field: _metrics(
+            {mapping for mapping in observed if mapping[1] == field},
+            {mapping for mapping in expected if mapping[1] == field},
+        ).as_dict()
+        for field in fields
+    }
+
+    return {
+        "input_path": str(input_path),
+        "truth_path": str(truth_path),
+        "confidence_threshold": confidence_threshold,
+        "overall": overall.as_dict(),
+        "by_field": by_field,
+        "observed_count": len(observed),
+        "expected_count": len(expected),
+    }
+
+
+def benchmark_suite(
+    manifest_path: str | Path,
+    *,
+    confidence_threshold: float = 0.70,
+) -> dict[str, object]:
+    """Run a TSV manifest of known-answer benchmark cases."""
+
+    manifest = Path(manifest_path)
+    base_dir = manifest.parent
+    cases: list[dict[str, object]] = []
+    total_true_positive = 0
+    total_false_positive = 0
+    total_false_negative = 0
+    field_counts: dict[str, dict[str, int]] = {}
+
+    with manifest.open(newline="", encoding="utf-8") as handle:
+        rows = csv.DictReader(handle, delimiter="\t")
+        for row in rows:
+            name = row["name"].strip()
+            input_path = _resolve_manifest_path(base_dir, row["input_path"])
+            truth_path = _resolve_manifest_path(base_dir, row["truth_path"])
+            summary = benchmark_file(
+                input_path,
+                truth_path,
+                input_type=row.get("input_type", "tabular") or "tabular",
+                confidence_threshold=confidence_threshold,
+            )
+            case_summary = {
+                "name": name,
+                "description": row.get("description", ""),
+                **summary,
+            }
+            cases.append(case_summary)
+
+            overall = summary["overall"]
+            if isinstance(overall, dict):
+                total_true_positive += int(overall["true_positive"])
+                total_false_positive += int(overall["false_positive"])
+                total_false_negative += int(overall["false_negative"])
+
+            by_field = summary["by_field"]
+            if isinstance(by_field, dict):
+                for field, metrics in by_field.items():
+                    if not isinstance(metrics, dict):
+                        continue
+                    counts = field_counts.setdefault(
+                        str(field),
+                        {"true_positive": 0, "false_positive": 0, "false_negative": 0},
+                    )
+                    counts["true_positive"] += int(metrics["true_positive"])
+                    counts["false_positive"] += int(metrics["false_positive"])
+                    counts["false_negative"] += int(metrics["false_negative"])
+
+    overall_metrics = _metrics_from_counts(
+        total_true_positive,
+        total_false_positive,
+        total_false_negative,
+    )
+    by_field_summary = {
+        field: _metrics_from_counts(
+            counts["true_positive"],
+            counts["false_positive"],
+            counts["false_negative"],
+        ).as_dict()
+        for field, counts in sorted(field_counts.items())
+    }
+
+    return {
+        "manifest_path": str(manifest),
+        "confidence_threshold": confidence_threshold,
+        "case_count": len(cases),
+        "overall": overall_metrics.as_dict(),
+        "by_field": by_field_summary,
+        "cases": cases,
+    }
+
+
+def write_benchmark_json(summary: dict[str, object], path: str | Path) -> None:
+    """Write benchmark summary JSON."""
+
+    Path(path).write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+
+def _observed_mappings(records: list[dict[str, object]]) -> set[MappingKey]:
+    return {
+        (str(record["sample_id"]), str(record["field_type"]), str(record["ontology_id"]))
+        for record in records
+        if record.get("accepted") and record.get("ontology_id") and record.get("backend") != "inference"
+    }
+
+
+def _truth_mappings(path: str | Path) -> set[MappingKey]:
+    with Path(path).open(newline="", encoding="utf-8") as handle:
+        rows = csv.DictReader(handle, delimiter="\t")
+        return {
+            (row["sample_id"], row["field_type"], row["ontology_id"])
+            for row in rows
+            if row.get("sample_id") and row.get("field_type") and row.get("ontology_id")
+        }
+
+
+def _metrics(observed: set[MappingKey], expected: set[MappingKey]) -> BenchmarkMetrics:
+    true_positive = len(observed & expected)
+    false_positive = len(observed - expected)
+    false_negative = len(expected - observed)
+    return _metrics_from_counts(true_positive, false_positive, false_negative)
+
+
+def _metrics_from_counts(
+    true_positive: int,
+    false_positive: int,
+    false_negative: int,
+) -> BenchmarkMetrics:
+    precision = true_positive / (true_positive + false_positive) if true_positive + false_positive else 0.0
+    recall = true_positive / (true_positive + false_negative) if true_positive + false_negative else 0.0
+    f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0
+    return BenchmarkMetrics(
+        true_positive=true_positive,
+        false_positive=false_positive,
+        false_negative=false_negative,
+        precision=round(precision, 4),
+        recall=round(recall, 4),
+        f1=round(f1, 4),
+    )
+
+
+def _resolve_manifest_path(base_dir: Path, value: str) -> Path:
+    path = Path(value)
+    return path if path.is_absolute() else base_dir / path

omicsmeta/cli.py

@@ -0,0 +1,252 @@
+"""Command-line interface for omicsmeta."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from omicsmeta.core.harmonizer import HarmonizationResult, Harmonizer, merge_results
+from omicsmeta.core.mapper import BuiltinMapper, Text2TermMapper, load_builtin_terms
+from omicsmeta.io.writers import write_html_report, write_tabular
+from omicsmeta.ontologies.resources import (
+    DEFAULT_CACHE_DIR,
+    all_resource_names,
+    build_ontology_index,
+    cached_resource_paths,
+    default_index_path,
+    download_resources,
+    resource_status,
+)
+
+INPUT_TYPES = ["tabular", "geo_soft", "biosample_xml", "sra_xml"]
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="omicsmeta")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    harmonize = subparsers.add_parser("harmonize", help="harmonize metadata from a file or GEO accession")
+    harmonize.add_argument("input", nargs="?", help="input metadata file")
+    harmonize.add_argument("--input-type", choices=INPUT_TYPES, default="tabular")
+    harmonize.add_argument("--geo-accession", help="GEO accession to fetch directly, such as GSE123456")
+    harmonize.add_argument("--output", required=True, help="harmonized output TSV")
+    harmonize.add_argument("--unmapped", required=True, help="unmapped terms TSV")
+    harmonize.add_argument("--unmapped-summary-output", help="deduplicated unmapped-term review TSV")
+    harmonize.add_argument("--sample-output", help="sample-wide output TSV with one row per input sample")
+    harmonize.add_argument("--report", required=True, help="HTML QC report")
+    harmonize.add_argument("--confidence-threshold", type=float, default=0.70)
+    harmonize.add_argument("--mapper", choices=["builtin", "text2term"], default="builtin")
+    harmonize.add_argument(
+        "--ontology-obo",
+        action="append",
+        default=[],
+        help="local OBO file to load into the built-in mapper; may be repeated",
+    )
+    harmonize.add_argument(
+        "--no-default-terms",
+        action="store_true",
+        help="use only terms loaded from --ontology-obo with the built-in mapper",
+    )
+    harmonize.add_argument(
+        "--ontology-resource",
+        action="append",
+        choices=[*all_resource_names(), "all"],
+        default=[],
+        help="cached managed ontology resource to load into the built-in mapper; may be repeated",
+    )
+    harmonize.add_argument(
+        "--ontology-cache-dir",
+        default=str(DEFAULT_CACHE_DIR),
+        help="directory containing managed ontology resources",
+    )
+
+    batch = subparsers.add_parser("batch", help="harmonize multiple metadata files or GEO accessions")
+    batch.add_argument("--input", action="append", default=[], help="input metadata file; may be repeated")
+    batch.add_argument("--input-type", choices=INPUT_TYPES, default="tabular")
+    batch.add_argument("--geo-accession", action="append", default=[], help="GEO accession to fetch; may be repeated")
+    batch.add_argument("--output", required=True, help="combined harmonized output TSV")
+    batch.add_argument("--unmapped", required=True, help="combined unmapped terms TSV")
+    batch.add_argument("--unmapped-summary-output", help="deduplicated unmapped-term review TSV")
+    batch.add_argument("--sample-output", help="combined sample-wide output TSV")
+    batch.add_argument("--report", required=True, help="combined HTML QC report")
+    batch.add_argument("--confidence-threshold", type=float, default=0.70)
+    batch.add_argument("--mapper", choices=["builtin", "text2term"], default="builtin")
+    batch.add_argument("--ontology-obo", action="append", default=[])
+    batch.add_argument("--no-default-terms", action="store_true")
+    batch.add_argument("--ontology-resource", action="append", choices=[*all_resource_names(), "all"], default=[])
+    batch.add_argument("--ontology-cache-dir", default=str(DEFAULT_CACHE_DIR))
+
+    ontologies = subparsers.add_parser("ontologies", help="manage local ontology resources")
+    ontology_subparsers = ontologies.add_subparsers(dest="ontology_command", required=True)
+
+    list_cmd = ontology_subparsers.add_parser("list", help="list known ontology resources")
+    list_cmd.add_argument("--cache-dir", default=str(DEFAULT_CACHE_DIR))
+
+    download_cmd = ontology_subparsers.add_parser("download", help="download ontology resources")
+    download_cmd.add_argument("resources", nargs="*", choices=[*all_resource_names(), "all"])
+    download_cmd.add_argument("--cache-dir", default=str(DEFAULT_CACHE_DIR))
+    download_cmd.add_argument("--overwrite", action="store_true")
+
+    index_cmd = ontology_subparsers.add_parser("index", help="build a SQLite synonym index")
+    index_cmd.add_argument("--resource", action="append", choices=[*all_resource_names(), "all"], default=[])
+    index_cmd.add_argument("--ontology-obo", action="append", default=[])
+    index_cmd.add_argument("--cache-dir", default=str(DEFAULT_CACHE_DIR))
+    index_cmd.add_argument("--output", help="SQLite output path")
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    if args.command == "harmonize":
+        if not args.input and not args.geo_accession:
+            parser.error("harmonize requires an input file or --geo-accession")
+
+        mapper = _mapper(
+            args.mapper,
+            args.confidence_threshold,
+            ontology_paths=args.ontology_obo,
+            ontology_resources=args.ontology_resource,
+            ontology_cache_dir=args.ontology_cache_dir,
+            include_defaults=not args.no_default_terms,
+        )
+        harmonizer = Harmonizer(mapper=mapper, confidence_threshold=args.confidence_threshold)
+        if args.geo_accession:
+            result = harmonizer.from_geo(args.geo_accession)
+        else:
+            result = harmonizer.from_file(args.input, file_type=args.input_type)
+        _write_result_outputs(result, args)
+        return 0
+
+    if args.command == "batch":
+        if not args.input and not args.geo_accession:
+            parser.error("batch requires --input or --geo-accession")
+
+        mapper = _mapper(
+            args.mapper,
+            args.confidence_threshold,
+            ontology_paths=args.ontology_obo,
+            ontology_resources=args.ontology_resource,
+            ontology_cache_dir=args.ontology_cache_dir,
+            include_defaults=not args.no_default_terms,
+        )
+        harmonizer = Harmonizer(mapper=mapper, confidence_threshold=args.confidence_threshold)
+        sourced_results: list[tuple[str, HarmonizationResult]] = []
+        for input_path in args.input:
+            sourced_results.append((Path(input_path).name, harmonizer.from_file(input_path, file_type=args.input_type)))
+        for accession in args.geo_accession:
+            sourced_results.append((str(accession).upper(), harmonizer.from_geo(accession)))
+
+        _write_result_outputs(merge_results(sourced_results), args)
+        return 0
+
+    if args.command == "ontologies":
+        if args.ontology_command == "list":
+            _print_resource_status(args.cache_dir)
+            return 0
+        if args.ontology_command == "download":
+            names = _expand_resource_names(args.resources or ["all"])
+            paths = download_resources(names, cache_dir=args.cache_dir, overwrite=args.overwrite)
+            for path in paths:
+                print(path)
+            return 0
+        if args.ontology_command == "index":
+            names = _expand_resource_names(args.resource)
+            ontology_paths = []
+            if names:
+                ontology_paths.extend(cached_resource_paths(names, cache_dir=args.cache_dir))
+            ontology_paths.extend(Path(path) for path in args.ontology_obo)
+            if not ontology_paths:
+                parser.error("ontologies index requires --resource or --ontology-obo")
+            output = Path(args.output) if args.output else default_index_path(args.cache_dir)
+            output.parent.mkdir(parents=True, exist_ok=True)
+            count = build_ontology_index(ontology_paths, output_path=output)
+            print(f"Indexed {count} terms into {output}")
+            return 0
+
+    parser.error(f"Unhandled command: {args.command}")
+    return 2
+
+
+def _mapper(
+    name: str,
+    confidence_threshold: float,
+    *,
+    ontology_paths: list[str],
+    ontology_resources: list[str],
+    ontology_cache_dir: str,
+    include_defaults: bool,
+) -> BuiltinMapper | Text2TermMapper:
+    if name == "builtin":
+        resource_names = _expand_resource_names(ontology_resources)
+        resource_paths = cached_resource_paths(resource_names, cache_dir=ontology_cache_dir) if resource_names else []
+        terms = load_builtin_terms([*resource_paths, *ontology_paths], include_defaults=include_defaults)
+        return BuiltinMapper(terms=terms, confidence_threshold=confidence_threshold)
+    if name == "text2term":
+        if ontology_paths or ontology_resources or not include_defaults:
+            raise ValueError(
+                "--ontology-obo, --ontology-resource, and --no-default-terms are only supported by --mapper builtin"
+            )
+        return Text2TermMapper(confidence_threshold=confidence_threshold)
+    raise ValueError(f"Unsupported mapper: {name}")
+
+
+def _write_result_outputs(result: HarmonizationResult, args: argparse.Namespace) -> None:
+    write_tabular(result.harmonized, args.output)
+    write_tabular(result.unmapped, args.unmapped)
+    if getattr(args, "unmapped_summary_output", None):
+        write_tabular(
+            result.unmapped_summary,
+            args.unmapped_summary_output,
+            default_columns=[
+                "field_type",
+                "normalized_term",
+                "occurrence_count",
+                "batch_sources",
+                "sample_ids",
+                "columns",
+                "example_terms",
+                "best_candidate_id",
+                "best_candidate_label",
+                "best_candidate_ontology",
+                "best_candidate_confidence",
+            ],
+        )
+    if getattr(args, "sample_output", None):
+        write_tabular(
+            result.sample_table,
+            args.sample_output,
+            default_columns=["row_index", "sample_id", "batch_source"],
+        )
+    write_html_report(result.qc_summary, args.report)
+
+
+def _expand_resource_names(names: list[str]) -> list[str]:
+    if not names:
+        return []
+    if "all" in names:
+        return all_resource_names()
+    return names
+
+
+def _print_resource_status(cache_dir: str) -> None:
+    print("name\tcached\tsize_bytes\tpath\turl\tdescription")
+    for row in resource_status(cache_dir):
+        print(
+            "\t".join(
+                [
+                    str(row["name"]),
+                    str(row["cached"]).lower(),
+                    str(row["size_bytes"]),
+                    str(row["path"]),
+                    str(row["url"]),
+                    str(row["description"]),
+                ]
+            )
+        )
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

omicsmeta/core/__init__.py

@@ -0,0 +1,26 @@
+"""Core harmonization components."""
+
+from omicsmeta.core.detector import FieldDetection, detect_field, detect_fields
+from omicsmeta.core.fetcher import fetch_geo_rows, fetch_geo_soft
+from omicsmeta.core.harmonizer import HarmonizationResult, Harmonizer, merge_results
+from omicsmeta.core.mapper import BuiltinMapper, MappingResult, load_builtin_terms
+from omicsmeta.core.normalizer import normalize_text, split_terms
+from omicsmeta.core.types import FieldType, OntologyTerm
+
+__all__ = [
+    "BuiltinMapper",
+    "FieldDetection",
+    "FieldType",
+    "HarmonizationResult",
+    "Harmonizer",
+    "MappingResult",
+    "OntologyTerm",
+    "detect_field",
+    "detect_fields",
+    "fetch_geo_rows",
+    "fetch_geo_soft",
+    "load_builtin_terms",
+    "merge_results",
+    "normalize_text",
+    "split_terms",
+]

omicsmeta/core/detector.py

@@ -0,0 +1,205 @@
+"""Heuristic detection of metadata field types."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from collections.abc import Iterable, Mapping
+
+from omicsmeta.core.normalizer import normalize_text
+from omicsmeta.core.types import FieldType
+
+
+@dataclass(frozen=True)
+class NameHint:
+    """A weighted column-name signal for field detection."""
+
+    text: str
+    weight: float
+
+
+FIELD_NAME_HINTS: dict[FieldType, tuple[NameHint, ...]] = {
+    FieldType.DISEASE: (
+        NameHint("disease state", 0.82),
+        NameHint("disease", 0.78),
+        NameHint("diagnosis", 0.78),
+        NameHint("pathology", 0.72),
+        NameHint("tumor type", 0.72),
+        NameHint("tumour type", 0.72),
+        NameHint("cancer", 0.72),
+        NameHint("condition", 0.48),
+        NameHint("phenotype", 0.24),
+    ),
+    FieldType.TISSUE: (
+        NameHint("body site", 0.82),
+        NameHint("tissue", 0.78),
+        NameHint("organ", 0.72),
+        NameHint("anatomy", 0.72),
+        NameHint("sample type", 0.42),
+    ),
+    FieldType.CELL_LINE: (
+        NameHint("cell line", 0.84),
+        NameHint("cellline", 0.84),
+        NameHint("cell_line", 0.84),
+        NameHint("cell", 0.35),
+    ),
+    FieldType.SEX: (
+        NameHint("sex", 0.84),
+        NameHint("gender", 0.84),
+    ),
+    FieldType.AGE: (
+        NameHint("age at", 0.84),
+        NameHint("age", 0.78),
+        NameHint("developmental stage", 0.48),
+    ),
+    FieldType.TREATMENT: (
+        NameHint("treatment", 0.84),
+        NameHint("treated", 0.72),
+        NameHint("drug", 0.72),
+        NameHint("compound", 0.72),
+        NameHint("stimulus", 0.72),
+        NameHint("exposure", 0.72),
+    ),
+    FieldType.SPECIES: (
+        NameHint("organism", 0.84),
+        NameHint("species", 0.78),
+        NameHint("taxon", 0.78),
+        NameHint("taxid", 0.78),
+    ),
+}
+
+SEX_VALUES = {"male", "female", "m", "f", "man", "woman"}
+SPECIES_VALUES = {
+    "homo sapiens",
+    "human",
+    "mus musculus",
+    "mouse",
+    "rattus norvegicus",
+    "rat",
+}
+KNOWN_CELL_LINES = {"a549", "hela", "hek293", "jurkat", "k562", "mcf-7", "mcf7", "u87"}
+DISEASE_MARKERS = {
+    "adenocarcinoma",
+    "cancer",
+    "carcinoma",
+    "disease",
+    "fibrosis",
+    "glioblastoma",
+    "leukemia",
+    "lymphoma",
+    "melanoma",
+    "tumor",
+    "tumour",
+}
+TISSUE_MARKERS = {
+    "blood",
+    "brain",
+    "breast",
+    "colon",
+    "heart",
+    "kidney",
+    "liver",
+    "lung",
+    "spleen",
+    "tissue",
+}
+TREATMENT_MARKERS = {"treated", "untreated", "control", "vehicle", "drug", "stimulated"}
+AGE_RE = re.compile(r"^\d+(\.\d+)?\s*(day|days|week|weeks|month|months|year|years|yr|yrs|y)?$", re.I)
+
+
+@dataclass(frozen=True)
+class FieldDetection:
+    """Detected semantic type for one metadata column."""
+
+    field_type: FieldType
+    confidence: float
+    signals: tuple[str, ...] = ()
+
+
+def detect_field(column_name: str, values: Iterable[object] = ()) -> FieldDetection:
+    """Detect the semantic role of a metadata column."""
+
+    normalized_name = normalize_text(column_name, expand_abbreviations=False)
+    value_list = [normalize_text(value, expand_abbreviations=False) for value in values if str(value).strip()]
+
+    scores: dict[FieldType, float] = {field_type: 0.0 for field_type in FieldType}
+    signals: dict[FieldType, list[str]] = {field_type: [] for field_type in FieldType}
+
+    for field_type, hints in FIELD_NAME_HINTS.items():
+        for hint in hints:
+            hint_norm = normalize_text(hint.text, expand_abbreviations=False)
+            if hint_norm and hint_norm in normalized_name:
+                scores[field_type] += hint.weight
+                signals[field_type].append(f"name:{hint.text}:{hint.weight:.2f}")
+                break
+
+    for field_type in (
+        FieldType.DISEASE,
+        FieldType.TISSUE,
+        FieldType.CELL_LINE,
+        FieldType.SEX,
+        FieldType.AGE,
+        FieldType.TREATMENT,
+        FieldType.SPECIES,
+    ):
+        score = _value_score(field_type, value_list)
+        if score:
+            scores[field_type] += score
+            signals[field_type].append(f"values:{score:.2f}")
+
+    best_type = max(scores, key=scores.get)
+    best_score = scores[best_type]
+    if best_type == FieldType.UNKNOWN or best_score < 0.35:
+        return FieldDetection(FieldType.UNKNOWN, 0.0, ())
+
+    return FieldDetection(best_type, min(best_score, 0.99), tuple(signals[best_type]))
+
+
+def detect_fields(rows: Iterable[Mapping[str, object]]) -> dict[str, FieldDetection]:
+    """Detect field types for all columns in a row-oriented table."""
+
+    row_list = list(rows)
+    columns: list[str] = []
+    seen: set[str] = set()
+    for row in row_list:
+        for column in row:
+            if column not in seen:
+                seen.add(column)
+                columns.append(column)
+
+    detections: dict[str, FieldDetection] = {}
+    for column in columns:
+        detections[column] = detect_field(column, (row.get(column, "") for row in row_list))
+    return detections
+
+
+def _value_score(field_type: FieldType, values: list[str]) -> float:
+    if not values:
+        return 0.0
+
+    sample = values[:50]
+    matches = sum(1 for value in sample if _matches_field(field_type, value))
+    if matches == 0:
+        return 0.0
+
+    proportion = matches / len(sample)
+    return min(0.70, 0.20 + proportion * 0.55)
+
+
+def _matches_field(field_type: FieldType, value: str) -> bool:
+    if field_type == FieldType.SEX:
+        return value in SEX_VALUES
+    if field_type == FieldType.SPECIES:
+        return value in SPECIES_VALUES or value.startswith("ncbitaxon:")
+    if field_type == FieldType.CELL_LINE:
+        tokens = set(re.split(r"[^a-z0-9-]+", value))
+        return value in KNOWN_CELL_LINES or bool(tokens & KNOWN_CELL_LINES)
+    if field_type == FieldType.AGE:
+        return bool(AGE_RE.match(value))
+    if field_type == FieldType.DISEASE:
+        return any(marker in value for marker in DISEASE_MARKERS)
+    if field_type == FieldType.TISSUE:
+        return value in TISSUE_MARKERS or value.endswith(" tissue")
+    if field_type == FieldType.TREATMENT:
+        return any(marker in value for marker in TREATMENT_MARKERS)
+    return False