ontometer 0.1.0__py3-none-any.whl

ontometer/__init__.py

@@ -0,0 +1,42 @@
+"""OntoMeter — ontology quality assessment.
+
+Evaluates a single ontology against the OQuaRE quality model and, where a
+language model is configured, adds a qualitative review of the things structural
+metrics cannot see.
+
+    >>> from ontometer import evaluate
+    >>> result = evaluate("my-ontology.ttl", with_review=False)
+    >>> result.assessment.metrics["DITOnto"].score
+    5
+
+The corpus-scale research half lives in :mod:`ontometer.corpus` and needs the
+``corpus`` extra.
+"""
+
+from ontometer.assessment import Assessment, MetricResult, assess
+from ontometer.evaluation import Evaluation, evaluate
+from ontometer.onto.load import LoadError, load_graph, load_view
+from ontometer.onto.view import OntologyView, build_view
+from ontometer.review.findings import Finding, ReviewReport, Severity
+from ontometer.settings import LLMConfig, LLMProvider
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "Assessment",
+    "Evaluation",
+    "Finding",
+    "LLMConfig",
+    "LLMProvider",
+    "LoadError",
+    "MetricResult",
+    "OntologyView",
+    "ReviewReport",
+    "Severity",
+    "__version__",
+    "assess",
+    "build_view",
+    "evaluate",
+    "load_graph",
+    "load_view",
+]

ontometer/assessment.py

@@ -0,0 +1,104 @@
+"""Rolling raw metrics up into scores, sub-characteristics and characteristics."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from statistics import fmean
+
+from ontometer.metrics.oquare import METRICS_BY_CODE, compute_metrics
+from ontometer.metrics.seeds import seed_terms
+from ontometer.model.load import QualityModel, Scales, load_quality_model, load_scales
+from ontometer.onto.view import OntologyView
+
+
+@dataclass(frozen=True, slots=True)
+class MetricResult:
+    """One metric's raw value, its 1-5 score, and how it was computed."""
+
+    code: str
+    name: str
+    formula: str
+    value: float
+    score: int
+    family: str
+
+
+@dataclass(slots=True)
+class Assessment:
+    """Everything computed about one ontology, before any model is consulted."""
+
+    metrics: dict[str, MetricResult] = field(default_factory=dict)
+    subcharacteristics: dict[str, float] = field(default_factory=dict)
+    characteristics: dict[str, float] = field(default_factory=dict)
+    unsupported: dict[str, list[str]] = field(default_factory=dict)
+    seeds: dict[str, list[str]] = field(default_factory=dict)
+    aggregation: str = "mean"
+
+    @property
+    def overall(self) -> float:
+        """Mean of the characteristic scores.
+
+        OQuaRE defines no overall score. This one exists because a single number
+        is what makes two runs comparable at a glance, and it is computed the
+        same way as every other rollup — but it is the weakest claim in the
+        report and should not be read as a verdict.
+        """
+        values = list(self.characteristics.values())
+        return fmean(values) if values else 0.0
+
+    def worst_metrics(self, count: int = 5) -> list[MetricResult]:
+        """The *count* lowest-scoring metrics, worst first.
+
+        Ties break on the raw value's distance from its nearest band edge, so
+        the ordering is stable rather than dictionary order.
+        """
+        return sorted(self.metrics.values(), key=lambda m: (m.score, m.code))[:count]
+
+    def strongest_metrics(self, count: int = 3) -> list[MetricResult]:
+        return sorted(self.metrics.values(), key=lambda m: (-m.score, m.code))[:count]
+
+
+def assess(
+    view: OntologyView,
+    model: QualityModel | None = None,
+    scales: Scales | None = None,
+) -> Assessment:
+    """Compute metrics for *view* and roll them up through the quality model."""
+    model = model or load_quality_model()
+    scales = scales or load_scales()
+
+    raw = compute_metrics(view)
+    results: dict[str, MetricResult] = {}
+    for code, value in raw.items():
+        spec = METRICS_BY_CODE[code]
+        results[code] = MetricResult(
+            code=code,
+            name=spec.name,
+            formula=spec.formula,
+            value=value,
+            score=scales.score(code, value),
+            family=scales.family_of(code),
+        )
+
+    subcharacteristics: dict[str, float] = {}
+    characteristics: dict[str, float] = {}
+    for char_name, (_, subs) in model.characteristics.items():
+        char_scores: list[float] = []
+        for sub_name, codes in subs.items():
+            supporting = [results[c].score for c in codes if c in results]
+            if not supporting:
+                continue
+            score = fmean(supporting)
+            subcharacteristics[f"{char_name}.{sub_name}"] = score
+            char_scores.append(score)
+        if char_scores:
+            characteristics[char_name] = fmean(char_scores)
+
+    return Assessment(
+        metrics=results,
+        subcharacteristics=subcharacteristics,
+        characteristics=characteristics,
+        unsupported=model.unsupported_subcharacteristics(),
+        seeds=seed_terms(view),
+        aggregation=model.aggregation,
+    )

ontometer/cli/__init__.py

@@ -0,0 +1 @@
+"""Command-line entry points."""

ontometer/cli/_entry.py

@@ -0,0 +1,19 @@
+"""Console-script shims.
+
+Every entry point goes through here so that a missing optional dependency
+produces a sentence telling the user what to install, rather than a traceback
+ending in a module name they have no reason to recognise.
+"""
+
+from __future__ import annotations
+
+import sys
+
+
+def ontometer() -> None:
+    """Entry point for the ``ontometer`` command."""
+    try:
+        from ontometer.cli.main import cli
+    except ImportError as exc:  # pragma: no cover - depends on install shape
+        sys.exit(f"ontometer could not start: {exc}")
+    cli()

ontometer/cli/main.py

@@ -0,0 +1,172 @@
+"""The ``ontometer`` command."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import click
+
+from ontometer import __version__
+from ontometer.evaluation import evaluate as run_evaluate
+from ontometer.model.load import load_quality_model, load_scales
+from ontometer.onto.load import LoadError
+from ontometer.report.render import to_dict, to_markdown
+from ontometer.settings import LLMConfig, LLMProvider
+
+
+@click.group()
+@click.version_option(__version__, prog_name="ontometer")
+def cli() -> None:
+    """Ontology quality assessment."""
+
+
+@cli.command("eval")
+@click.argument("path", type=click.Path(exists=True, dir_okay=False, path_type=Path))
+@click.option(
+    "--format",
+    "output_format",
+    type=click.Choice(["md", "json"]),
+    default="md",
+    show_default=True,
+    help="Report format.",
+)
+@click.option(
+    "-o",
+    "--output",
+    type=click.Path(dir_okay=False, path_type=Path),
+    default=None,
+    help="Write the report here instead of stdout.",
+)
+@click.option(
+    "--review/--no-review",
+    default=True,
+    show_default=True,
+    help="Run the qualitative LLM review. Metrics are computed either way.",
+)
+@click.option(
+    "--provider",
+    type=click.Choice([p.value for p in LLMProvider]),
+    default=None,
+    help="Override LLM_PROVIDER.",
+)
+@click.option("--model", "model_name", default=None, help="Override LLM_MODEL_NAME.")
+@click.option("--base-url", default=None, help="Override LLM_BASE_URL.")
+@click.option(
+    "--instruction",
+    default="",
+    help="Extra guidance passed through to the review, e.g. a domain focus.",
+)
+@click.option(
+    "--fail-under",
+    type=float,
+    default=None,
+    help="Exit non-zero if the overall score falls below this. For CI.",
+)
+def eval_command(
+    path: Path,
+    output_format: str,
+    output: Path | None,
+    review: bool,
+    provider: str | None,
+    model_name: str | None,
+    base_url: str | None,
+    instruction: str,
+    fail_under: float | None,
+) -> None:
+    """Evaluate the ontology at PATH."""
+    # Start from the environment, then apply only the flags that were given.
+    # Splatting a dict into the settings model would work but erases every
+    # field type, so each override is assigned by name.
+    config = LLMConfig()
+    if provider:
+        config.provider = LLMProvider(provider)
+    if model_name:
+        config.model_name = model_name
+    if base_url:
+        config.base_url = base_url
+
+    try:
+        evaluation = run_evaluate(
+            path,
+            with_review=review,
+            config=config,
+            user_instruction=instruction,
+        )
+    except LoadError as exc:
+        raise click.ClickException(str(exc)) from exc
+
+    rendered = (
+        json.dumps(to_dict(evaluation), indent=2)
+        if output_format == "json"
+        else to_markdown(evaluation)
+    )
+
+    if output:
+        output.write_text(rendered, encoding="utf-8")
+        click.echo(f"Wrote {output}", err=True)
+    else:
+        click.echo(rendered)
+
+    if evaluation.review_skipped_reason and review:
+        click.echo(f"note: review not run — {evaluation.review_skipped_reason}", err=True)
+
+    if fail_under is not None and evaluation.assessment.overall < fail_under:
+        click.echo(
+            f"overall {evaluation.assessment.overall:.2f} is below --fail-under {fail_under}",
+            err=True,
+        )
+        sys.exit(1)
+
+
+@cli.command("model")
+@click.option(
+    "--what",
+    type=click.Choice(["metrics", "scales", "characteristics", "unsupported"]),
+    default="metrics",
+    show_default=True,
+    help="Which part of the quality model to print.",
+)
+def model_command(what: str) -> None:
+    """Show the quality model this build uses.
+
+    The model is configuration, not code — this is how you read what it
+    currently says before overriding it.
+    """
+    if what == "metrics":
+        from ontometer.metrics.oquare import METRICS
+
+        for spec in METRICS:
+            click.echo(f"{spec.code:<10} {spec.name:<34} {spec.formula}")
+        return
+
+    if what == "scales":
+        scales = load_scales()
+        for name, family in scales.families.items():
+            click.echo(f"[{name}] better = {family.direction}, cut points = {family.bounds}")
+        click.echo("")
+        for code, family_name in sorted(scales.metric_family.items()):
+            click.echo(f"{code:<10} {family_name}")
+        return
+
+    model = load_quality_model()
+    if what == "characteristics":
+        for name, (title, subs) in model.characteristics.items():
+            click.echo(f"\n{title} ({name})")
+            for sub, codes in subs.items():
+                shown = ", ".join(codes) if codes else "— no supporting metric —"
+                click.echo(f"  {sub:<34} {shown}")
+        return
+
+    unsupported = model.unsupported_subcharacteristics()
+    total = sum(len(v) for v in unsupported.values())
+    click.echo(
+        f"{total} sub-characteristics have no supporting metric.\n"
+        f"These are not oversights — they are the parts of ontology quality that\n"
+        f"are not a function of the graph's shape, and are what the review covers.\n"
+    )
+    for name, subs in unsupported.items():
+        click.echo(f"{name}:")
+        for sub in subs:
+            click.echo(f"  - {sub}")

ontometer/corpus/README.md

@@ -0,0 +1,260 @@
+# The corpus: acquisition, processing, and what can be reproduced
+
+The corpus is fetched from ten registries, parsed, deduplicated, and turned into
+a dependency network. This document is the shortest path from a clean checkout
+to a rebuilt network, and an honest account of which parts of that are
+reproducible and which are not.
+
+## 1. The pipeline end to end
+
+```mermaid
+flowchart LR
+    REG["<b>Ten registries</b><br/>OntoPortal ×6 · OBO Foundry<br/>Ontohub · OKG · LOV"]
+
+    subgraph S1["Stage 1 — fetch_ontologies.py"]
+        direction TB
+        OBJ[("objects/&lt;hh&gt;/&lt;sha256&gt;<br/>raw files, content-addressed")]
+        MAN[["manifest.jsonl<br/>one line per attempt"]]
+    end
+
+    subgraph S2["Stage 2 — process.py"]
+        direction TB
+        DB[("corpus.db<br/>corpus_registry<br/>+ fetch_attempts")]
+        SIDE[("structure/&lt;hh&gt;/&lt;hash&gt;.parquet<br/>subclass edges + entities")]
+    end
+
+    subgraph S3["Stage 3 — derived"]
+        direction TB
+        NET[["network_nodes.parquet<br/>network_edges.parquet"]]
+        SM[["structural_metrics.parquet"]]
+        TC[["term_coupling.parquet"]]
+    end
+
+    REG --> OBJ
+    REG --> MAN
+    MAN -->|"Phase 0 seed"| DB
+    OBJ -->|"Phase 1 parse"| DB
+    OBJ -->|"Phase 1 parse"| SIDE
+    DB -->|"Phase 2 dedup<br/>file hash → IRI → content hash"| DB
+
+    DB --> NET
+    SIDE --> SM
+    OBJ --> TC
+
+    NET --> NB["research/<br/>analysis.ipynb"]
+    SM --> NB
+    TC --> NB
+
+    classDef tracked fill:#224777,stroke:#293241,color:#fff
+    classDef untracked fill:#E67E22,stroke:#293241,color:#fff
+    class MAN,DB,NET,SM,TC tracked
+    class OBJ,SIDE untracked
+```
+
+**Blue is tracked in git. Orange is not** — it lives under `$ONTOLOGY_DIR` and is
+far too large for a repository.
+
+The diagram shows the dominant data flow. Two details it elides: both metric
+scripts also read `corpus.db` for the list of ontologies to process, and
+`term_coupling.py` re-parses the raw files rather than reading the sidecars.
+
+## 2. Where everything lives
+
+There are exactly **two** input locations, and every other path is derived from
+them. Nothing is computed from `__file__` and nothing has a hardcoded default
+pointing into someone's home directory; see `ontometer/corpus/paths.py`.
+
+| Setting | Flag | Environment | Default |
+|---|---|---|---|
+| Tracked artifacts | `--data-dir` | `ONTOMETER_DATA_DIR` | `./data` |
+| Object store | `--ontology-dir` | `ONTOLOGY_DIR` | — (required by the stages that read raw files) |
+
+```
+--data-dir/                          --ontology-dir/
+├── corpus.db                        ├── objects/<hh>/<sha256><ext>
+├── manifest.jsonl                   └── structure/<hh>/<content_hash>.parquet
+├── network_nodes.parquet
+├── network_edges.parquet            ../_parse_scratch/     (sibling of the store)
+├── structural_metrics.parquet
+├── term_coupling.parquet
+├── fetch_failures.log
+└── dedup_exclusions.log
+```
+
+Every script takes both flags, so a run can be pointed anywhere:
+
+```bash
+uv run python run/process.py --data-dir /tmp/run1 --ontology-dir ~/data/ontologies
+```
+
+Only **fetch**, **parse**, and the two metric scripts need the object store.
+Seeding, dedup and network construction work from the tracked artifacts alone.
+
+## 3. What is tracked, and why
+
+`data/` is committed. That is deliberate, and it is not the usual "derived
+artifacts don't belong in git" case:
+
+| Artifact | Size | Regenerable? |
+|---|---|---|
+| `corpus.db` | ~34 MiB | Only from the raw object store — see §4 |
+| `manifest.jsonl` | ~13 MiB | No. It is the record of every fetch attempt, including failures and registries since dropped |
+| `network_*.parquet` | ~1.3 MiB | Yes, from `corpus.db` in seconds |
+| `structural_metrics.parquet` | ~0.4 MiB | Only from the object store, via a parallel re-parse of ~3,600 files |
+| `term_coupling.parquet` | ~0.3 MiB | Same |
+
+**The corpus cannot be re-fetched into existence.** Registries move underneath
+you: LOV was unreachable through 2026-07 and returned in 2026-08 carrying ~950
+ontologies; Ontohub is a legacy service; download URLs die routinely — 1,263 of
+the recorded attempts already failed at capture time. A crawl run today produces
+a *different* corpus, not this one. These files are the primary record, not a
+cache of something you could rebuild.
+
+### What was removed from the tracked artifacts
+
+`corpus.db` and `manifest.jsonl` used to carry the verbatim registry API
+response for every submission, in `source_registry_metadata`. It was 58 MiB —
+72% of the database — and the same blob was written a second time into the
+manifest, which was 92% this one field. Nothing ever read it.
+
+It is now projected to a field allowlist (`ontometer/corpus/metadata.py`):
+`description`, `name`, `acronym`, `hasDomain`, `group`, `hasOntologyLanguage`,
+`hasLicense`, `homepage`, `publication`, `version`, `status`. Dropped are the
+JSON-LD `links` and `@context` boilerplate — 24 MiB of templated URLs — and the
+`contact`, `administeredBy`, `reviews` and `projects` records, which carried the
+names and email addresses of several hundred ontology maintainers. A public
+corpus should not redistribute those, independently of what they weigh. Email
+addresses are redacted from the free-text fields that are kept.
+
+The projection is applied at **two** boundaries — when a `FetchRecord` is
+constructed, and again when the manifest is seeded into the database — because a
+manifest captured before the projection existed still holds the full response,
+and seeding from one must not put it back.
+
+`parse_error` is clamped to 2,000 characters: two rows accounted for 7.28 of the
+7.31 MiB it occupied, because an rdflib error echoed an entire unparseable file
+back into its own message.
+
+If you need the unprojected metadata, re-fetch it — `canonical_url` and
+`source_id` are recorded per row. `run/compact_corpus.py` applies the projection
+to existing artifacts and is idempotent.
+
+## 4. Getting the raw object store
+
+Everything that re-executes parsing needs the raw files.
+
+`retrieved_file` is stored **relative** to the object-store root — e.g.
+`objects/61/613f4e44….rdf` — so a corpus captured on one machine resolves on
+another. `file_hash` is the SHA-256 of the raw bytes, so a restored store can be
+verified against the database byte-for-byte rather than taken on trust.
+
+```bash
+export ONTOLOGY_DIR="$HOME/data/Ontometer/ontologies"
+mkdir -p "$ONTOLOGY_DIR"
+tar --zstd -xf ontologies_objects_raw.tar.zst -C "$ONTOLOGY_DIR"
+```
+
+**If you do not have the archive**, the pipeline can re-fetch from
+`canonical_url`, but understand what that gives you: a *new* capture, against
+today's registries, with today's failures. Each OntoPortal instance is
+independently administered and needs its own key (`BIOPORTAL`, `AGROPORTAL`,
+`BIODIVPORTAL`, `MATPORTAL`, `ONTOPORTAL_ASTRO`, `TECHNOPORTAL`); OKG resolution
+uses `GITHUB_TOKEN`.
+
+```bash
+uv run python run/fetch_ontologies.py --sources lov,obo,okg,ontohub --ontoportal all
+```
+
+Rows whose `file_hash` still matches are byte-identical to the original capture;
+rows that differ, or no longer resolve, are the drift.
+
+## 5. Rebuilding
+
+```bash
+uv sync --extra corpus        # add --extra research for the notebook
+```
+
+### Seed → parse → dedup
+
+```bash
+uv run python run/process.py --phases seed,parse,dedup
+```
+
+```mermaid
+flowchart LR
+    A["<b>Phase 0 · seed</b><br/>manifest → registry<br/><i>skips source_ids already present</i>"]
+    B["<b>Phase 1 · parse</b><br/>raw files → metadata<br/><i>only rows with parse_status NULL</i>"]
+    C["<b>Phase 2 · dedup</b><br/>flag canonical latest<br/><i>only rows not already marked</i>"]
+    A --> B --> C
+    S(["needs --ontology-dir"]) -.-> B
+```
+
+Each phase is idempotent and resume-safe, which means **against the committed
+`data/corpus.db` this is a no-op** — every phase is gated on work not already
+done. To re-execute the logic rather than replay a stored result, reset first.
+
+**Full reset** — rebuild the database from the committed manifest:
+
+```bash
+rm data/corpus.db
+uv run python run/process.py
+```
+
+`data/manifest.jsonl` is independent of `corpus.db` and is not touched by this.
+
+**Partial reset** — keep the provenance rows, force Phases 1 and 2 to redo their
+work by clearing the columns each phase gates on:
+
+```bash
+sqlite3 data/corpus.db "
+  UPDATE corpus_registry SET
+    parse_status = NULL, parse_error = NULL, parse_date = NULL, parse_format = NULL,
+    ontology_iri = NULL, ontology_version_iri = NULL, ontology_version_info = NULL,
+    triple_count = NULL, class_count = NULL, property_count = NULL, individual_count = NULL,
+    imports = NULL, content_hash = NULL,
+    referenced_namespaces = NULL, own_namespaces = NULL,
+    canonical_entry_id = NULL, is_duplicate = 0, is_latest_version = 0;
+"
+```
+
+Either way the structural sidecars are content-addressed and write-once, keyed
+by `content_hash`: re-parsing a file that hashes to an existing sidecar skips
+the write rather than erroring, so there is no need to clear
+`$ONTOLOGY_DIR/structure/` unless you are validating that logic itself.
+
+### Derived artifacts
+
+```bash
+uv run python -m ontometer.corpus.network.construction   # network_{nodes,edges}.parquet
+uv run python run/structural_metrics.py                  # needs the object store
+uv run python run/term_coupling.py                       # needs the object store
+```
+
+The network was previously a `.gpickle`; loading a pickle executes whatever is
+inside it, which is not a reasonable thing to ask of anyone who downloads a
+research artifact. The Parquet pair is also about a sixth of the size and
+round-trips the graph exactly.
+
+Then open `research/analysis.ipynb`.
+
+## 6. What reproducibility means here
+
+Three different claims, worth keeping apart:
+
+1. **The analysis is reproducible from the tracked artifacts alone.** Clone,
+   install the `corpus` extra, rebuild the network from `corpus.db`, run the
+   notebook. Nothing external is needed. This is the claim that matters for
+   checking the results.
+2. **The parse is reproducible given the object store.** With the raw archive,
+   every parse and dedup decision can be re-executed and compared; `file_hash`
+   makes the inputs verifiable.
+3. **The capture is not reproducible.** Re-crawling produces a different corpus,
+   because the registries are live services. That is a property of the subject
+   matter, not a gap in the tooling — and the drift between captures is itself
+   something the temporal work intends to measure.
+
+## Further reading
+
+- `ontometer/corpus/pipeline/README.md` — fetch → parse → dedup, phase by phase,
+  the streaming strategy for oversized files, and the dedup passes.
+- `ontometer/corpus/network/README.md` — the node and edge model, precisely.

ontometer/corpus/__init__.py

@@ -0,0 +1,29 @@
+"""Corpus-scale research: multi-registry fetch, parse, dedup, dependency network.
+
+This half of OntoMeter needs a heavier dependency set than the evaluator — HTTP
+scraping, dataframes, Parquet and graph libraries — so it sits behind an extra.
+Importing it without that extra raises a directive message rather than a bare
+``ModuleNotFoundError`` naming whichever transitive dependency happened to be
+missed first.
+
+    pip install "ontometer[corpus]"
+"""
+
+from __future__ import annotations
+
+_REQUIRED = ("bs4", "networkx", "pandas", "pyarrow", "requests")
+
+
+def _check_dependencies() -> None:
+    import importlib.util
+
+    missing = [name for name in _REQUIRED if importlib.util.find_spec(name) is None]
+    if missing:
+        raise ImportError(
+            "ontometer.corpus needs the 'corpus' extra. Install it with:\n"
+            '    pip install "ontometer[corpus]"\n'
+            f"(missing: {', '.join(missing)})"
+        )
+
+
+_check_dependencies()

ontometer/corpus/fetchers/__init__.py

@@ -0,0 +1,15 @@
+"""
+OntoMeter fetchers package.
+
+About:
+    Lightweight per-source ontology fetchers. Each module exposes a single
+    fetch_<source>() function that returns a list of FetchRecord objects.
+    All records share the normalized schema:
+        (source_id, canonical_url, retrieved_file, retrieval_date,
+         source_registry, source_registry_metadata)
+
+Last Updated: 2026-06-24
+Progress: Initial package skeleton.
+Version History:
+    - v1.0: Package created as part of per-source fetcher refactor.
+"""