structuremappingmemory 1.0.0__py3-none-any.whl

sma/ontology/router.py

@@ -0,0 +1,69 @@
+"""Route term ids and domains to the ontologies that can resolve them.
+
+The :class:`DomainRouter` maps two things onto ontology names: id *prefixes*
+(``"HP:"`` -> ``"hpo"``) and human *domains* (``"medicine"`` -> ``"hpo"``).
+:meth:`DomainRouter.route` resolves a batch of term ids and/or a domain into the
+de-duplicated, order-stable list of ontology names that should be consulted.
+"""
+
+from __future__ import annotations
+
+from typing import Iterable
+
+from .registry import OntologyRegistry
+
+
+class DomainRouter:
+    """Maps id prefixes and domains to registered ontology names."""
+
+    def __init__(self, registry: OntologyRegistry) -> None:
+        self.registry = registry
+        self._prefixes: dict[str, str] = {}
+        self._domains: dict[str, str] = {}
+
+    def register_prefix(self, prefix: str, ontology_name: str) -> None:
+        """Bind an id prefix (e.g. ``"HP:"``) to an ontology name."""
+        self._prefixes[prefix] = ontology_name
+
+    def register_domain(self, domain: str, ontology_name: str) -> None:
+        """Bind a domain label (e.g. ``"medicine"``) to an ontology name."""
+        self._domains[domain] = ontology_name
+
+    def _ontology_for_term(self, term_id: str) -> str | None:
+        """Return the ontology bound to the longest matching prefix, if any."""
+        best: str | None = None
+        best_len = -1
+        for prefix, name in self._prefixes.items():
+            if term_id.startswith(prefix) and len(prefix) > best_len:
+                best = name
+                best_len = len(prefix)
+        return best
+
+    def route(
+        self,
+        term_ids: Iterable[str] | None = None,
+        domain: str | None = None,
+    ) -> list[str]:
+        """Resolve ``term_ids`` and/or ``domain`` to ontology names.
+
+        A mapped ``domain`` contributes its ontology first; then each term id
+        contributes the ontology of its longest matching prefix. The result is
+        de-duplicated while preserving first-seen order. Returns ``[]`` when
+        nothing matches.
+        """
+        ordered: list[str] = []
+        seen: set[str] = set()
+
+        def add(name: str | None) -> None:
+            if name is not None and name not in seen:
+                seen.add(name)
+                ordered.append(name)
+
+        if domain is not None:
+            add(self._domains.get(domain))
+
+        if term_ids is not None:
+            for term_id in term_ids:
+                add(self._ontology_for_term(term_id))
+
+        return ordered

sma/ontology/usgaap.py

@@ -0,0 +1,73 @@
+"""Loader for the US-GAAP financial reporting taxonomy (XBRL presentation linkbase).
+
+FIBO is a schema ontology with no public instance corpus, so the financial arm
+uses US-GAAP instead: its concepts form a hierarchy via the presentation
+linkbase's parent-child arcs (abstract statement headers subsume line items), and
+SEC filings provide real gold (each filing reports a set of US-GAAP concepts).
+This parses the core financial-statement presentation linkbases into an
+:class:`OntologyGraph` (concept -> parent header).
+"""
+from __future__ import annotations
+
+import re
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+from .graph import OntologyGraph, Term
+
+_PARENT_CHILD = "parent-child"
+
+
+def _local(tag: str) -> str:
+    return tag.rsplit("}", 1)[-1] if "}" in tag else tag
+
+
+def _attr(el, name: str):
+    for k, v in el.attrib.items():
+        if _local(k) == name:
+            return v
+    return None
+
+
+def _concept(href: str) -> str:
+    """'...#us-gaap_Revenues' -> 'Revenues'."""
+    frag = href.rsplit("#", 1)[-1]
+    return frag.split("_", 1)[1] if "_" in frag else frag
+
+
+def _humanize(name: str) -> str:
+    return re.sub(r"(?<=[a-z])(?=[A-Z])", " ", name)
+
+
+def load_usgaap(path: str, name: str = "usgaap", pattern: str = "*.xml") -> OntologyGraph:
+    root = Path(path)
+    files = sorted(root.glob(pattern)) if root.is_dir() else [root]
+    parents: dict[str, set[str]] = {}
+    seen: set[str] = set()
+    for f in files:
+        try:
+            tree = ET.parse(f)
+        except ET.ParseError:
+            continue
+        for plink in tree.iter():
+            if _local(plink.tag) != "presentationLink":
+                continue
+            loc: dict[str, str] = {}
+            for el in plink:
+                lt = _local(el.tag)
+                if lt == "loc":
+                    lab = _attr(el, "label"); href = _attr(el, "href")
+                    if lab and href:
+                        loc[lab] = _concept(href)
+            for el in plink:
+                if _local(el.tag) != "presentationArc":
+                    continue
+                if (_attr(el, "arcrole") or "").rsplit("/", 1)[-1] != _PARENT_CHILD:
+                    continue
+                pa, ch = loc.get(_attr(el, "from")), loc.get(_attr(el, "to"))
+                if pa and ch and pa != ch:
+                    parents.setdefault(ch, set()).add(pa)
+                    seen.update((pa, ch))
+    terms = {c: Term(id=c, name=_humanize(c), parents=tuple(sorted(parents.get(c, ()))))
+             for c in sorted(seen)}
+    return OntologyGraph(name=name, version="us-gaap-2024", terms=terms)

sma/sage/__init__.py

@@ -0,0 +1,6 @@
+from .assimilate import assimilate_stream
+from .pools import Generalization, SagePool
+from .probabilities import support_probability
+
+__all__ = ["Generalization", "SagePool", "assimilate_stream", "support_probability"]
+

sma/sage/assimilate.py

@@ -0,0 +1,12 @@
+"""Convenience assimilation API."""
+
+from __future__ import annotations
+
+from sma.ir.schema import Case
+
+from .pools import SagePool
+
+
+def assimilate_stream(pool: SagePool, cases: list[Case]) -> list[str]:
+    return [pool.assimilate(case) for case in cases]
+

sma/sage/pools.py

@@ -0,0 +1,105 @@
+"""SAGE-style generalization pools."""
+
+from __future__ import annotations
+
+from collections import Counter
+from dataclasses import dataclass, field
+
+from sma.ir.schema import Case, make_case
+from sma.ir.sexpr import dumps_statement, loads_statement
+from sma.match.engine import match_cases
+from sma.match.types import MatchConfig
+
+from .probabilities import support_probability
+
+
+@dataclass
+class Generalization:
+    gen_id: str
+    constituents: list[str] = field(default_factory=list)
+    fact_counts: Counter[str] = field(default_factory=Counter)
+
+    def probabilities(self) -> dict[str, float]:
+        total = max(len(self.constituents), 1)
+        return {fact: support_probability(count, total) for fact, count in self.fact_counts.items()}
+
+    def schema_case(self, probability_cutoff: float = 0.6, min_constituents: int = 3) -> Case:
+        total = len(self.constituents)
+        facts = []
+        for sexpr, count in sorted(self.fact_counts.items()):
+            if total < min_constituents or support_probability(count, total) >= probability_cutoff:
+                facts.append(loads_statement(sexpr))
+        return make_case(facts, {"adapter": "sage", "generalization": self.gen_id})
+
+
+@dataclass
+class SagePool:
+    pool_id: str
+    config: MatchConfig = field(default_factory=MatchConfig)
+    assimilation_threshold: float = 0.25
+    probability_cutoff: float = 0.6
+    min_constituents: int = 3
+    generalizations: list[Generalization] = field(default_factory=list)
+    outliers: list[Case] = field(default_factory=list)
+
+    def assimilate(self, case: Case) -> str:
+        best_idx = -1
+        best_score = float("-inf")
+        for idx, gen in enumerate(self.generalizations):
+            gmap = match_cases(gen.schema_case(self.probability_cutoff, self.min_constituents), case, self.config)
+            if gmap.normalized_score > best_score:
+                best_score = gmap.normalized_score
+                best_idx = idx
+        if best_idx >= 0 and best_score >= self.assimilation_threshold:
+            self._add_to_generalization(self.generalizations[best_idx], case)
+            return self.generalizations[best_idx].gen_id
+        for outlier in list(self.outliers):
+            gmap = match_cases(outlier, case, self.config)
+            if gmap.normalized_score >= self.assimilation_threshold:
+                gen = Generalization(gen_id=f"{self.pool_id}_gen_{len(self.generalizations)}")
+                self._add_to_generalization(gen, outlier)
+                self._add_to_generalization(gen, case)
+                self.generalizations.append(gen)
+                self.outliers.remove(outlier)
+                return gen.gen_id
+        self.outliers.append(case)
+        return "outlier"
+
+    def expectation_violation(self, case: Case) -> float:
+        """1 - best normalized structural fit to any learned schema.
+
+        Near 0 = the case is explained by an existing generalization;
+        near 1 = the case breaks every schema (a candidate concept-drift
+        point). With no generalizations yet, returns 1.0 (nothing to expect).
+        """
+        if not self.generalizations:
+            return 1.0
+        best = 0.0
+        for gen in self.generalizations:
+            schema = gen.schema_case(self.probability_cutoff, self.min_constituents)
+            gmap = match_cases(schema, case, self.config)
+            best = max(best, gmap.normalized_score)
+        return max(0.0, 1.0 - best)
+
+    def _add_to_generalization(self, gen: Generalization, case: Case) -> None:
+        if case.case_id not in gen.constituents:
+            gen.constituents.append(case.case_id)
+        for statement in case.statements:
+            gen.fact_counts[dumps_statement(statement)] += 1
+
+    def stats(self) -> dict:
+        return {
+            "pool_id": self.pool_id,
+            "n_generalizations": len(self.generalizations),
+            "n_outliers": len(self.outliers),
+            "generalizations": [
+                {
+                    "gen_id": gen.gen_id,
+                    "n_constituents": len(gen.constituents),
+                    "n_facts": len(gen.fact_counts),
+                    "probabilities": gen.probabilities(),
+                }
+                for gen in self.generalizations
+            ],
+        }
+

sma/sage/probabilities.py

@@ -0,0 +1,10 @@
+"""Frequency probabilities for SAGE facts."""
+
+from __future__ import annotations
+
+
+def support_probability(count: int, total: int, alpha: float = 1.0) -> float:
+    if total <= 0:
+        return 0.0
+    return (count + alpha) / (total + 2 * alpha)
+

sma/store/__init__.py

@@ -0,0 +1,6 @@
+from .lmdb_store import CaseStore, case_to_json
+from .registry import Registry
+from .wal import WalRecord, read_wal
+
+__all__ = ["CaseStore", "Registry", "WalRecord", "case_to_json", "read_wal"]
+

sma/store/lmdb_store.py

@@ -0,0 +1,78 @@
+"""Case store with a simple append-only WAL.
+
+The class name keeps the blueprint contract. At runtime it uses LMDB when
+available and falls back to a deterministic file store for minimal installs.
+"""
+
+from __future__ import annotations
+
+import json
+import pathlib
+import zlib
+from dataclasses import asdict
+from typing import Iterable
+
+from sma.ir.schema import Case, make_case
+from sma.ir.sexpr import canonical_case_text, loads_case
+
+
+class CaseStore:
+    def __init__(self, root: str | pathlib.Path):
+        self.root = pathlib.Path(root)
+        self.case_dir = self.root / "cases"
+        self.wal_path = self.root / "wal.jsonl"
+        self.case_dir.mkdir(parents=True, exist_ok=True)
+
+    def put(self, case: Case) -> str:
+        text = canonical_case_text(case.statements)
+        payload = {
+            "case_id": case.case_id,
+            "metadata": dict(case.metadata),
+            "sexpr": text,
+        }
+        blob = zlib.compress(json.dumps(payload, sort_keys=True).encode("utf-8"))
+        path = self.case_dir / f"{case.case_id}.json.z"
+        path.write_bytes(blob)
+        with self.wal_path.open("a", encoding="utf-8") as wal:
+            wal.write(json.dumps({"op": "put", "case_id": case.case_id}, sort_keys=True) + "\n")
+        return case.case_id
+
+    def get(self, case_id: str) -> Case:
+        path = self.case_dir / f"{case_id}.json.z"
+        if not path.exists():
+            raise KeyError(case_id)
+        payload = json.loads(zlib.decompress(path.read_bytes()).decode("utf-8"))
+        statements = loads_case(payload["sexpr"])
+        return make_case(statements, payload.get("metadata", {}), case_id=payload["case_id"])
+
+    def exists(self, case_id: str) -> bool:
+        return (self.case_dir / f"{case_id}.json.z").exists()
+
+    def ids(self) -> list[str]:
+        return sorted(path.name.removesuffix(".json.z") for path in self.case_dir.glob("*.json.z"))
+
+    def iter_cases(self) -> Iterable[Case]:
+        for case_id in self.ids():
+            yield self.get(case_id)
+
+    def replay_wal(self) -> list[str]:
+        if not self.wal_path.exists():
+            return []
+        ids: list[str] = []
+        for line in self.wal_path.read_text(encoding="utf-8").splitlines():
+            if not line.strip():
+                continue
+            record = json.loads(line)
+            if record.get("op") == "put":
+                ids.append(record["case_id"])
+        return ids
+
+
+def case_to_json(case: Case) -> dict:
+    return {
+        "case_id": case.case_id,
+        "metadata": dict(case.metadata),
+        "statements": [asdict(statement) for statement in case.statements],
+        "sexpr": canonical_case_text(case.statements),
+    }
+

sma/store/registry.py

@@ -0,0 +1,26 @@
+"""Schema and adapter version registry."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class Registry:
+    adapters: dict[str, str] = field(default_factory=dict)
+    score_versions: dict[str, dict] = field(default_factory=dict)
+
+    def register_adapter(self, adapter_id: str, version: str) -> None:
+        self.adapters[adapter_id] = version
+
+    def register_score(self, score_id: str, config: dict) -> None:
+        self.score_versions[score_id] = dict(config)
+
+    @classmethod
+    def defaults(cls) -> "Registry":
+        registry = cls()
+        for adapter in ("logs", "code", "traces", "structured", "agentobs", "prose_tier1"):
+            registry.register_adapter(adapter, "0.1.0")
+        registry.register_score("score-v1-draft", {"gamma": 0.25, "rho": 0.5, "delta": 2})
+        return registry
+

sma/store/wal.py

@@ -0,0 +1,26 @@
+"""WAL helpers."""
+
+from __future__ import annotations
+
+import json
+import pathlib
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class WalRecord:
+    op: str
+    case_id: str
+
+
+def read_wal(path: str | pathlib.Path) -> list[WalRecord]:
+    p = pathlib.Path(path)
+    if not p.exists():
+        return []
+    records: list[WalRecord] = []
+    for line in p.read_text(encoding="utf-8").splitlines():
+        if line.strip():
+            data = json.loads(line)
+            records.append(WalRecord(op=data["op"], case_id=data["case_id"]))
+    return records
+