classifyre-cli 0.6.4__tar.gz → 0.6.5__tar.gz

classifyre_cli-0.6.5/.turbo/turbo-build.log

@@ -0,0 +1,3 @@
+$ uv sync
+Resolved 325 packages in 219ms
+Checked 51 packages in 1ms

{classifyre_cli-0.6.4 → classifyre_cli-0.6.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: classifyre-cli
-Version: 0.6.4
+Version: 0.6.5
 Summary: Classifyre CLI — scan and classify unstructured data sources
 License: MIT
 Keywords: data,ingestion,metadata,pii,secrets,unstructured

{classifyre_cli-0.6.4 → classifyre_cli-0.6.5}/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@classifyre/cli",
-  "version": "0.6.4",
+  "version": "0.6.5",
   "private": true,
   "scripts": {
     "build": "uv sync",

{classifyre_cli-0.6.4 → classifyre_cli-0.6.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "classifyre-cli"
-version = "0.6.4"
+version = "0.6.5"
 description = "Classifyre CLI — scan and classify unstructured data sources"
 readme = "README.md"
 # Upper bound is load-bearing: onnxruntime publishes no cp315 wheels (notably

{classifyre_cli-0.6.4 → classifyre_cli-0.6.5}/scripts/generate_sdk_completions.py

@@ -35,6 +35,7 @@ from src.notebook.contract import (  # noqa: E402
 from src.notebook.sdk import (  # noqa: E402
     Asset,
     Context,
+    Entity,
     FieldMapping,
     FlowType,
     NotebookFile,
@@ -259,6 +260,13 @@ def build() -> dict[str, Any]:
                 "documentation": _summary(Tag),
                 "insertText": "Tag",
             },
+            {
+                "label": "Entity",
+                "kind": "class",
+                "detail": 'Entity(name, type="ORGANIZATION", identifiers={...})',
+                "documentation": _summary(Entity),
+                "insertText": "Entity",
+            },
             {
                 "label": "Ref",
                 "kind": "class",
@@ -304,6 +312,10 @@ def build() -> dict[str, Any]:
                 "documentation": _summary(Tag),
                 "fields": _fields(Tag),
             },
+            "Entity": {
+                "documentation": _summary(Entity),
+                "fields": _fields(Entity),
+            },
             "AugmentedAsset": {
                 "documentation": _summary(augmentation_sdk.AugmentedAsset),
                 "members": _members(augmentation_sdk.AugmentedAsset),

{classifyre_cli-0.6.4 → classifyre_cli-0.6.5}/src/augmentation/sdk.py

@@ -44,11 +44,13 @@ from ..graph.edges import (
     FlowType,
     Ref,
     contains,
+    edge_to_payload,
     flow,
     references,
     same_as,
     uses,
 )
+from ..graph.edges import means as means_edge
 from ..notebook.files import ParsedContent, pages, parse
 from ..notebook.sdk import (
     MODULE_NAME,
@@ -141,6 +143,7 @@ class AugmentedAsset:
         self._added_metadata: dict[str, Any] = {}
         self._tags: dict[str, str] = {}
         self._links: list[str] = []
+        self._means: list[dict[str, Any]] = []
         self._new_urn: str | None = None
         self.warnings: list[str] = []
 
@@ -313,6 +316,33 @@ class AugmentedAsset:
         if target not in self._links:
             self._links.append(target)
 
+    def means(
+        self,
+        key: str,
+        *,
+        evidence: Mapping[str, Any] | None = None,
+        confidence: float | None = None,
+    ) -> None:
+        """Declare that this asset is about a glossary concept or entity, by key.
+
+        Additive, like ``tag()`` and ``link()``: a DECLARED semantic link. The
+        term need not exist yet; the declaration links once it is defined. An
+        invalid key is skipped with a warning.
+        """
+        try:
+            edge = means_edge(
+                Ref.asset(self._hash),
+                Ref.term(str(key or "")),
+                evidence=dict(evidence or {}),
+                confidence=confidence,
+            )
+        except ValueError as exc:
+            self.warnings.append(f"Ignoring asset.means({key!r}): {exc}")
+            return
+        payload = edge_to_payload(edge)
+        if payload not in self._means:
+            self._means.append(payload)
+
     def set_urn(self, urn: str) -> None:
         """Set the asset's cross-system identity, only if the connector left it empty.
 
@@ -337,6 +367,7 @@ class AugmentedAsset:
             "metadata": dict(self._added_metadata),
             "tags": dict(self._tags),
             "links": list(self._links),
+            "means": list(self._means),
             "urn": self._new_urn,
             "warnings": list(self.warnings),
         }
@@ -418,6 +449,7 @@ def build_augmentation_module(context: AugmentContext) -> types.ModuleType:
     module.flow = flow  # type: ignore[attr-defined]
     module.contains = contains  # type: ignore[attr-defined]
     module.references = references  # type: ignore[attr-defined]
+    module.means = means_edge  # type: ignore[attr-defined]
     module.same_as = same_as  # type: ignore[attr-defined]
     module.uses = uses  # type: ignore[attr-defined]
     module.urn_for = urn_for  # type: ignore[attr-defined]
@@ -439,6 +471,7 @@ def build_augmentation_module(context: AugmentContext) -> types.ModuleType:
         "contains",
         "ctx",
         "flow",
+        "means",
         "pages",
         "parse",
         "references",
@@ -470,6 +503,7 @@ def augmentation_namespace(context: AugmentContext) -> dict[str, Any]:
         "flow": flow,
         "contains": contains,
         "references": references,
+        "means": means_edge,
         "same_as": same_as,
         "uses": uses,
         "urn_for": urn_for,

{classifyre_cli-0.6.4 → classifyre_cli-0.6.5}/src/augmentation/session.py

@@ -463,6 +463,11 @@ class AugmentationSession:
 
         self._ingest_edges(patch, asset_hash=asset_hash or None)
 
+        # asset.means(): declarations, already built as MEANS edges by the SDK.
+        declared = patch.get("means")
+        if isinstance(declared, list) and declared:
+            self._ingest_edges({"edges": declared}, asset_hash=asset_hash or None)
+
         for message in patch.get("warnings") or []:
             self._warn(f"Augmentation on {getattr(asset, 'name', asset_hash)}: {message}")
 

{classifyre_cli-0.6.4 → classifyre_cli-0.6.5}/src/graph/__init__.py

@@ -13,6 +13,7 @@ from .edges import (
     UsageType,
     contains,
     flow,
+    means,
     references,
     same_as,
     uses,
@@ -31,6 +32,7 @@ __all__ = [
     "UsageType",
     "contains",
     "flow",
+    "means",
     "references",
     "same_as",
     "uses",

{classifyre_cli-0.6.4 → classifyre_cli-0.6.5}/src/graph/edges.py

@@ -35,7 +35,7 @@ from dataclasses import dataclass, field
 from enum import StrEnum
 from typing import Any, ClassVar
 
-from ..utils.urn import Urn
+from ..utils.urn import Urn, UrnError
 
 __all__ = [
     "ContainmentType",
@@ -189,6 +189,10 @@ class _RefMeta(type):
         "uri": "Ref.urn(<a urn like 'company://at/firmenbuchnummer/606601k'>)",
         "url": "Ref.urn(<a urn like 'company://at/firmenbuchnummer/606601k'>)",
         "external": "Ref.urn(<a urn naming the object in its own system>)",
+        "terms": "Ref.term(<a glossary key like 'gmbh'>)",
+        "concept": "Ref.term(<a glossary key like 'gmbh'>)",
+        "glossary": "Ref.term(<a glossary key like 'gmbh'>)",
+        "meaning": "Ref.term(<a glossary key like 'gmbh'>)",
     }
 
     def __getattr__(cls, name: str) -> Any:
@@ -196,7 +200,8 @@ class _RefMeta(type):
         hint = f" Did you mean {suggestion}?" if suggestion else ""
         raise AttributeError(
             f"Ref has no '{name}'. An edge endpoint is one of "
-            f"Ref.asset(<asset id>), Ref.urn(<urn>) or Ref.finding(<finding id>)."
+            f"Ref.asset(<asset id>), Ref.urn(<urn>), Ref.finding(<finding id>) "
+            f"or Ref.term(<glossary key>)."
             f"{hint}"
         )
 
@@ -232,6 +237,18 @@ class Ref(metaclass=_RefMeta):
             raise ValueError("Ref.finding() needs a finding id")
         return Ref("finding", value)
 
+    @staticmethod
+    def term(key: str) -> Ref:
+        """A glossary concept or entity by key: ``Ref("urn", "term://glossary/<key>")``.
+
+        The term need not exist yet: an unknown key is kept as a pending
+        reference and links once someone defines it.
+        """
+        try:
+            return Ref("urn", str(Urn.term(key)))
+        except UrnError as error:
+            raise ValueError(str(error)) from error
+
 
 @dataclass(frozen=True)
 class Edge:
@@ -450,6 +467,38 @@ def references(
     )
 
 
+#: Relation type of a meaning declaration (C12), class REFERENCE.
+MEANS = "MEANS"
+
+
+def means(
+    subject: Ref,
+    term: Ref | str,
+    *,
+    confidence: float | None = None,
+    evidence: dict[str, Any] | None = None,
+    method: Method = Method.SYSTEM_CATALOG,
+) -> Edge:
+    """``subject`` is about a glossary concept or entity (a DECLARED link).
+
+    ``subject`` is an asset, a URN or a finding; ``term`` is ``Ref.term(key)``
+    or the bare key. ``evidence`` says where the meaning came from, e.g.
+    ``{"field": "Rechtsform", "value": "GES"}``.
+    """
+    target = Ref.term(term) if isinstance(term, str) else term
+    if target.kind != "urn" or not target.value.startswith("term://"):
+        raise ValueError("means() points at a glossary term: pass Ref.term(<key>) or the key")
+    return Edge(
+        frm=subject,
+        to=target,
+        edge_class=EdgeClass.REFERENCE,
+        relation_type=MEANS,
+        method=method,
+        confidence=_confidence(method, confidence),
+        evidence=dict(evidence or {}),
+    )
+
+
 def uses(actor: Ref, target: Ref, *, type: UsageType = UsageType.ACCESSED) -> Edge:
     """``actor`` touched ``target``. Answers "who", not "where did this come from"."""
     return Edge(

{classifyre_cli-0.6.4 → classifyre_cli-0.6.5}/src/notebook/sdk.py

@@ -42,6 +42,7 @@ from ..graph.edges import (
     UsageType,
     contains,
     flow,
+    means,
     references,
     same_as,
     uses,
@@ -101,6 +102,112 @@ class Tag:
         object.__setattr__(self, "severity", normalized)
 
 
+#: What kind of named thing an entity is. Decides which finding labels its name
+#: is matched under: a PERSON is found among person names, never among
+#: organisations.
+ENTITY_TYPES = ("PERSON", "ORGANIZATION", "LOCATION", "REFERENCE", "OTHER")
+
+#: The asset-metadata key a declared entity travels under.
+ENTITY_METADATA_KEY = "_entity"
+
+
+@dataclass(frozen=True)
+class Entity:
+    """The named thing a record *is*: a company, a person, an account.
+
+    A register or a customer master does not merely mention ACME Holding GmbH --
+    its record is ACME. Saying so::
+
+        Asset(
+            id=fn,
+            urn=urn_for("firmenbuch", fn),
+            entity=Entity(
+                type="ORGANIZATION",
+                name="ACME Holding GmbH",
+                identifiers={"vat": "ATU12345678", "fn": "123456a"},
+                aliases=["ACME Holding"],
+            ),
+        )
+
+    creates the entity in the glossary (or updates it), anchored to this
+    record, and from then on every document in every other source that carries
+    that name or one of those identifiers is a mention of it -- the e-mail that
+    quotes the VAT number, the contract that names the company.
+
+    ``identifiers`` are keyed by the *label a detector finds them under*
+    (``iban``, ``email``, a custom detector's label): that label is what links
+    a finding to the entity, so an identifier under a label nothing detects
+    links nothing. ``key`` pins the glossary key, for a source that already
+    refers to the entity with ``Ref.term(key)``.
+
+    The asset needs a ``urn`` (or the entity a ``key``): it is what makes the
+    declaration the same entity on the next scan instead of a new one. A
+    declared entity is the register's word and is approved from the start; what
+    an operator changes afterwards is kept -- a later scan only adds.
+    """
+
+    name: str
+    type: str = "OTHER"
+    key: str | None = None
+    identifiers: Mapping[str, str] = field(default_factory=dict)
+    aliases: tuple[str, ...] | list[str] = field(default_factory=tuple)
+    attributes: Mapping[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        name = str(self.name or "").strip()
+        if not name:
+            raise ValueError("Entity.name is required and must be a non-empty string")
+        object.__setattr__(self, "name", name)
+        kind = str(self.type or "OTHER").strip().upper()
+        if kind not in ENTITY_TYPES:
+            raise ValueError(
+                f"Entity.type must be one of {', '.join(ENTITY_TYPES)}, got {self.type!r}"
+            )
+        object.__setattr__(self, "type", kind)
+        key = str(self.key).strip().lower() if self.key is not None else ""
+        object.__setattr__(self, "key", key or None)
+        if not isinstance(self.identifiers, Mapping):
+            raise TypeError(
+                'Entity.identifiers must map a label to a value, e.g. {"vat": "ATU12345678"}'
+            )
+        object.__setattr__(
+            self,
+            "identifiers",
+            {
+                str(label).strip(): str(value).strip()
+                for label, value in self.identifiers.items()
+                if str(label).strip() and value is not None and str(value).strip()
+            },
+        )
+        if isinstance(self.aliases, str | bytes):
+            raise TypeError("Entity.aliases must be a list of names, not a single string")
+        object.__setattr__(
+            self,
+            "aliases",
+            tuple(
+                dict.fromkeys(str(alias).strip() for alias in self.aliases if str(alias).strip())
+            ),
+        )
+        if not isinstance(self.attributes, Mapping):
+            raise TypeError("Entity.attributes must be a mapping")
+        object.__setattr__(self, "attributes", dict(self.attributes))
+
+    def to_metadata(self) -> dict[str, Any]:
+        """The declaration as it travels on the asset. Empty parts are left out,
+        so adding a field here never changes the checksum of an asset that does
+        not use it."""
+        payload: dict[str, Any] = {"name": self.name, "type": self.type}
+        if self.key:
+            payload["key"] = self.key
+        if self.identifiers:
+            payload["identifiers"] = dict(sorted(self.identifiers.items()))
+        if self.aliases:
+            payload["aliases"] = list(self.aliases)
+        if self.attributes:
+            payload["attributes"] = dict(self.attributes)
+        return payload
+
+
 @dataclass
 class Asset:
     """One thing worth scanning.
@@ -168,6 +275,10 @@ class Asset:
     #: chose it (``metadata["_cohort"]``), which is how the platform measures
     #: each band's yield. Never part of the asset's checksum.
     cohort: CohortItem | None = None
+    #: The named thing this record *is* -- see :class:`Entity`. Recorded as
+    #: ``metadata["_entity"]``, so it is part of the checksum only for assets
+    #: that declare one. Needs ``urn`` (or ``Entity.key``).
+    entity: Entity | None = None
 
     def __post_init__(self) -> None:
         self.id = str(self.id).strip()
@@ -206,6 +317,17 @@ class Asset:
                     "key": self.cohort.key,
                 },
             }
+        if self.entity is not None:
+            if not isinstance(self.entity, Entity):
+                raise TypeError("Asset.entity must be an Entity(...)")
+            if not (self.urn and str(self.urn).strip()) and not self.entity.key:
+                # Without either, the next scan could not tell this is the same
+                # entity and would create it again.
+                raise ValueError(
+                    "Asset.entity needs Asset.urn (build one with urn_for(...)) or "
+                    "Entity.key, so the next scan updates the same entity"
+                )
+            self.metadata = {**self.metadata, ENTITY_METADATA_KEY: self.entity.to_metadata()}
         if not isinstance(self.extract, bool):
             raise TypeError(f"Asset.extract must be True or False (got {self.extract!r})")
         reason = str(self.reference_reason).strip() if self.reference_reason is not None else ""
@@ -843,6 +965,8 @@ def build_module(context: Context) -> types.ModuleType:
     module.flow = flow  # type: ignore[attr-defined]
     module.contains = contains  # type: ignore[attr-defined]
     module.references = references  # type: ignore[attr-defined]
+    module.means = means  # type: ignore[attr-defined]
+    module.Entity = Entity  # type: ignore[attr-defined]
     module.same_as = same_as  # type: ignore[attr-defined]
     module.uses = uses  # type: ignore[attr-defined]
     module.urn_for = urn_for  # type: ignore[attr-defined]
@@ -860,6 +984,7 @@ def build_module(context: Context) -> types.ModuleType:
         "Asset",
         "Context",
         "ContainmentType",
+        "Entity",
         "FieldMapping",
         "FieldTransform",
         "FlowType",
@@ -873,6 +998,7 @@ def build_module(context: Context) -> types.ModuleType:
         "contains",
         "ctx",
         "flow",
+        "means",
         "pages",
         "parse",
         "references",
@@ -903,6 +1029,7 @@ def namespace(context: Context) -> dict[str, Any]:
         "__builtins__": __builtins__,
         "Asset": Asset,
         "Tag": Tag,
+        "Entity": Entity,
         "Ref": Ref,
         "FieldMapping": FieldMapping,
         "FlowType": FlowType,
@@ -910,6 +1037,7 @@ def namespace(context: Context) -> dict[str, Any]:
         "flow": flow,
         "contains": contains,
         "references": references,
+        "means": means,
         "same_as": same_as,
         "uses": uses,
         "urn_for": urn_for,
@@ -959,7 +1087,7 @@ def iter_relationships(value: Any) -> Iterable[Edge]:
         else:
             raise TypeError(
                 "relationships() must yield edges built with flow(), contains(), "
-                "references(), same_as() or uses() -- got "
+                "references(), means(), same_as() or uses() -- got "
                 f"{type(item).__name__}. These are not interchangeable: only "
                 "flow() is lineage, and only it answers 'what breaks if this changes'."
             )

{classifyre_cli-0.6.4 → classifyre_cli-0.6.5}/src/utils/urn.py

@@ -20,11 +20,22 @@ character for character -- a test in each language pins the same table of cases.
 from __future__ import annotations
 
 import logging
+import re
 from dataclasses import dataclass
 from enum import StrEnum
 from typing import Any
 
-__all__ = ["CasePolicy", "Urn", "UrnError", "normalize_urn", "normalize_urn_or_none"]
+__all__ = [
+    "TERM_KEY_PATTERN",
+    "CasePolicy",
+    "Urn",
+    "UrnError",
+    "normalize_urn",
+    "normalize_urn_or_none",
+]
+
+#: A glossary term key (contract C8), the same pattern as the API's.
+TERM_KEY_PATTERN = re.compile(r"[a-z0-9][a-z0-9._-]{0,99}")
 
 
 logger = logging.getLogger(__name__)
@@ -95,6 +106,10 @@ _register(_Platform("powerbi"))
 _register(_Platform("kafka"))
 _register(_Platform("file"))
 
+# Glossary terms (contract C8): ``term://glossary/<key>``. Keys are lower-case
+# by definition, so both halves fold.
+_register(_Platform("term", path_case=CasePolicy.LOWER))
+
 _DEFAULT_PLATFORM = _Platform("", authority_case=CasePolicy.LOWER, path_case=CasePolicy.PRESERVE)
 
 
@@ -254,6 +269,18 @@ class Urn:
     def kafka(cls, bootstrap: str, topic: str) -> Urn:
         return cls.of("kafka", bootstrap, topic)
 
+    @classmethod
+    def term(cls, key: str) -> Urn:
+        """A glossary term by its key (C8): ``term://glossary/<key>``."""
+        value = (key or "").strip().lower()
+        if not TERM_KEY_PATTERN.fullmatch(value):
+            raise UrnError(
+                f"Not a glossary term key: {key!r}. Keys are lower-case letters, digits, "
+                "'.', '_' and '-', starting with a letter or digit, at most 100 characters "
+                "(e.g. 'gmbh', 'bank-account-iban')."
+            )
+        return cls.of("term", "glossary", value)
+
 
 def _host_port(host: str, port: int | str | None) -> str:
     """``host:port``, or bare host when no port was configured."""

{classifyre_cli-0.6.4 → classifyre_cli-0.6.5}/tests/test_augmentation_sdk.py

@@ -72,6 +72,7 @@ def test_empty_writes_are_warnings_not_errors() -> None:
         "metadata": {},
         "tags": {},
         "links": [],
+        "means": [],
         "urn": None,
         "warnings": asset.warnings,
     }

classifyre_cli-0.6.5/tests/test_notebook_asset_entity.py

@@ -0,0 +1,82 @@
+"""A record that *is* a named thing declares it (G5 R12)."""
+
+from __future__ import annotations
+
+import pytest
+
+from src.notebook.sdk import ENTITY_METADATA_KEY, Asset, Context, Entity, build_module, namespace
+
+
+def test_entity_is_importable_from_classifyre() -> None:
+    module = build_module(Context())
+    assert module.Entity is Entity
+    assert "Entity" in module.__all__
+    assert namespace(Context())["Entity"] is Entity
+
+
+def test_a_declared_entity_travels_in_the_asset_metadata() -> None:
+    asset = Asset(
+        id="123456a",
+        urn="firmenbuch://at/fn/123456a",
+        entity=Entity(
+            type="organization",
+            name=" ACME Holding GmbH ",
+            identifiers={"vat": "ATU12345678", "fn": "123456a"},
+            aliases=["ACME Holding", "ACME Holding", " "],
+        ),
+    )
+
+    assert asset.metadata[ENTITY_METADATA_KEY] == {
+        "name": "ACME Holding GmbH",
+        "type": "ORGANIZATION",
+        # Sorted, so the same declaration always hashes to the same checksum.
+        "identifiers": {"fn": "123456a", "vat": "ATU12345678"},
+        "aliases": ["ACME Holding"],
+    }
+
+
+def test_an_asset_without_an_entity_keeps_its_metadata_untouched() -> None:
+    # The declaration joins the checksum basis only when set, so existing
+    # corpora are not re-checksummed for a feature they do not use.
+    asset = Asset(id="a", metadata={"k": "v"})
+    assert asset.metadata == {"k": "v"}
+
+
+def test_the_notebooks_own_metadata_is_kept() -> None:
+    asset = Asset(
+        id="a",
+        urn="crm://customer/1",
+        metadata={"segment": "retail"},
+        entity=Entity(name="Jane Doe", type="PERSON"),
+    )
+    assert asset.metadata["segment"] == "retail"
+    assert asset.metadata[ENTITY_METADATA_KEY] == {"name": "Jane Doe", "type": "PERSON"}
+
+
+def test_a_key_stands_in_for_a_urn() -> None:
+    asset = Asset(id="a", entity=Entity(name="ACME", key="ACME-Holding"))
+    assert asset.metadata[ENTITY_METADATA_KEY]["key"] == "acme-holding"
+
+
+def test_an_entity_needs_a_urn_or_a_key() -> None:
+    # Without either the next scan would create the entity again.
+    with pytest.raises(ValueError, match="urn"):
+        Asset(id="a", entity=Entity(name="ACME"))
+
+
+def test_blank_identifiers_are_dropped() -> None:
+    entity = Entity(name="ACME", identifiers={"vat": " ", "": "x", "iban": " AT61 "})
+    assert entity.identifiers == {"iban": "AT61"}
+
+
+def test_bad_shapes_are_rejected_with_what_was_wanted() -> None:
+    with pytest.raises(ValueError, match="non-empty"):
+        Entity(name="  ")
+    with pytest.raises(ValueError, match="one of"):
+        Entity(name="ACME", type="COMPANY")
+    with pytest.raises(TypeError, match="label"):
+        Entity(name="ACME", identifiers=["ATU12345678"])  # type: ignore[arg-type]
+    with pytest.raises(TypeError, match="list of names"):
+        Entity(name="ACME", aliases="ACME Holding")
+    with pytest.raises(TypeError, match="Entity"):
+        Asset(id="a", urn="x://a", entity={"name": "ACME"})  # type: ignore[arg-type]