boarddata 6.6.2__tar.gz → 6.7.1__tar.gz

{boarddata-6.6.2 → boarddata-6.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boarddata
-Version: 6.6.2
+Version: 6.7.1
 Summary: Python SDK for the BoardData V2 REST API
 License: MIT
 Classifier: Development Status :: 4 - Beta
@@ -21,6 +21,9 @@ Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: mypy; extra == "dev"
+Provides-Extra: dq
+Requires-Dist: psycopg[binary]>=3; extra == "dq"
+Requires-Dist: boto3; extra == "dq"
 
 # boarddata
 

{boarddata-6.6.2 → boarddata-6.7.1}/__init__.py

@@ -6,7 +6,7 @@ from .cache import FileTokenCache, ScalewaySecretTokenCache
 from .client import BoardDataClient
 from .errors import BoardDataError, ExportTooLargeError
 
-__version__ = "6.6.2"
+__version__ = "6.7.1"
 __all__ = [
     "Base",
     "BoardDataClient",

{boarddata-6.6.2 → boarddata-6.7.1}/_criteria.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 from typing import Any
 
-from .types.criteria import CriteriaCatalogItem, CriteriaThemeItem
+from .types.criteria import CriteriaCatalogItem, CriteriaFamilyItem, CriteriaThemeItem
 
 
 class CriteriaMixin:
@@ -25,6 +25,19 @@ class CriteriaMixin:
         """
         return self._get("criteria-themes/")  # type: ignore[attr-defined]
 
+    def list_criteria_families(self) -> list[CriteriaFamilyItem]:
+        """List the 17 criteria families (business-meaning classification).
+
+        Returns:
+            List of criteria families in ``sort_order`` (not paginated).
+            Slugs are a cross-repo contract — extraction committers
+            preflight their families.json against this list.
+
+        Raises:
+            BoardDataError: On non-2xx API response.
+        """
+        return self._get("criteria-families/")  # type: ignore[attr-defined]
+
     def list_criteria(
         self,
         theme: str | None = None,

{boarddata-6.6.2 → boarddata-6.7.1}/boarddata.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boarddata
-Version: 6.6.2
+Version: 6.7.1
 Summary: Python SDK for the BoardData V2 REST API
 License: MIT
 Classifier: Development Status :: 4 - Beta
@@ -21,6 +21,9 @@ Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: mypy; extra == "dev"
+Provides-Extra: dq
+Requires-Dist: psycopg[binary]>=3; extra == "dq"
+Requires-Dist: boto3; extra == "dq"
 
 # boarddata
 

{boarddata-6.6.2 → boarddata-6.7.1}/boarddata.egg-info/SOURCES.txt

@@ -42,11 +42,55 @@ pyproject.toml
 ./client.py
 ./errors.py
 ./py.typed
+./dq/__init__.py
+./dq/adjudicate.py
+./dq/apply.py
+./dq/context.py
+./dq/db.py
+./dq/domains.py
+./dq/models.py
+./dq/oracle.py
+./dq/registry.py
+./dq/report.py
+./dq/s3.py
+./dq/verify_s3.py
+./dq/rules/__init__.py
+./dq/rules/cross_company.py
+./dq/rules/lexical.py
+./scripts/__init__.py
+./scripts/dq_apply.py
+./scripts/dq_batch.py
+./scripts/dq_plan.py
+./scripts/dq_revert.py
+./scripts/dq_scan.py
+./scripts/dq_verify_s3.py
 boarddata.egg-info/PKG-INFO
 boarddata.egg-info/SOURCES.txt
 boarddata.egg-info/dependency_links.txt
 boarddata.egg-info/requires.txt
 boarddata.egg-info/top_level.txt
+dq/__init__.py
+dq/adjudicate.py
+dq/apply.py
+dq/context.py
+dq/db.py
+dq/domains.py
+dq/models.py
+dq/oracle.py
+dq/registry.py
+dq/report.py
+dq/s3.py
+dq/verify_s3.py
+dq/rules/__init__.py
+dq/rules/cross_company.py
+dq/rules/lexical.py
+scripts/__init__.py
+scripts/dq_apply.py
+scripts/dq_batch.py
+scripts/dq_plan.py
+scripts/dq_revert.py
+scripts/dq_scan.py
+scripts/dq_verify_s3.py
 tests/__init__.py
 tests/conftest.py
 tests/test_assemblies.py

{boarddata-6.6.2 → boarddata-6.7.1}/boarddata.egg-info/requires.txt

@@ -7,3 +7,7 @@ typing_extensions>=4.0
 pytest>=7.0
 ruff
 mypy
+
+[dq]
+psycopg[binary]>=3
+boto3

boarddata-6.7.1/dq/__init__.py

@@ -0,0 +1 @@
+"""Document data-quality detection & remediation toolkit."""

boarddata-6.7.1/dq/adjudicate.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+from .context import ScanContext
+from .domains import domain_matches, host_of, is_shared_source
+from .models import DocFinding, FixAction
+from .oracle import file_key, is_s3_key
+
+
+def _owner_has_own_copy(ctx: ScanContext, finding: DocFinding, owner_cid: str) -> str | None:
+    """Return the owner's own document id for the same file, if any."""
+    target_key = file_key({"url": finding.url}, ctx)
+    for d in ctx.iter_documents():
+        if d["company_id"] == owner_cid and file_key(d, ctx) == target_key and d["id"] != finding.document_id:
+            return d["id"]
+    return None
+
+
+def adjudicate(finding: DocFinding, ctx: ScanContext) -> FixAction:
+    owner = finding.true_owner_guess
+    current_company = ctx.company(finding.company_id) if finding.company_id else None
+    before = {
+        "company_id": finding.company_id,
+        # ISIN is not readable back from the document endpoint, so the executor
+        # relies on the plan carrying it for repoint-revert / delete-recreate.
+        "isin": current_company["isin"] if current_company else None,
+        "url": finding.url,
+        "assembly_links": ctx.assembly_links(finding.document_id),
+        "field_source_count": ctx.field_source_count(finding.document_id),
+    }
+    ev = {"rule": finding.rule, "true_owner": owner}
+    if finding.url and not is_s3_key(finding.url) and is_shared_source(host_of(finding.url)):
+        # Platform/shared-source hosts (e.g. live.euronext.com) are
+        # legitimately cross-company REGARDLESS of owner inference. Without
+        # this top-level guard, a host that matches a real company's own
+        # domain (domain_matches("live.euronext.com", "euronext.com") is
+        # True) can get an inferred owner and fall into the owner-known
+        # branch below, which would emit delete/repoint against a
+        # co-attached issuer -- violating the do-not-touch guarantee for
+        # shared/platform sources. Guarded to web URLs only (S3-key docs are
+        # never contaminated by a web host).
+        return FixAction(finding.document_id, "keep", None, before, ev, confidence="low")
+    if not owner:
+        # No inferable owner. If this is a web-hosted (non-S3) doc whose host
+        # is not an allowlisted shared/platform source, and the company's own
+        # (known) domain positively contradicts that host, and the row powers
+        # no FieldSources, it's a pure contaminant -> cascade-delete it.
+        # Deletion requires a *known* domain to disprove attribution against;
+        # an unknown domain can never justify a delete (fall through to keep).
+        url = finding.url
+        if url and not is_s3_key(url):
+            host = host_of(url)
+            if not is_shared_source(host):
+                dom = ctx.domain_for_company(finding.company_id) if finding.company_id else None
+                if dom and not domain_matches(host, dom) and before["field_source_count"] == 0:
+                    return FixAction(finding.document_id, "delete", None, before, ev, confidence="medium")
+        return FixAction(finding.document_id, "keep", None, before, ev, confidence="low")
+
+    owner_cid = owner["company_id"]
+    owner_copy = _owner_has_own_copy(ctx, finding, owner_cid)
+    fs = ctx.field_source_count(finding.document_id)
+
+    if fs > 0:
+        # Row powers provenance fields -> reparent into the owner's copy if it
+        # exists, else repoint the FK so the fields keep a source.
+        if owner_copy:
+            return FixAction(finding.document_id, "merge", owner_copy, before, ev, "high")
+        return FixAction(finding.document_id, "repoint", owner_cid, before, ev, "medium")
+    if owner_copy:
+        return FixAction(finding.document_id, "delete", None, before, ev, "high")
+    return FixAction(finding.document_id, "repoint", owner_cid, before, ev, "medium")

boarddata-6.7.1/dq/apply.py

@@ -0,0 +1,171 @@
+"""Execute a FixPlan against the BoardData REST API, recording an UndoManifest.
+
+This is the write side of the toolkit (``dq_apply`` / ``dq_revert``). Every
+mutation goes through the SDK — ``delete_document`` / ``update_document`` /
+``create_document`` — so it runs through backend validation, permissions and
+the pghistory audit trail. There is no direct DB write anywhere here.
+
+Reversibility, by action:
+  * ``repoint`` — full. Restore the document's previous ISIN.
+  * ``delete``  — partial. The document row is recreated from a full pre-delete
+    snapshot, but its assembly↔document link(s) cannot be restored through the
+    current API (the link is read-only / has no create endpoint). The links are
+    recorded in the manifest so the gap is explicit, never silent.
+  * ``merge``   — refused. Destructive and not yet revertible; ``apply`` raises
+    rather than perform an irreversible operation.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Protocol
+
+from .models import FixAction, FixPlan, UndoManifest
+
+# Content fields we snapshot from get_document before a mutation and replay on
+# revert. NOTE: DocumentDetail exposes ``company_name``, not ``isin`` — the
+# company identity (ISIN) is not readable back from the document endpoint, so it
+# must travel in the plan's before_state (see apply_action).
+_DOC_FIELDS = (
+    "title",
+    "document_type",
+    "publication_date",
+    "fiscal_year",
+    "language",
+    "url",
+    "original_url",
+)
+
+
+class ApplyError(RuntimeError):
+    """A FixAction could not be applied or reverted safely."""
+
+
+# Actions apply_plan will actually execute. Kept in sync with apply_action's
+# own dispatch (merge/unknown raise there too, as a defensive backstop) --
+# this set is what the plan-level pre-flight in apply_plan checks against.
+_APPLIABLE_ACTIONS = {"keep", "delete", "repoint"}
+
+
+class FixClient(Protocol):
+    """The slice of ``BoardDataClient`` the executor depends on."""
+
+    def get_document(self, uid: str) -> dict[str, Any]: ...
+    def create_document(self, title: str, document_type: str, **kwargs: Any) -> dict[str, Any]: ...
+    def update_document(self, uid: str, **kwargs: Any) -> dict[str, Any]: ...
+    def delete_document(self, uid: str, *, cascade_field_sources: bool = False) -> None: ...
+    def merge_document(self, uid: str, *, into: str) -> dict[str, Any]: ...
+
+
+def _snapshot(client: FixClient, uid: str) -> dict[str, Any]:
+    """Full field snapshot of a document, taken before it is mutated."""
+    doc = client.get_document(uid)
+    return {k: doc.get(k) for k in _DOC_FIELDS}
+
+
+def apply_action(action: FixAction, client: FixClient) -> dict[str, Any] | None:
+    """Execute one action and return its UndoManifest entry (None for no-op)."""
+    if action.action == "keep":
+        return None
+
+    doc_id = action.document_id
+    before = _snapshot(client, doc_id)  # content fields, captured before any mutation
+    # ISIN is not returned by get_document; the plan carries it in before_state.
+    before["isin"] = action.before_state.get("isin")
+
+    if action.action == "delete":
+        client.delete_document(doc_id)
+        return {
+            "action": "delete",
+            "document_id": doc_id,
+            "before": before,
+            # links are gone (backend cascade) and not API-restorable — record them
+            "assembly_links": list(action.before_state.get("assembly_links", [])),
+            "reversible": "partial",
+        }
+
+    if action.action == "repoint":
+        if not action.before_state.get("isin"):
+            # No origin ISIN captured -> revert_entry's update_document(isin=None)
+            # would send an empty PATCH and silently fail to restore it, even
+            # though the manifest would claim reversible="full". Refuse rather
+            # than record a revert path that doesn't actually work.
+            raise ApplyError(f"repoint {doc_id}: refusing — origin ISIN unknown, revert could not restore it")
+        target_isin = (action.evidence.get("true_owner") or {}).get("isin")
+        if not target_isin:
+            raise ApplyError(f"repoint {doc_id}: no target ISIN in evidence.true_owner")
+        client.update_document(doc_id, isin=target_isin)
+        return {
+            "action": "repoint",
+            "document_id": doc_id,
+            "before": before,  # carries the previous ISIN
+            "to_isin": target_isin,
+            "reversible": "full",
+        }
+
+    if action.action == "merge":
+        raise ApplyError(f"merge {doc_id}: refused — merge is destructive and not yet revertible by dq_revert")
+
+    raise ApplyError(f"unknown action {action.action!r} for document {doc_id}")
+
+
+def apply_plan(plan: FixPlan, client: FixClient, *, applied_at: str) -> UndoManifest:
+    """Apply every action in ``plan``, returning an UndoManifest for dq_revert.
+
+    Pre-flight: reject the WHOLE plan atomically if any action is a ``merge``
+    or an unrecognized action, before any mutation is issued. Without this, a
+    plan like [delete, merge] would apply the delete against prod and only
+    then raise on the merge -- leaving an unrevertable mutation with no
+    manifest ever written (run_apply writes manifest_<stamp>.json only after
+    apply_plan returns successfully).
+    """
+    for action in plan.actions:
+        if action.action not in _APPLIABLE_ACTIONS:
+            raise ApplyError(
+                f"plan rejected before any mutation: action {action.action!r} for document "
+                f"{action.document_id!r} is not directly applicable (merge is refused as "
+                "destructive/unrevertible; unknown actions are rejected)"
+            )
+
+    manifest = UndoManifest(plan_ref=plan.source_scan, applied_at=applied_at, entries=[])
+    for action in plan.actions:
+        entry = apply_action(action, client)
+        if entry is not None:
+            manifest.entries.append(entry)
+    return manifest
+
+
+def revert_entry(entry: dict[str, Any], client: FixClient) -> dict[str, Any]:
+    """Undo a single manifest entry."""
+    act = entry["action"]
+    doc_id = entry["document_id"]
+    before = entry.get("before", {})
+
+    if act == "repoint":
+        client.update_document(doc_id, isin=before.get("isin"))
+        return {"document_id": doc_id, "reverted": "repoint", "status": "ok"}
+
+    if act == "delete":
+        new = client.create_document(
+            title=before.get("title") or "",
+            document_type=before.get("document_type") or "other",
+            publication_date=before.get("publication_date"),
+            fiscal_year=before.get("fiscal_year"),
+            isin=before.get("isin"),
+            language=before.get("language"),
+            url=before.get("url"),
+            original_url=before.get("original_url"),
+        )
+        return {
+            "document_id": doc_id,
+            "reverted": "delete",
+            "status": "partial",  # doc recreated (new id); assembly links not restored
+            "new_document_id": new.get("id"),
+            "unrestored_assembly_links": list(entry.get("assembly_links", [])),
+        }
+
+    raise ApplyError(f"cannot revert action {act!r} for document {doc_id}")
+
+
+def revert(manifest: UndoManifest, client: FixClient) -> list[dict[str, Any]]:
+    """Undo every entry, most-recent first."""
+    return [revert_entry(e, client) for e in reversed(manifest.entries)]

boarddata-6.7.1/dq/context.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from typing import Any, Protocol, runtime_checkable
+
+
+@runtime_checkable
+class ScanContext(Protocol):
+    """Read-only query surface the rules depend on."""
+
+    def url_groups_multi_company(self) -> list[dict[str, Any]]:
+        """Groups of documents sharing an exact URL across >1 company.
+        Each: {url, companies: [{document_id, company_id}]}."""
+
+    def hash_groups_multi_company(self) -> list[dict[str, Any]]:
+        """Groups of S3 docs sharing sha256 across >1 company.
+        Each: {sha256, companies: [{document_id, company_id, url}]}."""
+
+    def iter_documents(self, isin: str | None = None) -> list[dict[str, Any]]:
+        """Documents (optionally one company); rows: {id, url, title,
+        document_type, company_id, fiscal_year}."""
+
+    def company(self, company_id: str | None) -> dict[str, Any] | None:
+        """{id, isin, name, website, deleted_at} or None."""
+
+    def company_by_isin(self, isin: str) -> dict[str, Any] | None: ...
+    def all_companies(self) -> list[dict[str, Any]]: ...
+    def assembly_links(self, document_id: str) -> list[dict[str, Any]]:
+        """[{assembly_id, assembly_company_id, category, doc_type}]."""
+
+    def field_source_count(self, document_id: str) -> int: ...
+    def client_visible_ids(self) -> set[str]:
+        """Doc ids with >=1 assembly link OR >=1 field source, computed in bulk.
+
+        Batched equivalent of ``bool(assembly_links(id)) or field_source_count(id) > 0``
+        so whole-fleet rules do not issue two queries per document."""
+
+    def domain_for_company(self, company_id: str) -> str | None: ...
+    def s3_meta(self, url: str) -> dict[str, Any] | None:
+        """{sha256, size_bytes} for an S3-key url, else None."""

boarddata-6.7.1/dq/db.py

@@ -0,0 +1,155 @@
+from __future__ import annotations
+
+from typing import Any
+
+import psycopg
+from psycopg.conninfo import conninfo_to_dict
+
+from .context import ScanContext  # noqa: F401  (documents the interface)
+from .domains import host_of
+from .s3 import S3Reader
+
+URL_GROUPS_SQL = """
+SELECT url, json_agg(json_build_object('document_id', id::text,
+                                       'company_id', company_id::text)) AS companies
+FROM documents_document
+WHERE url <> '' AND company_id IS NOT NULL
+GROUP BY url
+HAVING count(DISTINCT company_id) > 1
+"""
+
+HASH_MEMBERS_SQL = """
+SELECT id::text AS document_id, company_id::text AS company_id, url
+FROM documents_document
+WHERE url ~ '^(s3://[^/]+/)?[A-Za-z0-9]+/[0-9]{4}/' AND company_id IS NOT NULL
+"""
+
+DOC_ALL_SQL = """
+SELECT id::text, url, title, document_type, company_id::text, fiscal_year
+FROM documents_document
+"""
+
+# One company. Kept separate from DOC_ALL_SQL because a single "%(company_id)s
+# IS NULL OR ..." parameter is type-ambiguous to Postgres on the NULL path
+# (`could not determine data type of parameter $1`).
+DOC_BY_COMPANY_SQL = DOC_ALL_SQL + "WHERE company_id = %(company_id)s"
+
+ASSEMBLY_LINKS_SQL = """
+SELECT ad.assembly_id::text, a.company_id::text AS assembly_company_id,
+       ad.category, ad.doc_type
+FROM assemblies_assemblydocument ad
+JOIN assemblies_assembly a ON a.id = ad.assembly_id
+WHERE ad.document_id = %(doc_id)s
+"""
+
+FIELD_SOURCE_COUNT_SQL = "SELECT count(*) FROM core_fieldsource WHERE document_id = %(doc_id)s"
+
+CLIENT_VISIBLE_IDS_SQL = """
+SELECT document_id::text AS id FROM assemblies_assemblydocument WHERE document_id IS NOT NULL
+UNION
+SELECT document_id::text AS id FROM core_fieldsource WHERE document_id IS NOT NULL
+"""
+
+COMPANIES_SQL = "SELECT id::text, isin, name, website, deleted_at FROM companies_company"
+
+
+def _require_local_host(dsn: str) -> None:
+    """Reject any DSN that could connect somewhere other than 127.0.0.1.
+
+    Parses via libpq (handles both URI and keyword/value DSNs) and checks BOTH
+    ``host`` and ``hostaddr``: a URI like ``...@127.0.0.1/db?hostaddr=10.0.0.5``
+    passes a naive host check yet libpq connects to the hostaddr.
+    """
+    try:
+        info = conninfo_to_dict(dsn)
+    except Exception as exc:  # unparseable DSN -> refuse
+        raise ValueError(f"Unparseable DSN: {exc}") from exc
+    host, hostaddr = info.get("host"), info.get("hostaddr")
+    if not host and not hostaddr:
+        raise ValueError("DSN must pin host=127.0.0.1 (the operator tunnel); none given.")
+    for label, value in (("host", host), ("hostaddr", hostaddr)):
+        if value and value != "127.0.0.1":
+            raise ValueError(
+                f"Refusing DSN {label} {value!r}: connect to the operator tunnel via "
+                f"127.0.0.1 (IPv4) only, never localhost/remote."
+            )
+
+
+def _domain_from_website(website: str | None) -> str | None:
+    """Authoritative domain from a stored website, tolerating a missing scheme."""
+    if not website:
+        return None
+    return host_of(website) or None
+
+
+class DbScanContext:
+    def __init__(self, dsn: str, s3reader: S3Reader, domain_overrides: dict[str, str] | None = None):
+        _require_local_host(dsn)
+        self._conn = psycopg.connect(dsn, row_factory=psycopg.rows.dict_row)
+        self._s3 = s3reader
+        self._companies = {c["id"]: c for c in self._q(COMPANIES_SQL)}
+        self._by_isin = {c["isin"]: c for c in self._companies.values() if c["isin"]}
+        self._domain_overrides = domain_overrides or {}
+        self._all_docs: list[dict[str, Any]] | None = None
+        self._cv_ids: set[str] | None = None
+
+    def _q(self, sql: str, params: dict[str, Any] | None = None) -> list[dict[str, Any]]:
+        with self._conn.cursor() as cur:
+            cur.execute(sql, params or {})
+            return cur.fetchall()
+
+    def url_groups_multi_company(self) -> list[dict[str, Any]]:
+        return [{"url": r["url"], "companies": r["companies"]} for r in self._q(URL_GROUPS_SQL)]
+
+    def hash_groups_multi_company(self) -> list[dict[str, Any]]:
+        from collections import defaultdict
+
+        buckets: dict[str, list[dict[str, Any]]] = defaultdict(list)
+        for r in self._q(HASH_MEMBERS_SQL):
+            meta = self._s3.meta(r["url"])
+            if meta and meta.get("sha256"):
+                buckets[meta["sha256"]].append(r)
+        return [{"sha256": h, "companies": m} for h, m in buckets.items() if len({x["company_id"] for x in m}) > 1]
+
+    def iter_documents(self, isin: str | None = None) -> list[dict[str, Any]]:
+        if isin is None:
+            # Read-only snapshot: cache the full table so repeated whole-fleet
+            # passes (rules R3-R5, adjudicate per finding) do not re-query it.
+            if self._all_docs is None:
+                self._all_docs = self._q(DOC_ALL_SQL)
+            return self._all_docs
+        company_id = self._by_isin.get(isin, {}).get("id")
+        if company_id is None:  # unknown ISIN -> empty, never the whole table
+            return []
+        return self._q(DOC_BY_COMPANY_SQL, {"company_id": company_id})
+
+    def company(self, company_id: str | None) -> dict[str, Any] | None:
+        return self._companies.get(company_id)
+
+    def company_by_isin(self, isin: str) -> dict[str, Any] | None:
+        return self._by_isin.get((isin or "").upper()) or self._by_isin.get(isin)
+
+    def all_companies(self) -> list[dict[str, Any]]:
+        return list(self._companies.values())
+
+    def assembly_links(self, document_id: str) -> list[dict[str, Any]]:
+        return self._q(ASSEMBLY_LINKS_SQL, {"doc_id": document_id})
+
+    def field_source_count(self, document_id: str) -> int:
+        return self._q(FIELD_SOURCE_COUNT_SQL, {"doc_id": document_id})[0]["count"]
+
+    def client_visible_ids(self) -> set[str]:
+        # One UNION query for the whole fleet, memoized: replaces two per-doc
+        # queries that made whole-fleet rules (R3/R5) O(2N) round-trips.
+        if self._cv_ids is None:
+            self._cv_ids = {r["id"] for r in self._q(CLIENT_VISIBLE_IDS_SQL)}
+        return self._cv_ids
+
+    def domain_for_company(self, company_id: str) -> str | None:
+        if company_id in self._domain_overrides:
+            return self._domain_overrides[company_id]
+        c = self._companies.get(company_id)
+        return _domain_from_website(c.get("website")) if c else None
+
+    def s3_meta(self, url: str) -> dict[str, Any] | None:
+        return self._s3.meta(url)

boarddata-6.7.1/dq/domains.py

@@ -0,0 +1,90 @@
+from __future__ import annotations
+
+from urllib.parse import urlparse
+
+# Official filing/regulator/association sources shared verbatim by many issuers
+# (BALO, AMF, MEDEF, FIR). A doc hosted here is legitimately cross-company.
+SHARED_SOURCE_DOMAINS = frozenset(
+    {
+        "journal-officiel.gouv.fr",  # BALO
+        "amf-france.org",  # AMF
+        "medef.com",  # MEDEF
+        "frenchsif.org",  # FIR
+    }
+)
+
+# Platform/newswire hosts that carry many issuers' notices/press releases.
+# NOTE: "live.euronext.com" is the platform host (exact) — the bare
+# "euronext.com" is NOT allowlisted, since that is a company's own domain.
+PLATFORM_DOMAINS = frozenset(
+    {
+        "live.euronext.com",
+        "globenewswire.com",
+        "businesswire.com",
+    }
+)
+
+# Trailing hyphenated markers on a company's finance/investor-relations
+# subdomain, e.g. "loreal-finance.com" for L'Oréal's investor site.
+_FINANCE_SUBDOMAIN_MARKERS = ("-finance", "-bourse", "-invest", "-corp")
+
+# Generic second-level labels that must NOT be treated as a company identity
+# after stripping a finance marker. If stripping results in one of these,
+# return the original label unchanged to avoid false "same owner" matches.
+_GENERIC_SLDS = frozenset(
+    {"invest", "corp", "finance", "bourse", "group", "groupe", "holding", "capital", "bank", "asset"}
+)
+
+# Union of all allowlisted domains for shared sources and platforms.
+# Used in is_shared_source to check if a host is within an allowlisted domain.
+_ALLOWED_HOST_DOMAINS = SHARED_SOURCE_DOMAINS | PLATFORM_DOMAINS
+
+
+def host_of(url: str) -> str:
+    netloc = urlparse(url if "//" in url else "//" + url).netloc.lower()
+    return netloc[4:] if netloc.startswith("www.") else netloc
+
+
+def is_shared_source(host: str) -> bool:
+    """True when `host` is (or is a subdomain of) an allowlisted shared-source
+    or platform/newswire domain. Directional: the host must be *within* the
+    allowlisted domain, not the reverse — a short allowlist entry must not
+    over-match an unrelated longer host (see domain_matches' reverse clause,
+    which this deliberately does not reuse)."""
+    if not host:
+        return False
+    h = host[4:] if host.startswith("www.") else host
+    return any(h == d or h.endswith("." + d) for d in _ALLOWED_HOST_DOMAINS)
+
+
+def _strip_finance_marker(host: str) -> str:
+    """Strip a trailing finance/IR marker (e.g. "-finance") from the
+    registrable second-level label only (the label immediately left of the
+    final TLD), so "loreal-finance.com" normalizes to "loreal.com" while
+    markers embedded in other labels (e.g. a subdomain) are left untouched.
+
+    If stripping results in a generic second-level label (e.g., "invest",
+    "corp", "finance"), return the original label unchanged to avoid
+    false "same owner" matches for unrelated generic domains."""
+    labels = host.split(".")
+    if len(labels) < 2:
+        return host
+    second_level = labels[-2]
+    for marker in _FINANCE_SUBDOMAIN_MARKERS:
+        if second_level.endswith(marker) and len(second_level) > len(marker):
+            stripped = second_level[: -len(marker)]
+            # Do not apply the alias if the result is a generic label.
+            if stripped not in _GENERIC_SLDS:
+                labels[-2] = stripped
+            break
+    return ".".join(labels)
+
+
+def domain_matches(host: str, domain: str) -> bool:
+    if not host or not domain:
+        return False
+    a = host[4:] if host.startswith("www.") else host
+    b = domain[4:] if domain.startswith("www.") else domain
+    a = _strip_finance_marker(a)
+    b = _strip_finance_marker(b)
+    return a == b or a.endswith("." + b) or b.endswith("." + a)