ohmydata 0.3.2__tar.gz → 0.3.3__tar.gz

{ohmydata-0.3.2 → ohmydata-0.3.3}/CHANGELOG.md

@@ -2,6 +2,13 @@
 
 ## Unreleased
 
+## 0.3.3 — 2026-09-23
+
+- Add `fetch_sec_submissions_closure` to fetch and retain a complete SEC
+  Submissions root and its advertised history pages using the injected SEC HTTP
+  client. It returns snapshot-backed discovery sources and fails explicitly on
+  missing pages, malformed schemas, conflicting accession metadata or bounds.
+
 ## 0.3.2 — 2026-09-13
 
 - Narrow OMD SEC v2 live XBRL compatibility fix:

{ohmydata-0.3.2 → ohmydata-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: ohmydata
-Version: 0.3.2
+Version: 0.3.3
 Summary: Financial and alternative market data ingestion SDK.
 Author: OMD contributors
 Requires-Python: <3.13,>=3.11
@@ -728,6 +728,43 @@ It preserves acceptance metadata, emits filing events and exact 8-K item 2.02
 events, and fails on missing pages or conflicting facts. Acceptance is not proof
 of the website's first publication time.
 
+`fetch_sec_submissions_closure` obtains that complete root and every advertised
+history page with an injected `SecHttpClient` and `SnapshotStore`. The client
+requires a caller supplied SEC User-Agent and applies its request spacing and
+retry policy for transient connection and HTTP status failures. A response body
+read failure is classified as transient; the caller retries the full closure.
+The fetcher validates bounded JSON, response length, CIK, page names,
+columns and duplicate accession metadata, then retains exact response bytes as
+append-only observations. A failed later page leaves earlier observations
+available for inspection but returns no complete closure. A new call fetches a
+fresh root and all pages it advertises; callers choose their own schedule and
+reconciliation windows. Up to 16 history pages, 8 MiB per page, 64 MiB total and
+100,000 filing rows are supported; exceeding a bound fails explicitly.
+
+```python
+from ohmydata.providers.sec import (
+    SecHttpClient,
+    fetch_sec_submissions_closure,
+    discover_sec_filing_events,
+)
+
+closure = fetch_sec_submissions_closure(snapshot_store, SecHttpClient(sec_user_agent), "0000000001")
+batch = discover_sec_filing_events(
+    snapshot_store,
+    closure.root_source,
+    closure.historical_sources,
+    policy=discovery_policy,
+    prior_cursor=prior_cursor,
+)
+```
+
+The closure is retained source evidence. Event discovery does not infer whether
+an amendment changes financial values. Existing retained document and SGML
+financial production paths take an explicit accession, while the live
+`SecFinancialsClient` still selects filings by form and date. This fetcher does
+not acquire the filing artifacts needed for exact-accession financial production.
+Event ledger/work processing remains a separate step.
+
 `SecEventDiscoveryLedger` atomically stores discovered events and their cursor in
 immutable generations. On POSIX filesystems, appenders cooperate through a writer
 lock; `load()` replays the retained source observations and reconstructs the

{ohmydata-0.3.2 → ohmydata-0.3.3}/README.md

@@ -702,6 +702,43 @@ It preserves acceptance metadata, emits filing events and exact 8-K item 2.02
 events, and fails on missing pages or conflicting facts. Acceptance is not proof
 of the website's first publication time.
 
+`fetch_sec_submissions_closure` obtains that complete root and every advertised
+history page with an injected `SecHttpClient` and `SnapshotStore`. The client
+requires a caller supplied SEC User-Agent and applies its request spacing and
+retry policy for transient connection and HTTP status failures. A response body
+read failure is classified as transient; the caller retries the full closure.
+The fetcher validates bounded JSON, response length, CIK, page names,
+columns and duplicate accession metadata, then retains exact response bytes as
+append-only observations. A failed later page leaves earlier observations
+available for inspection but returns no complete closure. A new call fetches a
+fresh root and all pages it advertises; callers choose their own schedule and
+reconciliation windows. Up to 16 history pages, 8 MiB per page, 64 MiB total and
+100,000 filing rows are supported; exceeding a bound fails explicitly.
+
+```python
+from ohmydata.providers.sec import (
+    SecHttpClient,
+    fetch_sec_submissions_closure,
+    discover_sec_filing_events,
+)
+
+closure = fetch_sec_submissions_closure(snapshot_store, SecHttpClient(sec_user_agent), "0000000001")
+batch = discover_sec_filing_events(
+    snapshot_store,
+    closure.root_source,
+    closure.historical_sources,
+    policy=discovery_policy,
+    prior_cursor=prior_cursor,
+)
+```
+
+The closure is retained source evidence. Event discovery does not infer whether
+an amendment changes financial values. Existing retained document and SGML
+financial production paths take an explicit accession, while the live
+`SecFinancialsClient` still selects filings by form and date. This fetcher does
+not acquire the filing artifacts needed for exact-accession financial production.
+Event ledger/work processing remains a separate step.
+
 `SecEventDiscoveryLedger` atomically stores discovered events and their cursor in
 immutable generations. On POSIX filesystems, appenders cooperate through a writer
 lock; `load()` replays the retained source observations and reconstructs the

{ohmydata-0.3.2 → ohmydata-0.3.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "ohmydata"
-version = "0.3.2"
+version = "0.3.3"
 description = "Financial and alternative market data ingestion SDK."
 readme = "README.md"
 requires-python = ">=3.11,<3.13"

{ohmydata-0.3.2 → ohmydata-0.3.3}/src/ohmydata/__init__.py

@@ -1,5 +1,5 @@
 """Provisional offline-only Oh My Data package scaffold."""
 
-__version__ = "0.3.2"
+__version__ = "0.3.3"
 
 __all__ = ["__version__"]

{ohmydata-0.3.2 → ohmydata-0.3.3}/src/ohmydata/providers/sec/__init__.py

@@ -266,6 +266,7 @@ from .sgml_financials import (
     produce_sec_financials_from_sgml,
 )
 from .structural_quality import SecStructuralQualityReport, evaluate_sec_structural_quality
+from .submissions import SecSubmissionsClosure, fetch_sec_submissions_closure
 from .unit_evidence import SecFinancialUnitEvidence
 from .xbrl_package import (
     SecXbrlPackage,
@@ -419,6 +420,7 @@ __all__ = [
     "SecStatementParseError",
     "SecStatementRow",
     "SecStructuralQualityReport",
+    "SecSubmissionsClosure",
     "SecTargetMetricInputs",
     "SecTransportEvidence",
     "SecUnavailableResult",
@@ -449,6 +451,7 @@ __all__ = [
     "evaluate_sec_structural_quality",
     "execute_sec_metric_recompute",
     "extract_from_artifact",
+    "fetch_sec_submissions_closure",
     "historical_basenames",
     "historical_submission_url",
     "latest_completed_quarter",

ohmydata-0.3.3/src/ohmydata/providers/sec/submissions.py

@@ -0,0 +1,143 @@
+"""Retain a complete SEC Submissions root and its advertised history pages."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any
+
+from ...core import RequestSpec, SnapshotMode, SnapshotStore
+from ._event_discovery_models import SecDiscoverySource
+from .edgar import historical_basenames, historical_submission_url
+from .errors import (
+    PermanentProviderError,
+    ResourceLimitError,
+    SchemaMismatchError,
+    TransientProviderError,
+)
+from .event_discovery import _strict_json, _submission_rows
+from .http import SecHttpClient, validate_sec_url
+
+_SOURCE_LIMIT = 8 * 1024**2
+_TOTAL_LIMIT = 64 * 1024**2
+_ROW_LIMIT = 100_000
+_SERIALIZATION = "sec-submissions-json-v1"
+
+
+@dataclass(frozen=True)
+class SecSubmissionsClosure:
+    """A retained root and every history page named by that exact root."""
+
+    cik: str
+    root_source: SecDiscoverySource
+    historical_sources: tuple[SecDiscoverySource, ...]
+
+
+def _exact_url(expected: str, actual: str) -> str:
+    if validate_sec_url(actual) != validate_sec_url(expected):
+        raise PermanentProviderError("SEC submissions redirect changed source URL")
+    return actual
+
+
+def _fetch(client: SecHttpClient, url: str, remaining: int) -> bytes:
+    if remaining <= 0:
+        raise ResourceLimitError("aggregate submissions bytes exceeded")
+    response = client.open(
+        url,
+        accept="application/json",
+        max_bytes=min(_SOURCE_LIMIT, remaining),
+        redirect_validator=lambda actual: _exact_url(url, actual),
+    )
+    try:
+        _exact_url(url, response.url)
+        try:
+            body: bytes = response.body.read()
+        except (TimeoutError, ConnectionError, OSError) as exc:
+            raise TransientProviderError("SEC submissions response read failed") from exc
+        if type(body) is not bytes or len(body) > min(_SOURCE_LIMIT, remaining):
+            raise ResourceLimitError("submissions response bytes exceeded")
+        declared_length = response.headers.get("Content-Length")
+        if declared_length is not None and len(body) != int(declared_length):
+            raise TransientProviderError("SEC submissions response length mismatch")
+        return body
+    finally:
+        response.body.close()
+
+
+def fetch_sec_submissions_closure(
+    store: SnapshotStore,
+    client: SecHttpClient,
+    cik: str,
+    *,
+    utc_now: Callable[[], datetime] = lambda: datetime.now(UTC),
+) -> SecSubmissionsClosure:
+    """Fetch and append immutable observations for one complete advertised closure.
+
+    Each HTTP GET uses the injected SEC client's limiter and classified retries.
+    Failed later pages leave earlier observations retained, but no closure is returned.
+    A subsequent invocation fetches a fresh root and all pages it advertises.
+    """
+    if type(cik) is not str or re.fullmatch(r"[0-9]{10}", cik) is None:
+        raise ValueError("CIK must be ten digits")
+    if not isinstance(store, SnapshotStore) or not isinstance(client, SecHttpClient):
+        raise TypeError("invalid submissions fetch dependencies")
+
+    root_url = f"https://data.sec.gov/submissions/CIK{cik}.json"
+    root_bytes = _fetch(client, root_url, _TOTAL_LIMIT)
+    root = _strict_json(root_bytes)
+    filings = root.get("filings")
+    if not isinstance(filings, dict) or not isinstance(filings.get("files"), list):
+        raise SchemaMismatchError("historical submissions references missing")
+    names = historical_basenames(root, cik)
+    observed = len(root_bytes)
+    rows: dict[str, dict[str, object]] = {}
+    row_count = 0
+
+    def validate_rows(payload: dict[str, Any], *, child: bool) -> None:
+        nonlocal row_count
+        for row in _submission_rows(payload, cik, child=child):
+            row_count += 1
+            if row_count > _ROW_LIMIT:
+                raise ResourceLimitError("aggregate filing row limit exceeded")
+            accession = row["accessionNumber"]
+            if (
+                not isinstance(accession, str)
+                or re.fullmatch(r"[0-9]{10}-[0-9]{2}-[0-9]{6}", accession) is None
+            ):
+                raise SchemaMismatchError("invalid accession")
+            old = rows.setdefault(accession, row)
+            if old != row:
+                raise SchemaMismatchError("conflicting submission metadata")
+
+    validate_rows(root, child=False)
+
+    def retain(
+        endpoint: str, parameters: dict[str, str], url: str, body: bytes
+    ) -> SecDiscoverySource:
+        fetched_at = utc_now()
+        observation = store.observe(
+            RequestSpec("sec", endpoint, parameters),
+            body,
+            fetched_at,
+            _SERIALIZATION,
+            SnapshotMode.APPEND,
+        )
+        return SecDiscoverySource(url, observation)
+
+    root_source = retain("edgar_submissions", {"cik": cik}, root_url, root_bytes)
+    children: list[SecDiscoverySource] = []
+    for name in names:
+        url = historical_submission_url(cik, name)
+        body = _fetch(client, url, _TOTAL_LIMIT - observed)
+        observed += len(body)
+        child = _strict_json(body)
+        validate_rows(child, child=True)
+        children.append(
+            retain("edgar_submissions_history", {"cik": cik, "basename": name}, url, body)
+        )
+    return SecSubmissionsClosure(cik, root_source, tuple(children))
+
+
+__all__ = ["SecSubmissionsClosure", "fetch_sec_submissions_closure"]

{ohmydata-0.3.2 → ohmydata-0.3.3}/uv.lock

@@ -491,7 +491,7 @@ wheels = [
 
 [[package]]
 name = "ohmydata"
-version = "0.3.2"
+version = "0.3.3"
 source = { editable = "." }
 
 [package.optional-dependencies]