isa-data 1.2.3__tar.gz → 1.2.4__tar.gz

{isa_data-1.2.3 → isa_data-1.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: isa-data
-Version: 1.2.3
+Version: 1.2.4
 Summary: isA_Data — data analytics, data-fabric, and data-product SDK for the isA platform
 Author: Data Analytics Team
 License: Proprietary

{isa_data-1.2.3 → isa_data-1.2.4}/isa_data/__init__.py

@@ -7,7 +7,7 @@ Provides tools and services for data extraction, processing, and analysis.
 """
 
 # Package metadata
-__version__ = "1.2.3"
+__version__ = "1.2.4"
 __author__ = "Data Analytics Team"
 
 # Core functionality - only import what actually exists and works

isa_data-1.2.4/isa_data/adapters/source_adapters/bigquery_snapshot.py

@@ -0,0 +1,146 @@
+"""Read one BigQuery date as a typed Parquet snapshot using jobs.query.
+
+This synchronous adapter is intended for a batch runner (async callers should
+use asyncio.to_thread). It never converts rows through JSON or pandas. The
+caller owns source governance, checkpoint persistence and final sink publication.
+"""
+from __future__ import annotations
+
+import os
+import re
+import tempfile
+from dataclasses import dataclass
+from datetime import date
+from pathlib import Path
+from typing import Any, Callable, Mapping
+
+from google.cloud import bigquery
+from google.oauth2 import service_account
+import pyarrow as pa
+import pyarrow.parquet as pq
+
+
+class SnapshotError(RuntimeError):
+    """Snapshot did not satisfy completeness or schema requirements."""
+
+
+@dataclass(frozen=True)
+class SnapshotResult:
+    row_count: int
+    job: dict[str, str]
+    schema: list[dict[str, Any]]
+
+
+class BigQuerySnapshotReader:
+    """REST pagination preserves the result's Arrow types and query snapshot.
+
+    A durable on_started callback must save the job identity before the first
+    page is consumed. On retry, resume_job replays that same result from page
+    one to a new temporary file; partial pages are never appended or published.
+    Expired jobs fail explicitly. Start a new run to fetch late corrections.
+    on_batch receives each native Arrow batch before writing, for caller-owned
+    source totals. It runs again during replay; reset aggregates before retries.
+    """
+
+    def __init__(self, client: Any, *, timeout: int = 600, page_size: int = 10000):
+        if type(timeout) is not int or type(page_size) is not int or timeout <= 0 or page_size <= 0:
+            raise ValueError("timeout and page_size must be positive")
+        self.client = client
+        self.timeout = timeout
+        self.page_size = page_size
+
+    @classmethod
+    def from_service_account(cls, info: Mapping[str, Any], *, project_id: str):
+        credentials = service_account.Credentials.from_service_account_info(
+            dict(info), scopes=["https://www.googleapis.com/auth/bigquery.readonly"],
+        )
+        return cls(bigquery.Client(project=project_id, credentials=credentials))
+
+    def export_day(
+        self, *, table: str, date_column: str, business_date: date,
+        destination: Path, resume_job: Mapping[str, str] | None = None,
+        on_started: Callable[[dict[str, str]], None] | None = None,
+        minimum_rows: int = 1,
+        on_batch: Callable[[pa.RecordBatch], None] | None = None,
+    ) -> SnapshotResult:
+        if not re.fullmatch(r"[A-Za-z0-9_-]+\.[A-Za-z0-9_]+\.[A-Za-z0-9_]+", table):
+            raise ValueError("table must be a plain project.dataset.table identifier")
+        if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", date_column):
+            raise ValueError("date_column must be a plain identifier")
+        if type(business_date) is not date or type(minimum_rows) is not int or minimum_rows < 1:
+            raise ValueError("a business date and positive minimum_rows are required")
+        sql = f"SELECT * FROM `{table}` WHERE `{date_column}` = @business_date"
+        if resume_job is not None:
+            if set(resume_job) != {"job_id", "location", "project"} or not all(isinstance(value, str) and value.strip() for value in resume_job.values()):
+                raise ValueError("resume job identity is incomplete")
+            job = self.client.get_job(
+                resume_job["job_id"], location=resume_job["location"],
+                project=resume_job["project"], timeout=60,
+            )
+            # A checkpoint identity is only a locator, not proof that its job
+            # queried this source/date. Rebind to the authority's stored config.
+            expected_parameter = bigquery.ScalarQueryParameter(
+                "business_date", "DATE", business_date,
+            ).to_api_repr()
+            if (job.job_type != "query" or job.query != sql
+                    or job.use_legacy_sql is not False
+                    or [p.to_api_repr() for p in job.query_parameters] != [expected_parameter]):
+                raise SnapshotError("resume job does not match the requested source and business date")
+            if any(getattr(job, key) != value for key, value in resume_job.items()):
+                raise SnapshotError("resume job identity differs from its checkpoint")
+        else:
+            config = bigquery.QueryJobConfig(
+                use_legacy_sql=False, use_query_cache=False,
+                query_parameters=[bigquery.ScalarQueryParameter("business_date", "DATE", business_date)],
+            )
+            job = self.client.query(
+                sql, job_config=config, api_method=bigquery.enums.QueryApiMethod.QUERY,
+                timeout=60, job_retry=None,
+            )
+        identity = {"job_id": job.job_id, "location": job.location, "project": job.project}
+        if not all(identity.values()):
+            raise SnapshotError("BigQuery returned no durable job identity")
+        if on_started:
+            on_started(identity)
+        # Job retries can rerun SQL against a newer source snapshot. Only retry
+        # transport reads of this durable job; never permit automatic reruns.
+        rows = job.result(page_size=self.page_size, timeout=self.timeout, job_retry=None)
+        if rows.total_rows is None or rows.total_rows < minimum_rows:
+            raise SnapshotError("query returned fewer than the required rows")
+
+        destination = Path(destination)
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        fd, temporary = tempfile.mkstemp(prefix=".bigquery-", suffix=".parquet", dir=destination.parent)
+        os.close(fd)
+        writer = None
+        schema = None
+        count = 0
+        try:
+            # Explicit REST path: no BigQuery Storage API client or broader scope.
+            for batch in rows.to_arrow_iterable(bqstorage_client=None):
+                if schema is None:
+                    schema = batch.schema
+                    writer = pq.ParquetWriter(temporary, schema, compression="zstd")
+                elif not schema.equals(batch.schema, check_metadata=True):
+                    raise SnapshotError("query schema changed between pages")
+                if on_batch:
+                    on_batch(batch)
+                writer.write_batch(batch)
+                count += batch.num_rows
+            if writer:
+                writer.close()
+                writer = None
+            if schema is None or count != rows.total_rows:
+                raise SnapshotError("paginated row count differs from the query result")
+            physical = pq.ParquetFile(temporary)
+            if physical.metadata.num_rows != count or not physical.schema_arrow.equals(schema):
+                raise SnapshotError("Parquet readback differs from the query result")
+            os.replace(temporary, destination)
+            return SnapshotResult(
+                row_count=count, job=identity,
+                schema=[{"name": f.name, "type": str(f.type), "nullable": f.nullable} for f in schema],
+            )
+        finally:
+            if writer:
+                writer.close()
+            Path(temporary).unlink(missing_ok=True)

{isa_data-1.2.3 → isa_data-1.2.4}/isa_data/contracts/data_products/access_entitlement_v1.schema.json

@@ -161,8 +161,9 @@
       "type": "object"
     },
     "LifecycleOperation": {
-      "description": "Privileged Data Product lifecycle stages (Stages 3 through 8).",
+      "description": "Privileged Data Product lifecycle stages.\n\n``collect`` is the upstream stage (Stage -1): it acquires rows from an\nexternal source into the raw landing zone, before any product exists.  It\nis privileged for the same reason the later stages are — it runs a\ncredentialed workload and writes storage — so it compiles through an\nentitlement package like every other operation here.  The remaining\nmembers are the lakehouse stages 3 through 8.",
       "enum": [
+        "collect",
         "deploy",
         "run",
         "serve",
@@ -174,8 +175,11 @@
       "type": "string"
     },
     "SemanticPrivilege": {
-      "description": "Provider-neutral table privileges understood by Data Product plans.",
+      "description": "Provider-neutral privileges understood by Data Product plans.",
       "enum": [
+        "collection.execute",
+        "object-storage.write",
+        "source.read",
         "lineage.read",
         "lineage.reconcile",
         "metadata.read",

{isa_data-1.2.3 → isa_data-1.2.4}/isa_data/contracts/data_products/access_intent.py

@@ -154,8 +154,17 @@ class AccessEnvironment(StrEnum):
 
 
 class LifecycleOperation(StrEnum):
-    """Privileged Data Product lifecycle stages (Stages 3 through 8)."""
+    """Privileged Data Product lifecycle stages.
 
+    ``collect`` is the upstream stage (Stage -1): it acquires rows from an
+    external source into the raw landing zone, before any product exists.  It
+    is privileged for the same reason the later stages are — it runs a
+    credentialed workload and writes storage — so it compiles through an
+    entitlement package like every other operation here.  The remaining
+    members are the lakehouse stages 3 through 8.
+    """
+
+    COLLECT = "collect"
     DEPLOY = "deploy"
     RUN = "run"
     SERVE = "serve"
@@ -165,8 +174,18 @@ class LifecycleOperation(StrEnum):
 
 
 class SemanticPrivilege(StrEnum):
-    """Provider-neutral table privileges understood by Data Product plans."""
-
+    """Provider-neutral privileges understood by Data Product plans."""
+
+    # Collection privileges describe the upstream ``collect`` stage and are
+    # deliberately separate from the table privileges below.  ``source.read``
+    # is the declared source manifest and its credential *binding* — never a
+    # credential value.  ``collection.execute`` is one governed collection
+    # run of that source; a Kubernetes Job is only one provider's realization
+    # of it.  ``object-storage.write`` is the raw landing zone the run writes,
+    # which is not a table and must not be collapsed into ``table.write``.
+    COLLECTION_EXECUTE = "collection.execute"
+    OBJECT_STORAGE_WRITE = "object-storage.write"
+    SOURCE_READ = "source.read"
     # Metadata/lineage capabilities are intentionally distinct from table
     # privileges.  ``metadata.sync`` and ``lineage.reconcile`` describe the
     # fixed, idempotent side effects of the governed Dataphin reconciliation

{isa_data-1.2.3 → isa_data-1.2.4}/isa_data/contracts/data_products/access_intent_v1.schema.json

@@ -162,8 +162,9 @@
       "type": "object"
     },
     "LifecycleOperation": {
-      "description": "Privileged Data Product lifecycle stages (Stages 3 through 8).",
+      "description": "Privileged Data Product lifecycle stages.\n\n``collect`` is the upstream stage (Stage -1): it acquires rows from an\nexternal source into the raw landing zone, before any product exists.  It\nis privileged for the same reason the later stages are — it runs a\ncredentialed workload and writes storage — so it compiles through an\nentitlement package like every other operation here.  The remaining\nmembers are the lakehouse stages 3 through 8.",
       "enum": [
+        "collect",
         "deploy",
         "run",
         "serve",
@@ -244,8 +245,11 @@
       "title": "SafeExtensionValue"
     },
     "SemanticPrivilege": {
-      "description": "Provider-neutral table privileges understood by Data Product plans.",
+      "description": "Provider-neutral privileges understood by Data Product plans.",
       "enum": [
+        "collection.execute",
+        "object-storage.write",
+        "source.read",
         "lineage.read",
         "lineage.reconcile",
         "metadata.read",

{isa_data-1.2.3 → isa_data-1.2.4}/isa_data.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: isa-data
-Version: 1.2.3
+Version: 1.2.4
 Summary: isA_Data — data analytics, data-fabric, and data-product SDK for the isA platform
 Author: Data Analytics Team
 License: Proprietary

{isa_data-1.2.3 → isa_data-1.2.4}/isa_data.egg-info/SOURCES.txt

@@ -33,6 +33,7 @@ isa_data/adapters/source_adapters/README.md
 isa_data/adapters/source_adapters/__init__.py
 isa_data/adapters/source_adapters/base_adapter.py
 isa_data/adapters/source_adapters/bigquery_adapter.py
+isa_data/adapters/source_adapters/bigquery_snapshot.py
 isa_data/adapters/source_adapters/clickhouse_adapter.py
 isa_data/adapters/source_adapters/deltalake_adapter.py
 isa_data/adapters/source_adapters/elasticsearch_adapter.py

{isa_data-1.2.3 → isa_data-1.2.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "isa-data"
-version = "1.2.3"
+version = "1.2.4"
 description = "isA_Data — data analytics, data-fabric, and data-product SDK for the isA platform"
 readme = "README.md"
 requires-python = ">=3.12"