opteryx-catalog 0.4.11__py3-none-any.whl → 0.4.26__py3-none-any.whl

opteryx_catalog/webhooks/__init__.py

@@ -0,0 +1,230 @@
+"""Webhook system for Opteryx Catalog events.
+
+This module provides webhook notification capabilities for key catalog events.
+Webhooks can be delivered either directly via HTTP or asynchronously via
+Google Cloud Tasks.
+
+Configuration:
+    OPTERYX_WEBHOOK_DOMAIN: Base domain for webhook delivery (e.g., router.opteryx.app)
+    OPTERYX_WEBHOOK_QUEUE: Cloud Tasks queue path for async delivery
+                           Format: projects/PROJECT/locations/LOCATION/queues/QUEUE
+                           If not set, webhooks are sent directly via HTTP
+
+Example:
+    export OPTERYX_WEBHOOK_DOMAIN=router.opteryx.app
+    export OPTERYX_WEBHOOK_QUEUE=projects/my-project/locations/us-central1/queues/webhooks
+
+Webhook Endpoint:
+    All webhooks are sent to: https://{OPTERYX_WEBHOOK_DOMAIN}/event
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+from typing import Any
+from typing import Optional
+
+import requests
+
+
+class WebhookManager:
+    """Manages webhook delivery for catalog events.
+
+    Supports two delivery modes:
+    1. Direct HTTP POST (when OPTERYX_WEBHOOK_QUEUE is not set)
+    2. Cloud Tasks async delivery (when OPTERYX_WEBHOOK_QUEUE is set)
+    """
+
+    def __init__(
+        self,
+        domain: Optional[str] = None,
+        queue_path: Optional[str] = None,
+        timeout: int = 10,
+    ):
+        """Initialize the webhook manager.
+
+        Args:
+            domain: Base domain for webhooks (e.g., 'hook.opteryx.app')
+                   Falls back to OPTERYX_WEBHOOK_DOMAIN env var
+            queue_path: Cloud Tasks queue path for async delivery
+                       Falls back to OPTERYX_WEBHOOK_QUEUE env var
+            timeout: HTTP timeout in seconds for direct delivery
+        """
+        self.domain = domain or os.getenv("OPTERYX_WEBHOOK_DOMAIN")
+        self.queue_path = queue_path or os.getenv("OPTERYX_WEBHOOK_QUEUE")
+        self.timeout = timeout
+        self.enabled = bool(self.domain)
+
+        # Initialize Cloud Tasks client only if needed
+        self._tasks_client = None
+        if self.enabled and self.queue_path:
+            try:
+                from google.cloud import tasks_v2
+
+                self._tasks_client = tasks_v2.CloudTasksClient()
+            except ImportError:
+                # Cloud Tasks not available, fall back to direct HTTP
+                self._tasks_client = None
+
+    def send(
+        self,
+        action: str,
+        workspace: str,
+        collection: str,
+        resource_type: str,
+        resource_name: str,
+        payload: Optional[dict[str, Any]] = None,
+    ) -> bool:
+        """Send a webhook notification.
+
+        Args:
+            action: Action type (create, delete, update, commit, execute)
+            workspace: Workspace name
+            collection: Collection name
+            resource_type: Type of resource (dataset, view, collection)
+            resource_name: Name of the resource
+            payload: Additional data to include in the webhook body
+
+        Returns:
+            True if webhook was sent successfully (or queued), False otherwise
+        """
+        if not self.enabled:
+            return False
+
+        # Simple endpoint URL
+        url = f"https://{self.domain}/event"
+
+        # Build the payload
+        body = self._build_payload(
+            action=action,
+            workspace=workspace,
+            collection=collection,
+            resource_type=resource_type,
+            resource_name=resource_name,
+            additional=payload or {},
+        )
+
+        # Deliver via Cloud Tasks or direct HTTP
+        if self._tasks_client and self.queue_path:
+            return self._send_via_cloud_tasks(url, body)
+        else:
+            return self._send_direct(url, body)
+
+    def _build_payload(
+        self,
+        action: str,
+        workspace: str,
+        collection: str,
+        resource_type: str,
+        resource_name: str,
+        additional: dict[str, Any],
+    ) -> dict[str, Any]:
+        """Build the webhook payload.
+
+        Returns a standardized payload with event metadata and additional data.
+        """
+        return {
+            "event": {
+                "action": action,
+                "workspace": workspace,
+                "collection": collection,
+                "resource_type": resource_type,
+                "resource_name": resource_name,
+                "timestamp": int(time.time() * 1000),  # milliseconds
+            },
+            "data": additional,
+        }
+
+    def _send_direct(self, url: str, payload: dict[str, Any]) -> bool:
+        """Send webhook directly via HTTP POST.
+
+        Args:
+            url: Full webhook URL
+            payload: JSON payload
+
+        Returns:
+            True if successful (2xx response), False otherwise
+        """
+        try:
+            response = requests.post(
+                url,
+                json=payload,
+                timeout=self.timeout,
+                headers={
+                    "Content-Type": "application/json",
+                    "User-Agent": "opteryx-catalog-webhook/1.0",
+                },
+            )
+            return response.status_code >= 200 and response.status_code < 300
+        except Exception:
+            # Log errors in production; for now, silently fail
+            return False
+
+    def _send_via_cloud_tasks(self, url: str, payload: dict[str, Any]) -> bool:
+        """Send webhook asynchronously via Cloud Tasks.
+
+        Args:
+            url: Full webhook URL
+            payload: JSON payload
+
+        Returns:
+            True if task was created successfully, False otherwise
+        """
+        if not self._tasks_client:
+            # Fall back to direct delivery if client unavailable
+            return self._send_direct(url, payload)
+
+        try:
+            from google.cloud import tasks_v2
+
+            # Create the task
+            task = tasks_v2.Task(
+                http_request=tasks_v2.HttpRequest(
+                    http_method=tasks_v2.HttpMethod.POST,
+                    url=url,
+                    headers={
+                        "Content-Type": "application/json",
+                        "User-Agent": "opteryx-catalog-webhook/1.0",
+                    },
+                    body=json.dumps(payload).encode(),
+                )
+            )
+
+            # Queue the task
+            self._tasks_client.create_task(
+                request=tasks_v2.CreateTaskRequest(
+                    parent=self.queue_path,
+                    task=task,
+                )
+            )
+            return True
+        except Exception:
+            # Log errors in production; for now, silently fail
+            return False
+
+
+# Global webhook manager instance
+_webhook_manager: Optional[WebhookManager] = None
+
+
+def get_webhook_manager() -> WebhookManager:
+    """Get or create the global webhook manager instance."""
+    global _webhook_manager
+    if _webhook_manager is None:
+        _webhook_manager = WebhookManager()
+    return _webhook_manager
+
+
+def send_webhook(
+    action: str,
+    workspace: str,
+    collection: str,
+    resource_type: str,
+    resource_name: str,
+    payload: Optional[dict[str, Any]] = None,
+) -> bool:
+    """Convenience function to send a webhook via the global manager."""
+    manager = get_webhook_manager()
+    return manager.send(action, workspace, collection, resource_type, resource_name, payload)

opteryx_catalog/webhooks/events.py

@@ -0,0 +1,177 @@
+"""Event definitions and payload builders for webhook notifications.
+
+This module provides helper functions to create standardized payloads
+for different types of catalog events.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+from typing import Optional
+
+
+def dataset_created_payload(
+    schema: Any,
+    location: Optional[str] = None,
+    properties: Optional[dict[str, Any]] = None,
+) -> dict[str, Any]:
+    """Build payload for dataset creation event.
+
+    Args:
+        schema: Dataset schema (arrow or pyiceberg schema)
+        location: GCS location of the dataset
+        properties: Additional dataset properties
+
+    Returns:
+        Payload dictionary with dataset metadata
+    """
+    payload = {
+        "location": location,
+        "properties": properties or {},
+    }
+
+    # Include schema information if available
+    try:
+        if hasattr(schema, "names"):  # PyArrow schema
+            payload["schema"] = {
+                "fields": [
+                    {"name": name, "type": str(schema.field(name).type)} for name in schema.names
+                ]
+            }
+    except Exception:
+        pass
+
+    return payload
+
+
+def dataset_deleted_payload() -> dict[str, Any]:
+    """Build payload for dataset deletion event."""
+    return {}
+
+
+def dataset_updated_payload(
+    description: Optional[str] = None,
+    properties: Optional[dict[str, Any]] = None,
+) -> dict[str, Any]:
+    """Build payload for dataset update event.
+
+    Args:
+        description: New description
+        properties: Updated properties
+
+    Returns:
+        Payload dictionary with updated fields
+    """
+    return {
+        "description": description,
+        "properties": properties or {},
+    }
+
+
+def dataset_commit_payload(
+    snapshot_id: int,
+    sequence_number: int,
+    record_count: int,
+    file_count: int,
+) -> dict[str, Any]:
+    """Build payload for dataset commit (append) event.
+
+    Args:
+        snapshot_id: New snapshot ID
+        sequence_number: Sequence number of the commit
+        record_count: Number of records added
+        file_count: Number of files added
+
+    Returns:
+        Payload dictionary with commit metadata
+    """
+    return {
+        "snapshot_id": snapshot_id,
+        "sequence_number": sequence_number,
+        "record_count": record_count,
+        "file_count": file_count,
+    }
+
+
+def collection_created_payload(
+    properties: Optional[dict[str, Any]] = None,
+) -> dict[str, Any]:
+    """Build payload for collection creation event.
+
+    Args:
+        properties: Collection properties
+
+    Returns:
+        Payload dictionary with collection metadata
+    """
+    return {
+        "properties": properties or {},
+    }
+
+
+def view_created_payload(
+    definition: str,
+    properties: Optional[dict[str, Any]] = None,
+) -> dict[str, Any]:
+    """Build payload for view creation event.
+
+    Args:
+        definition: SQL definition of the view
+        properties: Additional view properties
+
+    Returns:
+        Payload dictionary with view metadata
+    """
+    return {
+        "definition": definition,
+        "properties": properties or {},
+    }
+
+
+def view_deleted_payload() -> dict[str, Any]:
+    """Build payload for view deletion event."""
+    return {}
+
+
+def view_updated_payload(
+    description: Optional[str] = None,
+    properties: Optional[dict[str, Any]] = None,
+) -> dict[str, Any]:
+    """Build payload for view update event.
+
+    Args:
+        description: New description
+        properties: Updated properties
+
+    Returns:
+        Payload dictionary with updated fields
+    """
+    return {
+        "description": description,
+        "properties": properties or {},
+    }
+
+
+def view_executed_payload(
+    execution_time_ms: Optional[int] = None,
+    row_count: Optional[int] = None,
+    error: Optional[str] = None,
+) -> dict[str, Any]:
+    """Build payload for view execution event.
+
+    Args:
+        execution_time_ms: Execution time in milliseconds
+        row_count: Number of rows returned
+        error: Error message if execution failed
+
+    Returns:
+        Payload dictionary with execution metadata
+    """
+    payload = {}
+    if execution_time_ms is not None:
+        payload["execution_time_ms"] = execution_time_ms
+    if row_count is not None:
+        payload["row_count"] = row_count
+    if error is not None:
+        payload["error"] = error
+    return payload

{opteryx_catalog-0.4.11.dist-info → opteryx_catalog-0.4.26.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opteryx-catalog
-Version: 0.4.11
+Version: 0.4.26
 Summary: Opteryx Cloud Catalog
 Author-email: joocer <justin.joyce@joocer.com>
 License:                                  Apache License
@@ -218,12 +218,14 @@ Classifier: Operating System :: OS Independent
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: google-cloud-firestore==2.22.0
-Requires-Dist: google-cloud-storage==3.7.0
+Requires-Dist: google-cloud-firestore==2.23.0
+Requires-Dist: google-cloud-storage==3.8.0
 Requires-Dist: orso==0.0.226
 Requires-Dist: opteryx-core
-Requires-Dist: pyarrow==22.0.0
+Requires-Dist: pyarrow==23.0.0
 Requires-Dist: requests==2.32.5
+Provides-Extra: webhooks
+Requires-Dist: google-cloud-tasks>=2.16.0; extra == "webhooks"
 Dynamic: license-file
 
 # pyiceberg-firestore-gcs
@@ -233,17 +235,18 @@ lightweight catalog interface. This package provides an opinionated
 catalog implementation for storing table metadata documents in Firestore and
 consolidated Parquet manifests in GCS.
 
-This project is intended to be used as a catalog component in GCP-based
-environments and provides utilities to interoperate with Avro/manifest-based
-workflows when needed.
+**Important:** This library is *modelled after* Apache Iceberg but is **not
+compatible** with Iceberg; it is a separate implementation with different
+storage conventions and metadata layout. This library is the catalog and
+metastore used by [opteryx.app](https://opteryx.app/) and uses **Firestore** as the primary
+metastore and **GCS** for data and manifest storage.
 
 ---
 
 ## Features ✅
 
 - Firestore-backed catalog and collection storage
- - GCS-based table metadata storage (with optional compatibility mode)
- - GCS-based table metadata storage; export/import utilities provide Avro interoperability
+- GCS-based table metadata storage; export/import utilities available for artifact conversion
 - Table creation, registration, listing, loading, renaming, and deletion
 - Commit operations that write updated metadata to GCS and persist references in Firestore
 - Simple, opinionated defaults (e.g., default GCS location derived from catalog properties)
@@ -299,7 +302,7 @@ print(tbl.metadata)
 - GCP authentication: Use `GOOGLE_APPLICATION_CREDENTIALS` or Application Default Credentials
 - `firestore_project` and `firestore_database` can be supplied when creating the catalog
 - `gcs_bucket` is recommended to allow `create_dataset` to write metadata automatically; otherwise pass `location` explicitly to `create_dataset`
- - The catalog does not write Avro/manifest-list artifacts in the hot path; use the provided export/import utilities for interoperability
+ - The catalog writes consolidated Parquet manifests and does not write manifest-list artifacts in the hot path. Use the provided export/import utilities for artifact conversion when necessary.
 
 Example environment variables:
 
@@ -308,15 +311,9 @@ export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account.json"
 export GOOGLE_CLOUD_PROJECT="my-gcp-project"
 ```
 
-### Interoperability
+### Manifest format
 
-This catalog implementation does not write Avro manifest-list/Avro manifest files
-in the hot path. Instead, table metadata is stored in Firestore and the runtime
-writes a consolidated Parquet manifest for fast query planning.
-
-If you need full Avro-compatible artifacts for other engines or tools, use the
-provided export/import utilities to transform between Avro manifests and the
-Parquet-first storage layout used by this catalog.
+This catalog writes consolidated Parquet manifests for fast query planning and stores table metadata in Firestore. Manifests and data files are stored in GCS. If you need different artifact formats, use the provided export/import utilities to convert manifests outside the hot path.
 
 ## API overview 📚
 

opteryx_catalog-0.4.26.dist-info/RECORD

@@ -0,0 +1,45 @@
+opteryx_catalog/__init__.py,sha256=cqGY7bl6iMBIqY_x6VTc5fAFH23M3XQeJYrHPX6FglY,902
+opteryx_catalog/exceptions.py,sha256=ZEaXmrrn030V8pfy8YMaLwzBWFms9OgZG21zVRGKlxM,652
+opteryx_catalog/opteryx_catalog.py,sha256=yL_9baNhvKS1U5kRqGt1xOqEigHhbGsWBT-pTwWn2kw,40178
+opteryx_catalog/catalog/__init__.py,sha256=yD7egf-dLd1z_CNXunz3ldLyLMMkSNbS3aKjGp3dKQY,119
+opteryx_catalog/catalog/compaction.py,sha256=GFwRiQssFh80w1uUqIy-oiz35HzkrDXi9wfPPOjv9uU,19483
+opteryx_catalog/catalog/dataset.py,sha256=XTW4HY7qZw3VIWIezO-xykQBrchZKGPmblQkBjUa9PY,59944
+opteryx_catalog/catalog/manifest.py,sha256=CcPfgTXQV4QyspFoAqFYMmx3qrfXUFNVxPjGhHPyvN8,17475
+opteryx_catalog/catalog/metadata.py,sha256=wGb1Z72oKrTJ7ANn7VOtuUq3Fu0RZo4233iGCoF_g1Y,3065
+opteryx_catalog/catalog/metastore.py,sha256=mS4qaaOMzcIu730Jm0K_Nq-4sNI8kIX3UscevYO5E08,1997
+opteryx_catalog/catalog/view.py,sha256=mUzfRGYqLRx_9BfZdGY5HNz6na9VMEPITrYKiI5m694,219
+opteryx_catalog/iops/__init__.py,sha256=_CxR-hg8XUD2cIFucb3aHyTFqwi41QmEDf9gXzXt3ZU,171
+opteryx_catalog/iops/base.py,sha256=1IW9qjDkQEMXvrA2J73VSBCdzkf2W5xVsWVnpNglL1U,1206
+opteryx_catalog/iops/fileio.py,sha256=CBbeSXZw52lgcE8lqzKHREACqufiVUId0gwtTt8PSpY,4711
+opteryx_catalog/iops/gcs.py,sha256=aB6hvSAQhbKTSyaLbAPgpXtSnvkI7fndXCRjaAZ1Dxo,8155
+opteryx_catalog/maki_nage/__init__.py,sha256=yBJlF6TsfFHNVz21xzCG6Ty4FSQOdByWA3fAkCSATNo,298
+opteryx_catalog/maki_nage/distogram.py,sha256=FMG4g423XOPAahd7Kg8YL5uB5UU5Ds39mV7JsFKlrTA,15976
+opteryx_catalog/maki_nage/tests/_test_histogram.py,sha256=rlT1mD8YTC2Ag5HpMrOK1DM_vaic0JW-dPrfpIz72Rs,1314
+opteryx_catalog/maki_nage/tests/test_bounds.py,sha256=4QCq0eaCKScv21917sRvvWn1No-zDei0FSIYx3eIqx8,508
+opteryx_catalog/maki_nage/tests/test_count.py,sha256=_qCRIY19oKla9qH079sS0JFQw7cxzmiFIXiDxChVMkA,469
+opteryx_catalog/maki_nage/tests/test_count_at.py,sha256=A2zbSZ6pusMvDoXQ90fr7gjIC4jORTpWbJyI9PnEtu4,2023
+opteryx_catalog/maki_nage/tests/test_quantile.py,sha256=dE-iR7O-tNrbfum8O1-QTswLa1pwWAyIEeKZOd1wB8w,2235
+opteryx_catalog/maki_nage/tests/test_stats.py,sha256=-6fhxttxP1QwbKGPjLBZmhB9I7aA5zJ3zlrAM9MmL9o,601
+opteryx_catalog/maki_nage/tests/test_update.py,sha256=UNbxbJe-GbubjQ2QIC2OXNV-jal881jQOxE7LkMkX8M,1114
+opteryx_catalog/webhooks/__init__.py,sha256=9vcC-m3Nzgmzu0x7jkU4ZEl1rXWovDEne5S1PcGll-w,7358
+opteryx_catalog/webhooks/events.py,sha256=82rRZd4tagbS2Lc19xrmPubvafG9YDs7J6XS-BdHIF4,4370
+opteryx_catalog-0.4.26.dist-info/licenses/LICENSE,sha256=mc5l20siqdcNQM54xALIWJhyaWsmQJ-NZt81UjgJejo,11351
+scripts/collect_byte_counts.py,sha256=F0I_HIg0PRPSJ5vh6z-9IYa3g-WO5fvFVkjwYyt3Ggo,1393
+scripts/create_dataset.py,sha256=K8zmQo3xbwc_yz2BxNK0IKj-DkDt3pFf13ycI6rgTHo,7798
+scripts/emit_full_single_file.py,sha256=r9-y3P_J_HZaLa56F4ifQ5mcq8wZbKu-l-e2zRNytMw,2464
+scripts/inspect_manifest_dryrun.py,sha256=i_6p9Bhfu3WlQVXDoGTEs4bBaVr09cvKLD9QQP5Taf4,13370
+scripts/inspect_single_file.py,sha256=P8E-Csqj-Sm3PcqgP95lh9idbSyAMqxIRKsEoCne8wo,5242
+scripts/inspect_single_file_gcs.py,sha256=R6-nQWxy-gHp3lFFBeB9vlsZVm6JD6FwUKHG_aFHmwc,3861
+scripts/read_dataset.py,sha256=hpBa8Qv1Oj6ffVIUmELGSri2eYHPpdqLnWFKgKpG-FM,9610
+tests/test_collections.py,sha256=JdAoRHw7tj4lIj2CQdLVrrbNiiaTD0onCKJCuI7EES0,882
+tests/test_compaction.py,sha256=7MLnfbGi3j17ZON8Qi9oq4i1UWkW0JigX46BBFWecMk,7871
+tests/test_dataset_metadata.py,sha256=bMzX2HiUnzFTyU3VkFuW5xjmFEP8cJSYPt1XF6IS0Qk,1019
+tests/test_describe_uncompressed.py,sha256=UTXM0-6wUepNFM2V-I6smoGCc3K2BDUcElLpClDD54E,4280
+tests/test_import.py,sha256=ZvoHW-rmcYqkW6TJKD_brgeePqHHbz2iTyRWKIBHGHk,137
+tests/test_pyproject.py,sha256=o3rS_GOems1oYQDH3UATfqc6XUwDTKZF2Q4cspU-NYc,206
+tests/test_refresh_manifest.py,sha256=5rGRmCNoQH5GeE-jGKgTDj8974eVT_GW6RQc5DYE9vo,10253
+tests/test_webhooks.py,sha256=dB15wwX-aJ18AhIUS_eOulUZtOyH1Lf6YMQFueRa8z0,5978
+opteryx_catalog-0.4.26.dist-info/METADATA,sha256=BD3PZ_VQfRcRtufRTngMTazm-lu2fPlXvfewOXnSvvo,22504
+opteryx_catalog-0.4.26.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+opteryx_catalog-0.4.26.dist-info/top_level.txt,sha256=HWATr4Wgxbg3c1X3EcsJ6cnHoR6ZAdTe1LQ2VssIBUo,30
+opteryx_catalog-0.4.26.dist-info/RECORD,,

{opteryx_catalog-0.4.11.dist-info → opteryx_catalog-0.4.26.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (80.10.2)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

scripts/collect_byte_counts.py

@@ -0,0 +1,42 @@
+import os
+import sys
+
+from google.cloud import firestore
+
+from opteryx_catalog import OpteryxCatalog
+
+sys.path.insert(0, os.path.join(sys.path[0], ".."))  # Add parent dir for pyiceberg_firestore_gcs
+sys.path.insert(1, os.path.join(sys.path[0], "../opteryx-core"))
+
+
+FIRESTORE_DATABASE = os.environ.get("FIRESTORE_DATABASE")
+BUCKET_NAME = os.environ.get("GCS_BUCKET")
+GCP_PROJECT_ID = os.environ.get("GCP_PROJECT_ID")
+
+
+def get_workspaces():
+    firestore_client = firestore.Client(project=GCP_PROJECT_ID, database=FIRESTORE_DATABASE)
+    workspaces = firestore_client.collections()
+    yield from [w.id for w in workspaces]
+
+
+for workspace in get_workspaces():
+    catalog = OpteryxCatalog(
+        workspace,
+        firestore_project=GCP_PROJECT_ID,
+        firestore_database=FIRESTORE_DATABASE,
+        gcs_bucket=BUCKET_NAME,
+    )
+
+    print(f"\nWorkspace '{workspace}':")
+    collections = catalog.list_collections()
+    for collection_name in collections:
+        print(f" Collection: {collection_name}")
+        collection = catalog.list_datasets(collection_name)
+        for dataset_name in collection:
+            print(f"  Dataset: {collection_name}.{dataset_name}")
+            dataset = catalog.load_dataset(f"{collection_name}.{dataset_name}")
+            print(f"   - {dataset_name} ({dataset.snapshot().summary.get('total-data-size')})")
+
+if __name__ == "__main__":
+    pass

scripts/emit_full_single_file.py

@@ -0,0 +1,81 @@
+import json
+import os
+import sys
+import time
+
+# Recursively convert non-JSON types (bytes, pyarrow Buffers, etc.) to hex or JSON-safe types
+from google.cloud import storage
+
+from opteryx_catalog.catalog.manifest import build_parquet_manifest_entry_from_bytes
+
+# Add local paths to sys.path to use local code instead of installed packages
+sys.path.insert(0, os.path.join(sys.path[0], ".."))  # Add parent dir for pyiceberg_firestore_gcs
+sys.path.insert(1, os.path.join(sys.path[0], "../opteryx-core"))
+sys.path.insert(1, os.path.join(sys.path[0], "../pyiceberg-firestore-gcs"))
+
+
+TARGET = "gs://opteryx_data/opteryx/ops/audit_log/data/188fa239430f10c3-59275747aed0-2.parquet"
+OUT = "artifacts/single_file_188fa239430f10c3_full.json"
+
+_, rest = TARGET.split("://", 1)
+bucket_name, path = rest.split("/", 1)
+client = storage.Client()
+blob = client.bucket(bucket_name).blob(path)
+print("Downloading", TARGET)
+data = blob.download_as_bytes()
+print("Downloaded bytes:", len(data))
+
+
+# read parquet bytes via a BufferReader
+entry = build_parquet_manifest_entry_from_bytes(data, TARGET, len(data)).to_dict()
+
+out = {
+    "_meta": {
+        "dataset": "opteryx.ops.audit_log",
+        "timestamp": int(time.time() * 1000),
+        "source": "single-file-full-json",
+    },
+    "file_path": TARGET,
+    "recomputed_full": entry,
+}
+
+
+def _hexify(obj):
+    # raw bytes-like
+    if isinstance(obj, (bytes, bytearray, memoryview)):
+        return obj.hex()
+    # pyarrow buffers/scalars -> try to extract python value
+    if hasattr(obj, "to_py"):
+        try:
+            val = obj.to_py()
+            return _hexify(val)
+        except Exception:
+            try:
+                # fallback: bytes representation
+                return bytes(obj).hex()
+            except Exception:
+                return str(obj)
+    # dict/list/tuple recursion
+    if isinstance(obj, dict):
+        return {k: _hexify(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [_hexify(v) for v in obj]
+    if isinstance(obj, tuple):
+        return tuple(_hexify(v) for v in obj)
+    # primitive
+    if isinstance(obj, (str, int, float, bool)) or obj is None:
+        return obj
+    # fallback to str
+    try:
+        return str(obj)
+    except Exception:
+        return None
+
+
+safe_out = _hexify(out)
+
+os.makedirs(os.path.dirname(OUT), exist_ok=True)
+with open(OUT, "w", encoding="utf-8") as of:
+    json.dump(safe_out, of, indent=2, ensure_ascii=False)
+
+print("WROTE", OUT)