opteryx-catalog 0.4.4__py3-none-any.whl → 0.4.26__py3-none-any.whl

opteryx_catalog/opteryx_catalog.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import os
 import time
 from typing import Any
 from typing import Iterable
@@ -21,6 +20,9 @@ from .exceptions import DatasetNotFound
 from .exceptions import ViewAlreadyExists
 from .exceptions import ViewNotFound
 from .iops.base import FileIO
+from .webhooks import send_webhook
+from .webhooks.events import dataset_created_payload
+from .webhooks.events import view_created_payload
 
 
 class OpteryxCatalog(Metastore):
@@ -28,7 +30,7 @@ class OpteryxCatalog(Metastore):
 
     Terminology: catalog -> workspace -> collection -> dataset|view
 
-    Stores table documents under the configured workspace in Firestore.
+    Stores dataset documents under the configured workspace in Firestore.
     Snapshots are stored in a `snapshots` subcollection under each
     dataset's document. Parquet manifests are written to GCS under the
     dataset location's `metadata/manifest-<snapshot_id>.parquet` path.
@@ -57,12 +59,8 @@ class OpteryxCatalog(Metastore):
             props_ref = self._catalog_ref.document("$properties")
             if not props_ref.get().exists:
                 now_ms = int(time.time() * 1000)
-                billing = (
-                    os.environ.get("BILLING_ACCOUNT_ID")
-                    or os.environ.get("BILLING_ACCOUNT")
-                    or None
-                )
-                owner = os.environ.get("WORKSPACE_OWNER") or None
+                billing = None
+                owner = None
                 props_ref.set(
                     {
                         "timestamp-ms": now_ms,
@@ -81,12 +79,9 @@ class OpteryxCatalog(Metastore):
             self.io = io
         else:
             if gcs_bucket:
-                try:
-                    from .iops.gcs import GcsFileIO
+                from .iops.gcs import GcsFileIO
 
-                    self.io = GcsFileIO()
-                except Exception:
-                    self.io = FileIO()
+                self.io = GcsFileIO()
             else:
                 self.io = FileIO()
 
@@ -109,7 +104,7 @@ class OpteryxCatalog(Metastore):
         return self._dataset_doc_ref(collection, dataset_name).collection("snapshots")
 
     def _views_collection(self, collection: str):
-        return self._namespace_ref(collection).collection("views")
+        return self._collection_ref(collection).collection("views")
 
     def _view_doc_ref(self, collection: str, view_name: str):
         return self._views_collection(collection).document(view_name)
@@ -125,7 +120,7 @@ class OpteryxCatalog(Metastore):
         if doc_ref.get().exists:
             raise DatasetAlreadyExists(f"Dataset already exists: {identifier}")
 
-        # Build default table metadata
+        # Build default dataset metadata
         location = f"gs://{self.gcs_bucket}/{self.workspace}/{collection}/{dataset_name}"
         metadata = DatasetMetadata(
             dataset_identifier=identifier,
@@ -149,11 +144,10 @@ class OpteryxCatalog(Metastore):
                 "timestamp-ms": now_ms,
                 "author": author,
                 "maintenance-policy": metadata.maintenance_policy,
+                "annotations": metadata.annotations,
             }
         )
 
-        # Persisted in primary `datasets` collection only.
-
         # Persist initial schema into `schemas` subcollection if provided
         if schema is not None:
             schema_id = self._write_schema(collection, dataset_name, schema, author=author)
@@ -175,13 +169,41 @@ class OpteryxCatalog(Metastore):
                 metadata.schemas = [
                     {"schema_id": schema_id, "columns": self._schema_to_columns(schema)}
                 ]
-            # update table doc to reference current schema
+            # update dataset doc to reference current schema
             doc_ref.update({"current-schema-id": metadata.current_schema_id})
 
+        # Send webhook notification
+        send_webhook(
+            action="create",
+            workspace=self.workspace,
+            collection=collection,
+            resource_type="dataset",
+            resource_name=dataset_name,
+            payload=dataset_created_payload(
+                schema=schema,
+                location=location,
+                properties=properties,
+            ),
+        )
+
         # Return SimpleDataset (attach this catalog so append() can persist)
         return SimpleDataset(identifier=identifier, _metadata=metadata, io=self.io, catalog=self)
 
-    def load_dataset(self, identifier: str) -> SimpleDataset:
+    def load_dataset(self, identifier: str, load_history: bool = False) -> SimpleDataset:
+        """Load a dataset from Firestore.
+
+        Args:
+            identifier: Dataset identifier in format 'collection.dataset_name'
+            load_history: If True, load all snapshots from Firestore (expensive for
+                large histories). If False (default), only load the current snapshot,
+                which is sufficient for most write operations.
+
+        Returns:
+            SimpleDataset instance with metadata loaded from Firestore.
+
+        Raises:
+            DatasetNotFound: If the dataset does not exist in Firestore.
+        """
         collection, dataset_name = identifier.split(".")
         doc_ref = self._dataset_doc_ref(collection, dataset_name)
         doc = doc_ref.get()
@@ -197,37 +219,69 @@ class OpteryxCatalog(Metastore):
             properties=data.get("properties") or {},
         )
 
-        # Load table-level timestamp/author and collection/workspace
+        # Load dataset-level timestamp/author and collection/workspace
         metadata.timestamp_ms = data.get("timestamp-ms")
         metadata.author = data.get("author")
-        # note: Firestore table doc stores the original collection and workspace
-        # under keys `collection` and `workspace`.
+        metadata.description = data.get("description")
+        metadata.describer = data.get("describer")
+        metadata.annotations = data.get("annotations") or []
 
-        # Load snapshots
+        # Load snapshots based on load_history flag
         snaps = []
-        for snap_doc in self._snapshots_collection(collection, dataset_name).stream():
-            sd = snap_doc.to_dict() or {}
-            snap = Snapshot(
-                snapshot_id=sd.get("snapshot-id"),
-                timestamp_ms=sd.get("timestamp-ms"),
-                author=sd.get("author"),
-                sequence_number=sd.get("sequence-number"),
-                user_created=sd.get("user-created"),
-                manifest_list=sd.get("manifest"),
-                schema_id=sd.get("schema-id"),
-                summary=sd.get("summary", {}),
-                operation_type=sd.get("operation-type"),
-                parent_snapshot_id=sd.get("parent-snapshot-id"),
-            )
-            snaps.append(snap)
+        if load_history:
+            # Load all snapshots from Firestore (expensive for large histories)
+            for snap_doc in self._snapshots_collection(collection, dataset_name).stream():
+                sd = snap_doc.to_dict() or {}
+                snap = Snapshot(
+                    snapshot_id=sd.get("snapshot-id"),
+                    timestamp_ms=sd.get("timestamp-ms"),
+                    author=sd.get("author"),
+                    sequence_number=sd.get("sequence-number"),
+                    user_created=sd.get("user-created"),
+                    manifest_list=sd.get("manifest"),
+                    schema_id=sd.get("schema-id"),
+                    summary=sd.get("summary", {}),
+                    operation_type=sd.get("operation-type"),
+                    parent_snapshot_id=sd.get("parent-snapshot-id"),
+                )
+                snaps.append(snap)
+            if snaps:
+                metadata.current_snapshot_id = snaps[-1].snapshot_id
+        else:
+            # Load only the current snapshot (efficient single read)
+            current_snap_id = data.get("current-snapshot-id")
+            if current_snap_id:
+                try:
+                    snap_doc = (
+                        self._snapshots_collection(collection, dataset_name)
+                        .document(str(current_snap_id))
+                        .get()
+                    )
+                    if snap_doc.exists:
+                        sd = snap_doc.to_dict() or {}
+                        snap = Snapshot(
+                            snapshot_id=sd.get("snapshot-id"),
+                            timestamp_ms=sd.get("timestamp-ms"),
+                            author=sd.get("author"),
+                            sequence_number=sd.get("sequence-number"),
+                            user_created=sd.get("user-created"),
+                            manifest_list=sd.get("manifest"),
+                            schema_id=sd.get("schema-id"),
+                            summary=sd.get("summary", {}),
+                            operation_type=sd.get("operation-type"),
+                            parent_snapshot_id=sd.get("parent-snapshot-id"),
+                        )
+                        snaps.append(snap)
+                        metadata.current_snapshot_id = current_snap_id
+                except Exception:
+                    pass
         metadata.snapshots = snaps
-        if snaps:
-            metadata.current_snapshot_id = snaps[-1].snapshot_id
 
         # Load schemas subcollection
-        try:
+        schemas_coll = doc_ref.collection("schemas")
+        # Load all schemas if requested; otherwise load only current schema
+        if load_history:
             schemas = []
-            schemas_coll = doc_ref.collection("schemas")
             for sdoc in schemas_coll.stream():
                 sd = sdoc.to_dict() or {}
                 schemas.append(
@@ -241,9 +295,23 @@ class OpteryxCatalog(Metastore):
                 )
             metadata.schemas = schemas
             metadata.current_schema_id = doc.to_dict().get("current-schema-id")
-        except Exception:
-            pass
-
+        else:
+            # Only load the current schema document for efficiency
+            current_schema_id = doc.to_dict().get("current-schema-id")
+            if current_schema_id:
+                sdoc = schemas_coll.document(str(current_schema_id)).get()
+                if sdoc.exists:
+                    sd = sdoc.to_dict() or {}
+                    metadata.schemas = [
+                        {
+                            "schema_id": sdoc.id,
+                            "columns": sd.get("columns", []),
+                            "timestamp-ms": sd.get("timestamp-ms"),
+                            "author": sd.get("author"),
+                            "sequence-number": sd.get("sequence-number"),
+                        }
+                    ]
+                    metadata.current_schema_id = current_schema_id
         return SimpleDataset(identifier=identifier, _metadata=metadata, io=self.io, catalog=self)
 
     def drop_dataset(self, identifier: str) -> None:
@@ -259,6 +327,13 @@ class OpteryxCatalog(Metastore):
         coll = self._datasets_collection(collection)
         return [doc.id for doc in coll.stream()]
 
+    def list_collections(self) -> Iterable[str]:
+        """List top-level collections (documents) in this workspace."""
+        try:
+            return [col.id for col in self._catalog_ref.list_documents() if col.id[0] != "$"]
+        except:
+            return []
+
     def create_collection(
         self,
         collection: str,
@@ -270,7 +345,7 @@ class OpteryxCatalog(Metastore):
 
         If `exists_ok` is False and the collection already exists, a KeyError is raised.
         """
-        doc_ref = self._namespace_ref(collection)
+        doc_ref = self._collection_ref(collection)
         if doc_ref.get().exists:
             if exists_ok:
                 return
@@ -285,6 +360,7 @@ class OpteryxCatalog(Metastore):
                 "properties": properties or {},
                 "timestamp-ms": now_ms,
                 "author": author,
+                "annotations": [],
             }
         )
 
@@ -292,11 +368,7 @@ class OpteryxCatalog(Metastore):
         self, collection: str, properties: dict | None = None, author: Optional[str] = None
     ) -> None:
         """Convenience wrapper that creates the collection only if missing."""
-        try:
-            self.create_collection(collection, properties=properties, exists_ok=True, author=author)
-        except Exception:
-            # Be conservative: surface caller-level warnings rather than failing
-            return
+        self.create_collection(collection, properties=properties, exists_ok=True, author=author)
 
     def dataset_exists(
         self, identifier_or_collection: str, dataset_name: Optional[str] = None
@@ -309,12 +381,14 @@ class OpteryxCatalog(Metastore):
         """
         # Normalize inputs
         if dataset_name is None:
-            # Expect a single collection like 'collection.table'
+            # Expect a single collection like 'collection.dataset'
             if "." not in identifier_or_collection:
                 raise ValueError(
-                    "collection must be 'collection.table' or pass dataset_name separately"
+                    "collection must be 'collection.dataset' or pass dataset_name separately"
                 )
             collection, dataset_name = identifier_or_collection.rsplit(".", 1)
+        else:
+            collection = identifier_or_collection
 
         try:
             doc_ref = self._dataset_doc_ref(collection, dataset_name)
@@ -334,6 +408,7 @@ class OpteryxCatalog(Metastore):
         author: str = None,
         description: Optional[str] = None,
         properties: dict | None = None,
+        update_if_exists: bool = False,
     ) -> CatalogView:
         """Create a view document and a statement version in the `statement` subcollection.
 
@@ -347,7 +422,22 @@ class OpteryxCatalog(Metastore):
 
         doc_ref = self._view_doc_ref(collection, view_name)
         if doc_ref.get().exists:
-            raise ViewAlreadyExists(f"View already exists: {collection}.{view_name}")
+            if not update_if_exists:
+                raise ViewAlreadyExists(f"View already exists: {collection}.{view_name}")
+            # Update existing view - get current sequence number
+            existing_doc = doc_ref.get().to_dict()
+            current_statement_id = existing_doc.get("statement-id")
+            if current_statement_id:
+                stmt_ref = doc_ref.collection("statement").document(current_statement_id)
+                stmt_doc = stmt_ref.get()
+                if stmt_doc.exists:
+                    sequence_number = stmt_doc.to_dict().get("sequence-number", 0) + 1
+                else:
+                    sequence_number = 1
+            else:
+                sequence_number = 1
+        else:
+            sequence_number = 1
 
         now_ms = int(time.time() * 1000)
         if author is None:
@@ -361,7 +451,7 @@ class OpteryxCatalog(Metastore):
                 "sql": sql,
                 "timestamp-ms": now_ms,
                 "author": author,
-                "sequence-number": 1,
+                "sequence-number": sequence_number,
             }
         )
 
@@ -383,12 +473,28 @@ class OpteryxCatalog(Metastore):
             }
         )
 
+        # Send webhook notification
+        send_webhook(
+            action="create" if not update_if_exists else "update",
+            workspace=self.workspace,
+            collection=collection,
+            resource_type="view",
+            resource_name=view_name,
+            payload=view_created_payload(
+                definition=sql,
+                properties=properties,
+            ),
+        )
+
         # Return a simple CatalogView wrapper
         v = CatalogView(name=view_name, definition=sql, properties=properties or {})
         # provide convenient attributes used by docs/examples
         setattr(v, "sql", sql)
         setattr(v, "metadata", type("M", (), {})())
         v.metadata.schema = schema
+        # Attach catalog and identifier for describe() method
+        setattr(v, "_catalog", self)
+        setattr(v, "_identifier", f"{collection}.{view_name}")
         return v
 
     def load_view(self, identifier: str | tuple) -> CatalogView:
@@ -410,27 +516,28 @@ class OpteryxCatalog(Metastore):
         stmt_id = data.get("statement-id")
         sql = None
         schema = data.get("schema")
-        try:
-            if stmt_id:
-                sdoc = doc_ref.collection("statement").document(str(stmt_id)).get()
-                if sdoc.exists:
-                    sql = (sdoc.to_dict() or {}).get("sql")
-            # fallback: pick the most recent statement
-            if not sql:
-                for s in doc_ref.collection("statement").stream():
-                    sd = s.to_dict() or {}
-                    if sd.get("sql"):
-                        sql = sd.get("sql")
-                        break
-        except Exception:
-            pass
+
+        sdoc = doc_ref.collection("statement").document(str(stmt_id)).get()
+        sql = (sdoc.to_dict() or {}).get("sql")
 
         v = CatalogView(name=view_name, definition=sql or "", properties=data.get("properties", {}))
         setattr(v, "sql", sql or "")
         setattr(v, "metadata", type("M", (), {})())
         v.metadata.schema = schema
+        # Populate metadata fields from the stored view document so callers
+        # expecting attributes like `timestamp_ms` won't fail.
         v.metadata.author = data.get("author")
         v.metadata.description = data.get("description")
+        v.metadata.timestamp_ms = data.get("timestamp-ms")
+        # Execution/operational fields (may be None)
+        v.metadata.last_execution_ms = data.get("last-execution-ms")
+        v.metadata.last_execution_data_size = data.get("last-execution-data-size")
+        v.metadata.last_execution_records = data.get("last-execution-records")
+        # Optional describer (used to flag LLM-generated descriptions)
+        v.metadata.describer = data.get("describer")
+        # Attach catalog and identifier for describe() method
+        setattr(v, "_catalog", self)
+        setattr(v, "_identifier", f"{collection}.{view_name}")
         return v
 
     def drop_view(self, identifier: str | tuple) -> None:
@@ -441,11 +548,9 @@ class OpteryxCatalog(Metastore):
 
         doc_ref = self._view_doc_ref(collection, view_name)
         # delete statement subcollection
-        try:
-            for d in doc_ref.collection("statement").stream():
-                doc_ref.collection("statement").document(d.id).delete()
-        except Exception:
-            pass
+        for d in doc_ref.collection("statement").stream():
+            doc_ref.collection("statement").document(d.id).delete()
+
         doc_ref.delete()
 
     def list_views(self, collection: str) -> Iterable[str]:
@@ -474,6 +579,8 @@ class OpteryxCatalog(Metastore):
                         "identifier must be 'collection.view' or pass view_name separately"
                     )
                 collection, view_name = identifier_or_collection.rsplit(".", 1)
+        else:
+            collection = identifier_or_collection
 
         try:
             doc_ref = self._view_doc_ref(collection, view_name)
@@ -501,40 +608,82 @@ class OpteryxCatalog(Metastore):
             updates["last-execution-time-ms"] = int(execution_time * 1000)
         updates["last-execution-ms"] = now_ms
         if updates:
-            try:
-                doc_ref.update(updates)
-            except Exception:
-                pass
+            doc_ref.update(updates)
+
+    def update_view_description(
+        self,
+        identifier: str | tuple,
+        description: str,
+        describer: Optional[str] = None,
+    ) -> None:
+        """Update the description for a view.
+
+        Args:
+            identifier: View identifier ('collection.view' or tuple)
+            description: The new description text
+            describer: Optional identifier for who/what created the description
+        """
+        if isinstance(identifier, tuple) or isinstance(identifier, list):
+            collection, view_name = identifier[0], identifier[1]
+        else:
+            collection, view_name = identifier.split(".")
+
+        doc_ref = self._view_doc_ref(collection, view_name)
+        updates = {
+            "description": description,
+        }
+        if describer is not None:
+            updates["describer"] = describer
+        doc_ref.update(updates)
+
+    def update_dataset_description(
+        self,
+        identifier: str | tuple,
+        description: str,
+        describer: Optional[str] = None,
+    ) -> None:
+        """Update the description for a dataset.
+
+        Args:
+            identifier: Dataset identifier in format 'collection.dataset_name'
+            description: The new description text
+            describer: Optional identifier for who/what created the description
+        """
+
+        if isinstance(identifier, tuple) or isinstance(identifier, list):
+            collection, dataset_name = identifier[0], identifier[1]
+        else:
+            collection, dataset_name = identifier.split(".")
+
+        doc_ref = self._dataset_doc_ref(collection, dataset_name)
+        updates = {
+            "description": description,
+        }
+        if describer is not None:
+            updates["describer"] = describer
+        doc_ref.update(updates)
 
     def write_parquet_manifest(
-        self, snapshot_id: int, entries: List[dict], table_location: str
+        self, snapshot_id: int, entries: List[dict], dataset_location: str
     ) -> Optional[str]:
         """Write a Parquet manifest for the given snapshot id and entries.
 
         Entries should be plain dicts convertible by pyarrow.Table.from_pylist.
-        The manifest will be written to <table_location>/metadata/manifest-<snapshot_id>.parquet
+        The manifest will be written to <dataset_location>/metadata/manifest-<snapshot_id>.parquet
         """
         import pyarrow as pa
         import pyarrow.parquet as pq
 
+        from .iops.fileio import WRITE_PARQUET_OPTIONS
+
         # If entries is None we skip writing; if entries is empty list, write
-        # an empty Parquet manifest (represents an empty table for this
+        # an empty Parquet manifest (represents an empty dataset for this
         # snapshot). This preserves previous manifests so older snapshots
         # remain readable.
         if entries is None:
             return None
 
-        # Print manifest entries so users can inspect the manifest when created
-        try:
-            pass
-
-            # print("[MANIFEST] Parquet manifest entries to write:")
-            # print(json.dumps(entries, indent=2, default=str))
-        except Exception:
-            # print("[MANIFEST] Parquet manifest entries:", entries)
-            pass
-
-        parquet_path = f"{table_location}/metadata/manifest-{snapshot_id}.parquet"
+        parquet_path = f"{dataset_location}/metadata/manifest-{snapshot_id}.parquet"
 
         # Use provided FileIO if it supports writing; otherwise write to GCS
         try:
@@ -546,146 +695,63 @@ class OpteryxCatalog(Metastore):
                     ("file_format", pa.string()),
                     ("record_count", pa.int64()),
                     ("file_size_in_bytes", pa.int64()),
+                    ("uncompressed_size_in_bytes", pa.int64()),
+                    ("column_uncompressed_sizes_in_bytes", pa.list_(pa.int64())),
+                    ("null_counts", pa.list_(pa.int64())),
                     ("min_k_hashes", pa.list_(pa.list_(pa.uint64()))),
                     ("histogram_counts", pa.list_(pa.list_(pa.int64()))),
                     ("histogram_bins", pa.int32()),
                     ("min_values", pa.list_(pa.int64())),
                     ("max_values", pa.list_(pa.int64())),
+                    ("min_values_display", pa.list_(pa.string())),
+                    ("max_values_display", pa.list_(pa.string())),
                 ]
             )
 
-            try:
-                table = pa.Table.from_pylist(entries, schema=schema)
-            except Exception:
-                # Diagnostic output to help find malformed manifest entries
-                try:
-                    print(
-                        "[MANIFEST DEBUG] Failed to convert entries to Parquet manifest table. Dumping entries:"
-                    )
-                    for i, ent in enumerate(entries):
-                        print(f" Entry {i}:")
-                        if isinstance(ent, dict):
-                            for k, v in ent.items():
-                                tname = type(v).__name__
-                                try:
-                                    s = repr(v)
-                                except Exception:
-                                    s = "<unreprable>"
-                                print(f"  - {k}: type={tname} repr={s[:200]}")
-                        else:
-                            print(
-                                f"  - non-dict entry: type={type(ent).__name__} repr={repr(ent)[:200]}"
-                            )
-                except Exception:
-                    pass
+            # Normalize entries to match schema expectations:
+            normalized = []
+            for ent in entries:
+                if not isinstance(ent, dict):
+                    normalized.append(ent)
+                    continue
+                e = dict(ent)
+                # Ensure list fields exist
+                e.setdefault("min_k_hashes", [])
+                e.setdefault("histogram_counts", [])
+                e.setdefault("histogram_bins", 0)
+                e.setdefault("column_uncompressed_sizes_in_bytes", [])
+                e.setdefault("null_counts", [])
+                e.setdefault("min_values_display", [])
+                e.setdefault("max_values_display", [])
+
+                # min/max values are stored as compressed int64 values
+                # display values are string representations for human readability
+                mv = e.get("min_values") or []
+                xv = e.get("max_values") or []
+                mv_disp = e.get("min_values_display") or []
+                xv_disp = e.get("max_values_display") or []
+
+                def truncate_display(v, max_len=32):
+                    """Truncate display value to max_len characters, adding '...' if longer."""
+                    if v is None:
+                        return None
+                    s = str(v)
+                    if len(s) > max_len:
+                        return s[:max_len] + "..."
+                    return s
+
+                # Ensure int64 values are properly typed for min/max
+                e["min_values"] = [int(v) if v is not None else None for v in mv]
+                e["max_values"] = [int(v) if v is not None else None for v in xv]
+                # Display values truncated to 32 chars with '...' suffix if longer
+                e["min_values_display"] = [truncate_display(v) for v in mv_disp]
+                e["max_values_display"] = [truncate_display(v) for v in xv_disp]
+                normalized.append(e)
+
+            table = pa.Table.from_pylist(normalized, schema=schema)
 
-                # Attempt to sanitize entries and retry conversion.
-                try:
-                    print("[MANIFEST DEBUG] Attempting to sanitize entries and retry")
-                    sanitized = []
-                    for ent in entries:
-                        if not isinstance(ent, dict):
-                            sanitized.append(ent)
-                            continue
-                        e2 = dict(ent)  # copy
-                        # Ensure numeric fields
-                        for k in ("record_count", "file_size_in_bytes", "histogram_bins"):
-                            v = e2.get(k)
-                            try:
-                                e2[k] = int(v) if v is not None else 0
-                            except Exception:
-                                e2[k] = 0
-                        # Ensure min_k_hashes is list[list[int]]
-                        mk = e2.get("min_k_hashes")
-                        if not isinstance(mk, list):
-                            e2["min_k_hashes"] = []
-                        else:
-                            new_mk = []
-                            for sub in mk:
-                                if isinstance(sub, list):
-                                    try:
-                                        new_mk.append([int(x) for x in sub])
-                                    except Exception:
-                                        new_mk.append([])
-                                else:
-                                    new_mk.append([])
-                            e2["min_k_hashes"] = new_mk
-                        # Ensure histogram_counts is list[list[int]]
-                        hc = e2.get("histogram_counts")
-                        if not isinstance(hc, list):
-                            e2["histogram_counts"] = []
-                        else:
-                            new_hc = []
-                            for sub in hc:
-                                if isinstance(sub, list):
-                                    try:
-                                        new_hc.append([int(x) for x in sub])
-                                    except Exception:
-                                        new_hc.append([])
-                                else:
-                                    new_hc.append([])
-                            e2["histogram_counts"] = new_hc
-                        # Sanitize min_values / max_values: must be list[int] or None
-                        # Sanitize min_values / max_values: coerce to int64 using to_int() if available
-                        try:
-                            from opteryx.compiled.structures.relation_statistics import to_int
-                        except Exception:
-
-                            def to_int(val):
-                                # Best-effort fallback: handle numpy types, strings and numbers
-                                try:
-                                    if val is None:
-                                        return None
-                                    if hasattr(val, "item"):
-                                        val = val.item()
-                                    if isinstance(val, (bytes, bytearray)):
-                                        val = val.decode(errors="ignore")
-                                    if isinstance(val, str):
-                                        # empty strings are invalid
-                                        if val == "":
-                                            return None
-                                        try:
-                                            return int(val)
-                                        except Exception:
-                                            return None
-                                    if isinstance(val, float):
-                                        return int(val)
-                                    return int(val)
-                                except Exception:
-                                    return None
-
-                        for key in ("min_values", "max_values"):
-                            mv = e2.get(key)
-                            if not isinstance(mv, list):
-                                e2[key] = [None]
-                            else:
-                                new_mv = []
-                                for x in mv:
-                                    try:
-                                        if x is None:
-                                            new_mv.append(None)
-                                            continue
-                                        # Use to_int to coerce into int64 semantics
-                                        v = x
-                                        if hasattr(v, "item"):
-                                            v = v.item()
-                                        coerced = to_int(v)
-                                        # to_int may return None-like sentinel; accept ints only
-                                        if coerced is None:
-                                            new_mv.append(None)
-                                        else:
-                                            new_mv.append(int(coerced))
-                                    except Exception:
-                                        new_mv.append(None)
-                                e2[key] = new_mv
-                        sanitized.append(e2)
-                    table = pa.Table.from_pylist(sanitized, schema=schema)
-                    print("[MANIFEST DEBUG] Sanitized entries converted successfully")
-                except Exception:
-                    print("[MANIFEST DEBUG] Sanitization failed; re-raising original exception")
-                    raise
             buf = pa.BufferOutputStream()
-            pq.write_table(table, buf, compression="zstd")
+            pq.write_table(table, buf, **WRITE_PARQUET_OPTIONS)
             data = buf.getvalue().to_pybytes()
 
             if self.io:
@@ -696,15 +762,6 @@ class OpteryxCatalog(Metastore):
                     out.close()
                 except Exception:
                     pass
-            elif self._storage_client and self.gcs_bucket:
-                # Write to GCS bucket
-                bucket = self._storage_client.bucket(self.gcs_bucket)
-                # object path: remove gs://bucket/ prefix
-                parsed = parquet_path
-                if parsed.startswith("gs://"):
-                    parsed = parsed[5 + len(self.gcs_bucket) + 1 :]
-                blob = bucket.blob(parsed)
-                blob.upload_from_string(data)
 
             return parquet_path
         except Exception as e:
@@ -713,7 +770,7 @@ class OpteryxCatalog(Metastore):
             raise e
 
     def save_snapshot(self, identifier: str, snapshot: Snapshot) -> None:
-        """Persist a single snapshot document for a table."""
+        """Persist a single snapshot document for a dataset."""
         namespace, dataset_name = identifier.split(".")
         snaps = self._snapshots_collection(namespace, dataset_name)
         doc_id = str(snapshot.snapshot_id)
@@ -749,9 +806,9 @@ class OpteryxCatalog(Metastore):
         snaps.document(doc_id).set(data)
 
     def save_dataset_metadata(self, identifier: str, metadata: DatasetMetadata) -> None:
-        """Persist table-level metadata and snapshots to Firestore.
+        """Persist dataset-level metadata and snapshots to Firestore.
 
-        This writes the table document and upserts snapshot documents.
+        This writes the dataset document and upserts snapshot documents.
         """
         collection, dataset_name = identifier.split(".")
         doc_ref = self._dataset_doc_ref(collection, dataset_name)
@@ -763,6 +820,7 @@ class OpteryxCatalog(Metastore):
                 "location": metadata.location,
                 "properties": metadata.properties,
                 "format-version": metadata.format_version,
+                "annotations": metadata.annotations,
                 "current-snapshot-id": metadata.current_snapshot_id,
                 "current-schema-id": metadata.current_schema_id,
                 "timestamp-ms": metadata.timestamp_ms,
@@ -777,10 +835,9 @@ class OpteryxCatalog(Metastore):
         # Metadata persisted in primary `datasets` collection only.
 
         snaps_coll = self._snapshots_collection(collection, dataset_name)
-        existing = {d.id for d in snaps_coll.stream()}
-        new_ids = set()
+        # Upsert snapshot documents. Do NOT delete existing snapshot documents
+        # here to avoid accidental removal of historical snapshots on save.
         for snap in metadata.snapshots:
-            new_ids.add(str(snap.snapshot_id))
             snaps_coll.document(str(snap.snapshot_id)).set(
                 {
                     "snapshot-id": snap.snapshot_id,
@@ -795,10 +852,6 @@ class OpteryxCatalog(Metastore):
                 }
             )
 
-        # Delete stale snapshots
-        for stale in existing - new_ids:
-            snaps_coll.document(stale).delete()
-
         # Persist schemas subcollection
         schemas_coll = doc_ref.collection("schemas")
         existing_schema_ids = {d.id for d in schemas_coll.stream()}
@@ -866,6 +919,7 @@ class OpteryxCatalog(Metastore):
                 "scale": scale,
                 "precision": precision,
                 "expectation-policies": [],
+                "annotations": [],
             }
 
             cols.append(typed)
@@ -873,7 +927,7 @@ class OpteryxCatalog(Metastore):
         return cols
 
     def _write_schema(self, namespace: str, dataset_name: str, schema: Any, author: str) -> str:
-        """Persist a schema document in the table's `schemas` subcollection and
+        """Persist a schema document in the dataset's `schemas` subcollection and
         return the new schema id.
         """
         import uuid