opteryx-catalog 0.4.4__py3-none-any.whl → 0.4.11__py3-none-any.whl

opteryx_catalog/catalog/dataset.py

@@ -8,6 +8,9 @@ from typing import Any
 from typing import Iterable
 from typing import Optional
 
+from .manifest import ParquetManifestEntry
+from .manifest import build_parquet_manifest_entry
+from .manifest import build_parquet_manifest_minmax_entry
 from .metadata import DatasetMetadata
 from .metadata import Snapshot
 from .metastore import Dataset
@@ -69,6 +72,26 @@ class SimpleDataset(Dataset):
     def metadata(self) -> DatasetMetadata:
         return self._metadata
 
+    def _next_sequence_number(self) -> int:
+        """Calculate the next sequence number.
+
+        Uses the current snapshot's sequence number + 1. Works efficiently
+        with load_history=False since we only need the most recent snapshot,
+        not the full history.
+
+        Returns:
+            The next sequence number (current snapshot's sequence + 1, or 1 if no snapshots).
+        """
+        if not self.metadata.snapshots:
+            # No snapshots yet - this is the first one
+            return 1
+
+        # Get the current (most recent) snapshot - should have the highest sequence number
+        current = self.snapshot()
+        if current:
+            seq = getattr(current, "sequence_number", None)
+            return int(seq) + 1 if seq is not None else 1
+
     def snapshot(self, snapshot_id: Optional[int] = None) -> Optional[Snapshot]:
         """Return a Snapshot.
 
@@ -95,20 +118,17 @@ class SimpleDataset(Dataset):
                 if doc.exists:
                     sd = doc.to_dict() or {}
                     snap = Snapshot(
-                        snapshot_id=int(
-                            sd.get("snapshot-id") or sd.get("snapshot_id") or snapshot_id
-                        ),
-                        timestamp_ms=int(sd.get("timestamp-ms") or sd.get("timestamp_ms") or 0),
+                        snapshot_id=int(sd.get("snapshot-id") or snapshot_id),
+                        timestamp_ms=int(sd.get("timestamp-ms", 0)),
                         author=sd.get("author"),
-                        sequence_number=sd.get("sequence-number") or sd.get("sequence_number"),
-                        user_created=sd.get("user-created") or sd.get("user_created"),
-                        manifest_list=sd.get("manifest") or sd.get("manifest_list"),
-                        schema_id=sd.get("schema-id") or sd.get("schema_id"),
+                        sequence_number=sd.get("sequence-number", 0),
+                        user_created=sd.get("user-created"),
+                        manifest_list=sd.get("manifest"),
+                        schema_id=sd.get("schema-id"),
                         summary=sd.get("summary", {}),
-                        operation_type=sd.get("operation-type") or sd.get("operation_type"),
-                        parent_snapshot_id=sd.get("parent-snapshot-id")
-                        or sd.get("parent_snapshot_id"),
-                        commit_message=sd.get("commit-message") or sd.get("commit_message"),
+                        operation_type=sd.get("operation-type"),
+                        parent_snapshot_id=sd.get("parent-snapshot-id"),
+                        commit_message=sd.get("commit-message"),
                     )
                     return snap
             except Exception:
@@ -227,148 +247,9 @@ class SimpleDataset(Dataset):
         if not hasattr(table, "schema"):
             raise TypeError("append() expects a pyarrow.Table-like object")
 
-        # Write parquet file with collision-resistant name
-        fname = f"{time.time_ns():x}-{self._get_node()}.parquet"
-        data_path = f"{self.metadata.location}/data/{fname}"
-        buf = pa.BufferOutputStream()
-        pq.write_table(table, buf, compression="zstd")
-        pdata = buf.getvalue().to_pybytes()
-
-        out = self.io.new_output(data_path).create()
-        out.write(pdata)
-        out.close()
-
-        # Prepare sketches/stats
-        K = 32
-        HBINS = 32
-        min_k_hashes: list[list[int]] = []
-        histograms: list[list[int]] = []
-        min_values: list[int] = []
-        max_values: list[int] = []
-
-        # Use draken for efficient hashing and compression when available.
-        import heapq
-
-        # canonical NULL flag for missing values
-        NULL_FLAG = -(1 << 63)
-
-        try:
-            import opteryx.draken as draken  # type: ignore
-
-            num_rows = int(table.num_rows)
-
-            for col_idx, col in enumerate(table.columns):
-                # hash column values to 64-bit via draken (new cpdef API)
-                vec = draken.Vector.from_arrow(col)
-                hashes = list(vec.hash())
-
-                # Decide whether to compute min-k/histogram for this column based
-                # on field type and, for strings, average length of values.
-                field_type = table.schema.field(col_idx).type
-                compute_min_k = False
-                if (
-                    pa.types.is_integer(field_type)
-                    or pa.types.is_floating(field_type)
-                    or pa.types.is_decimal(field_type)
-                ):
-                    compute_min_k = True
-                elif (
-                    pa.types.is_timestamp(field_type)
-                    or pa.types.is_date(field_type)
-                    or pa.types.is_time(field_type)
-                ):
-                    compute_min_k = True
-                elif pa.types.is_string(field_type) or pa.types.is_large_string(field_type):
-                    # compute average length from non-null values; only allow
-                    # min-k/histogram for short strings (avg <= 16)
-                    col_py = None
-                    try:
-                        col_py = col.to_pylist()
-                    except Exception:
-                        col_py = None
-
-                    if col_py is not None:
-                        lens = [len(x) for x in col_py if x is not None]
-                        if lens:
-                            avg_len = sum(lens) / len(lens)
-                            if avg_len <= 16:
-                                compute_min_k = True
-
-                # KMV: take K smallest hashes when allowed; otherwise store an
-                # empty list for this column.
-                if compute_min_k:
-                    smallest = heapq.nsmallest(K, hashes)
-                    col_min_k = sorted(smallest)
-                else:
-                    col_min_k = []
-
-                # For histogram decisions follow the same rule as min-k
-                compute_hist = compute_min_k
-
-                # Use draken.compress() to get canonical int64 per value
-                mapped = list(vec.compress())
-                non_nulls_mapped = [m for m in mapped if m != NULL_FLAG]
-                if non_nulls_mapped:
-                    vmin = min(non_nulls_mapped)
-                    vmax = max(non_nulls_mapped)
-                    col_min = int(vmin)
-                    col_max = int(vmax)
-                    if compute_hist:
-                        if vmin == vmax:
-                            col_hist = [0] * HBINS
-                            col_hist[-1] = len(non_nulls_mapped)
-                        else:
-                            col_hist = [0] * HBINS
-                            span = float(vmax - vmin)
-                            for m in non_nulls_mapped:
-                                b = int(((float(m) - float(vmin)) / span) * (HBINS - 1))
-                                if b < 0:
-                                    b = 0
-                                if b >= HBINS:
-                                    b = HBINS - 1
-                                col_hist[b] += 1
-                    else:
-                        col_hist = [0] * HBINS
-                else:
-                    # no non-null values; histogram via hash buckets
-                    col_min = NULL_FLAG
-                    col_max = NULL_FLAG
-                    if compute_hist:
-                        col_hist = [0] * HBINS
-                        for h in hashes:
-                            b = (h >> (64 - 5)) & 0x1F
-                            col_hist[b] += 1
-                    else:
-                        col_hist = [0] * HBINS
-
-                min_k_hashes.append(col_min_k)
-                histograms.append(col_hist)
-                min_values.append(col_min)
-                max_values.append(col_max)
-        except Exception:
-            # If draken or its dependencies are unavailable, fall back to
-            # conservative defaults so we can still write the manifest and
-            # snapshot without failing the append operation.
-            num_cols = table.num_columns
-            min_k_hashes = [[] for _ in range(num_cols)]
-            HBINS = 32
-            histograms = [[0] * HBINS for _ in range(num_cols)]
-            min_values = [NULL_FLAG] * num_cols
-            max_values = [NULL_FLAG] * num_cols
-
-        entries = [
-            {
-                "file_path": data_path,
-                "file_format": "parquet",
-                "record_count": int(table.num_rows),
-                "file_size_in_bytes": len(pdata),
-                "min_k_hashes": min_k_hashes,
-                "histogram_counts": histograms,
-                "histogram_bins": HBINS,
-                "min_values": min_values,
-                "max_values": max_values,
-            }
-        ]
+        # Write table and build manifest entry
+        manifest_entry = self._write_table_and_build_entry(table)
+        entries = [manifest_entry.to_dict()]
 
         # persist manifest: for append, merge previous manifest entries
         # with the new entries so the snapshot's manifest is cumulative.
@@ -384,35 +265,15 @@ class SimpleDataset(Dataset):
                 prev_manifest_path = prev_snap.manifest_list
                 try:
                     # Prefer FileIO when available
-                    if self.io and hasattr(self.io, "new_input"):
-                        inp = self.io.new_input(prev_manifest_path)
-                        with inp.open() as f:
-                            prev_data = f.read()
-                        import pyarrow as pa
-                        import pyarrow.parquet as pq
-
-                        prev_table = pq.read_table(pa.BufferReader(prev_data))
-                        prev_rows = prev_table.to_pylist()
-                        merged_entries = prev_rows + merged_entries
-                    else:
-                        # Fall back to catalog storage client (GCS)
-                        if (
-                            self.catalog
-                            and getattr(self.catalog, "_storage_client", None)
-                            and getattr(self.catalog, "gcs_bucket", None)
-                        ):
-                            bucket = self.catalog._storage_client.bucket(self.catalog.gcs_bucket)
-                            parsed = prev_manifest_path
-                            if parsed.startswith("gs://"):
-                                parsed = parsed[5 + len(self.catalog.gcs_bucket) + 1 :]
-                            blob = bucket.blob(parsed)
-                            prev_data = blob.download_as_bytes()
-                            import pyarrow as pa
-                            import pyarrow.parquet as pq
-
-                            prev_table = pq.read_table(pa.BufferReader(prev_data))
-                            prev_rows = prev_table.to_pylist()
-                            merged_entries = prev_rows + merged_entries
+                    inp = self.io.new_input(prev_manifest_path)
+                    with inp.open() as f:
+                        prev_data = f.read()
+                    import pyarrow as pa
+                    import pyarrow.parquet as pq
+
+                    prev_table = pq.read_table(pa.BufferReader(prev_data))
+                    prev_rows = prev_table.to_pylist()
+                    merged_entries = prev_rows + merged_entries
                 except Exception:
                     # If we can't read the previous manifest, continue with
                     # just the new entries (don't fail the append).
@@ -433,63 +294,52 @@ class SimpleDataset(Dataset):
             commit_message = f"commit by {author}"
 
         recs = int(table.num_rows)
-        fsize = len(pdata)
+        fsize = int(getattr(manifest_entry, "file_size_in_bytes", 0))
+        # Calculate uncompressed size from the manifest entry
+        added_data_size = manifest_entry.uncompressed_size_in_bytes
         added_data_files = 1
         added_files_size = fsize
         added_records = recs
         deleted_data_files = 0
         deleted_files_size = 0
+        deleted_data_size = 0
         deleted_records = 0
 
         prev = self.snapshot()
         if prev and prev.summary:
-            try:
-                prev_total_files = int(prev.summary.get("total-data-files", 0))
-            except Exception:
-                prev_total_files = 0
-            try:
-                prev_total_size = int(prev.summary.get("total-files-size", 0))
-            except Exception:
-                prev_total_size = 0
-            try:
-                prev_total_records = int(prev.summary.get("total-records", 0))
-            except Exception:
-                prev_total_records = 0
+            prev_total_files = int(prev.summary.get("total-data-files", 0))
+            prev_total_size = int(prev.summary.get("total-files-size", 0))
+            prev_total_data_size = int(prev.summary.get("total-data-size", 0))
+            prev_total_records = int(prev.summary.get("total-records", 0))
         else:
             prev_total_files = 0
             prev_total_size = 0
+            prev_total_data_size = 0
             prev_total_records = 0
 
         total_data_files = prev_total_files + added_data_files - deleted_data_files
         total_files_size = prev_total_size + added_files_size - deleted_files_size
+        total_data_size = prev_total_data_size + added_data_size - deleted_data_size
         total_records = prev_total_records + added_records - deleted_records
 
         summary = {
             "added-data-files": added_data_files,
             "added-files-size": added_files_size,
+            "added-data-size": added_data_size,
             "added-records": added_records,
             "deleted-data-files": deleted_data_files,
             "deleted-files-size": deleted_files_size,
+            "deleted-data-size": deleted_data_size,
             "deleted-records": deleted_records,
             "total-data-files": total_data_files,
             "total-files-size": total_files_size,
+            "total-data-size": total_data_size,
             "total-records": total_records,
         }
 
         # sequence number
         try:
-            max_seq = 0
-            for s in self.metadata.snapshots:
-                seq = getattr(s, "sequence_number", None)
-                if seq is None:
-                    continue
-                try:
-                    ival = int(seq)
-                except Exception:
-                    continue
-                if ival > max_seq:
-                    max_seq = ival
-            next_seq = max_seq + 1
+            next_seq = self._next_sequence_number()
         except Exception:
             next_seq = 1
 
@@ -518,6 +368,136 @@ class SimpleDataset(Dataset):
         if self.catalog and hasattr(self.catalog, "save_dataset_metadata"):
             self.catalog.save_dataset_metadata(self.identifier, self.metadata)
 
+    def _write_table_and_build_entry(self, table: Any):
+        """Write a PyArrow table to storage and return a ParquetManifestEntry.
+
+        This centralizes the IO and manifest construction so other operations
+        (e.g. `overwrite`) can reuse the same behavior as `append`.
+        """
+        # Write parquet file with collision-resistant name
+        fname = f"{time.time_ns():x}-{self._get_node()}.parquet"
+        data_path = f"{self.metadata.location}/data/{fname}"
+
+        import pyarrow as pa
+        import pyarrow.parquet as pq
+
+        buf = pa.BufferOutputStream()
+        pq.write_table(table, buf, compression="zstd")
+        pdata = buf.getvalue().to_pybytes()
+
+        out = self.io.new_output(data_path).create()
+        out.write(pdata)
+        out.close()
+
+        # Build manifest entry with statistics
+        manifest_entry = build_parquet_manifest_entry(table, data_path, len(pdata))
+        return manifest_entry
+
+    def overwrite(self, table: Any, author: str = None, commit_message: Optional[str] = None):
+        """Replace the dataset entirely with `table` in a single snapshot.
+
+        Semantics:
+        - Write the provided table as new data file(s)
+        - Create a new parquet manifest that contains only the new entries
+        - Create a snapshot that records previous files as deleted and the
+          new files as added (logical replace)
+        """
+        # Similar validation as append
+        snapshot_id = int(time.time() * 1000)
+
+        if not hasattr(table, "schema"):
+            raise TypeError("overwrite() expects a pyarrow.Table-like object")
+
+        if author is None:
+            raise ValueError("author must be provided when overwriting a dataset")
+
+        # Write new data and build manifest entries (single table -> single entry)
+        manifest_entry = self._write_table_and_build_entry(table)
+        new_entries = [manifest_entry.to_dict()]
+
+        # Write manifest containing only the new entries
+        manifest_path = None
+        if self.catalog and hasattr(self.catalog, "write_parquet_manifest"):
+            manifest_path = self.catalog.write_parquet_manifest(
+                snapshot_id, new_entries, self.metadata.location
+            )
+
+        # Compute deltas: previous manifest becomes deleted
+        prev = self.snapshot(None)
+        prev_total_files = 0
+        prev_total_size = 0
+        prev_total_data_size = 0
+        prev_total_records = 0
+        if prev and prev.summary:
+            prev_total_files = int(prev.summary.get("total-data-files", 0))
+            prev_total_size = int(prev.summary.get("total-files-size", 0))
+            prev_total_data_size = int(prev.summary.get("total-data-size", 0))
+            prev_total_records = int(prev.summary.get("total-records", 0))
+
+        deleted_data_files = prev_total_files
+        deleted_files_size = prev_total_size
+        deleted_data_size = prev_total_data_size
+        deleted_records = prev_total_records
+
+        added_data_files = len(new_entries)
+        added_files_size = sum(e.get("file_size_in_bytes", 0) for e in new_entries)
+        added_data_size = sum(e.get("uncompressed_size_in_bytes", 0) for e in new_entries)
+        added_records = sum(e.get("record_count", 0) for e in new_entries)
+
+        total_data_files = added_data_files
+        total_files_size = added_files_size
+        total_data_size = added_data_size
+        total_records = added_records
+
+        summary = {
+            "added-data-files": added_data_files,
+            "added-files-size": added_files_size,
+            "added-data-size": added_data_size,
+            "added-records": added_records,
+            "deleted-data-files": deleted_data_files,
+            "deleted-files-size": deleted_files_size,
+            "deleted-data-size": deleted_data_size,
+            "deleted-records": deleted_records,
+            "total-data-files": total_data_files,
+            "total-files-size": total_files_size,
+            "total-data-size": total_data_size,
+            "total-records": total_records,
+        }
+
+        # sequence number
+        try:
+            next_seq = self._next_sequence_number()
+        except Exception:
+            next_seq = 1
+
+        parent_id = self.metadata.current_snapshot_id
+
+        if commit_message is None:
+            commit_message = f"overwrite by {author}"
+
+        snap = Snapshot(
+            snapshot_id=snapshot_id,
+            timestamp_ms=snapshot_id,
+            author=author,
+            sequence_number=next_seq,
+            user_created=True,
+            operation_type="overwrite",
+            parent_snapshot_id=parent_id,
+            manifest_list=manifest_path,
+            schema_id=self.metadata.current_schema_id,
+            commit_message=commit_message,
+            summary=summary,
+        )
+
+        # Replace in-memory snapshots
+        self.metadata.snapshots.append(snap)
+        self.metadata.current_snapshot_id = snapshot_id
+
+        if self.catalog and hasattr(self.catalog, "save_snapshot"):
+            self.catalog.save_snapshot(self.identifier, snap)
+        if self.catalog and hasattr(self.catalog, "save_dataset_metadata"):
+            self.catalog.save_dataset_metadata(self.identifier, self.metadata)
+
     def add_files(self, files: list[str], author: str = None, commit_message: Optional[str] = None):
         """Add filenames to the dataset manifest without writing the files.
 
@@ -540,45 +520,20 @@ class SimpleDataset(Dataset):
         prev_total_records = 0
         prev_entries = []
         if prev and prev.summary:
-            try:
-                prev_total_files = int(prev.summary.get("total-data-files", 0))
-            except Exception:
-                prev_total_files = 0
-            try:
-                prev_total_size = int(prev.summary.get("total-files-size", 0))
-            except Exception:
-                prev_total_size = 0
-            try:
-                prev_total_records = int(prev.summary.get("total-records", 0))
-            except Exception:
-                prev_total_records = 0
-
+            prev_total_files = int(prev.summary.get("total-data-files", 0))
+            prev_total_size = int(prev.summary.get("total-files-size", 0))
+            prev_total_records = int(prev.summary.get("total-records", 0))
         if prev and getattr(prev, "manifest_list", None):
             # try to read prev manifest entries
             try:
                 import pyarrow as pa
                 import pyarrow.parquet as pq
 
-                if self.io and hasattr(self.io, "new_input"):
-                    inp = self.io.new_input(prev.manifest_list)
-                    with inp.open() as f:
-                        data = f.read()
-                    table = pq.read_table(pa.BufferReader(data))
-                    prev_entries = table.to_pylist()
-                else:
-                    if (
-                        self.catalog
-                        and getattr(self.catalog, "_storage_client", None)
-                        and getattr(self.catalog, "gcs_bucket", None)
-                    ):
-                        bucket = self.catalog._storage_client.bucket(self.catalog.gcs_bucket)
-                        parsed = prev.manifest_list
-                        if parsed.startswith("gs://"):
-                            parsed = parsed[5 + len(self.catalog.gcs_bucket) + 1 :]
-                        blob = bucket.blob(parsed)
-                        data = blob.download_as_bytes()
-                        table = pq.read_table(pa.BufferReader(data))
-                        prev_entries = table.to_pylist()
+                inp = self.io.new_input(prev.manifest_list)
+                with inp.open() as f:
+                    data = f.read()
+                table = pq.read_table(pa.BufferReader(data))
+                prev_entries = table.to_pylist()
             except Exception:
                 prev_entries = []
 
@@ -601,146 +556,47 @@ class SimpleDataset(Dataset):
             seen.add(fp)
 
             # Attempt to read file bytes and parquet metadata
-            file_size = 0
-            record_count = 0
-            min_values = []
-            max_values = []
+            # Use rugo's metadata reader which is much faster (microseconds per file)
             try:
-                import pyarrow as pa
-                import pyarrow.parquet as pq
-
-                data = None
-                if self.io and hasattr(self.io, "new_input"):
-                    inp = self.io.new_input(fp)
-                    with inp.open() as f:
-                        data = f.read()
-                else:
-                    if (
-                        self.catalog
-                        and getattr(self.catalog, "_storage_client", None)
-                        and getattr(self.catalog, "gcs_bucket", None)
-                    ):
-                        bucket = self.catalog._storage_client.bucket(self.catalog.gcs_bucket)
-                        parsed = fp
-                        if parsed.startswith("gs://"):
-                            parsed = parsed[5 + len(self.catalog.gcs_bucket) + 1 :]
-                        blob = bucket.blob(parsed)
-                        data = blob.download_as_bytes()
+                inp = self.io.new_input(fp)
+                with inp.open() as f:
+                    data = f.read()
 
                 if data:
-                    file_size = len(data)
-                    pf = pq.ParquetFile(pa.BufferReader(data))
-                    record_count = int(pf.metadata.num_rows or 0)
-
-                    # Prefer computing min/max via draken.compress() over
-                    # relying on Parquet footer stats which may contain
-                    # heterogenous or non-numeric values. Fall back to
-                    # footer stats only if draken is unavailable.
-                    try:
-                        import opteryx.draken as draken  # type: ignore
-
-                        table = pq.read_table(pa.BufferReader(data))
-                        ncols = table.num_columns
-                        mins = [None] * ncols
-                        maxs = [None] * ncols
-
-                        NULL_FLAG = -(1 << 63)
-
-                        for ci in range(ncols):
-                            try:
-                                col = table.column(ci)
-                                # combine chunks if needed
-                                if hasattr(col, "combine_chunks"):
-                                    arr = col.combine_chunks()
-                                else:
-                                    arr = col
-                                vec = draken.Vector.from_arrow(arr)
-                                mapped = list(vec.compress())
-                                non_nulls = [m for m in mapped if m != NULL_FLAG]
-                                if non_nulls:
-                                    mins[ci] = int(min(non_nulls))
-                                    maxs[ci] = int(max(non_nulls))
-                                else:
-                                    mins[ci] = None
-                                    maxs[ci] = None
-                            except Exception:
-                                # per-column fallback: leave None
-                                mins[ci] = None
-                                maxs[ci] = None
-                    except Exception:
-                        # Draken not available; fall back to Parquet footer stats
-                        ncols = pf.metadata.num_columns
-                        mins = [None] * ncols
-                        maxs = [None] * ncols
-                        for rg in range(pf.num_row_groups):
-                            for ci in range(ncols):
-                                col_meta = pf.metadata.row_group(rg).column(ci)
-                                stats = getattr(col_meta, "statistics", None)
-                                if not stats:
-                                    continue
-                                smin = getattr(stats, "min", None)
-                                smax = getattr(stats, "max", None)
-                                if smin is None and smax is None:
-                                    continue
-
-                                def _to_py(v):
-                                    try:
-                                        return int(v)
-                                    except Exception:
-                                        try:
-                                            return float(v)
-                                        except Exception:
-                                            try:
-                                                if isinstance(v, (bytes, bytearray)):
-                                                    return v.decode("utf-8", errors="ignore")
-                                            except Exception:
-                                                pass
-                                            return v
-
-                                if smin is not None:
-                                    sval = _to_py(smin)
-                                    if mins[ci] is None:
-                                        mins[ci] = sval
-                                    else:
-                                        try:
-                                            if sval < mins[ci]:
-                                                mins[ci] = sval
-                                        except Exception:
-                                            pass
-                                if smax is not None:
-                                    sval = _to_py(smax)
-                                    if maxs[ci] is None:
-                                        maxs[ci] = sval
-                                    else:
-                                        try:
-                                            if sval > maxs[ci]:
-                                                maxs[ci] = sval
-                                        except Exception:
-                                            pass
-
-                    # normalize lists to empty lists when values missing
-                    min_values = [m for m in mins if m is not None]
-                    max_values = [m for m in maxs if m is not None]
+                    manifest_entry = build_parquet_manifest_minmax_entry(data, fp)
+                else:
+                    # Empty file, create placeholder entry
+                    manifest_entry = ParquetManifestEntry(
+                        file_path=fp,
+                        file_format="parquet",
+                        record_count=0,
+                        null_counts=[],
+                        file_size_in_bytes=0,
+                        uncompressed_size_in_bytes=0,
+                        column_uncompressed_sizes_in_bytes=[],
+                        min_k_hashes=[],
+                        histogram_counts=[],
+                        histogram_bins=0,
+                        min_values=[],
+                        max_values=[],
+                    )
             except Exception:
                 # If metadata read fails, fall back to placeholders
-                file_size = 0
-                record_count = 0
-                min_values = []
-                max_values = []
-
-            new_entries.append(
-                {
-                    "file_path": fp,
-                    "file_format": "parquet",
-                    "record_count": int(record_count),
-                    "file_size_in_bytes": int(file_size),
-                    "min_k_hashes": [],
-                    "histogram_counts": [],
-                    "histogram_bins": 0,
-                    "min_values": min_values,
-                    "max_values": max_values,
-                }
-            )
+                manifest_entry = ParquetManifestEntry(
+                    file_path=fp,
+                    file_format="parquet",
+                    record_count=0,
+                    null_counts=[],
+                    file_size_in_bytes=0,
+                    uncompressed_size_in_bytes=0,
+                    column_uncompressed_sizes_in_bytes=[],
+                    min_k_hashes=[],
+                    histogram_counts=[],
+                    histogram_bins=0,
+                    min_values=[],
+                    max_values=[],
+                )
+            new_entries.append(manifest_entry.to_dict())
 
         merged_entries = prev_entries + new_entries
 
@@ -754,41 +610,43 @@ class SimpleDataset(Dataset):
         # Build summary deltas
         added_data_files = len(new_entries)
         added_files_size = 0
+        added_data_size = 0
         added_records = 0
+        # Sum uncompressed sizes from new entries
+        for entry in new_entries:
+            added_data_size += entry.get("uncompressed_size_in_bytes", 0)
         deleted_data_files = 0
         deleted_files_size = 0
+        deleted_data_size = 0
         deleted_records = 0
 
+        prev_total_data_size = (
+            int(prev.summary.get("total-data-size", 0)) if prev and prev.summary else 0
+        )
+
         total_data_files = prev_total_files + added_data_files - deleted_data_files
         total_files_size = prev_total_size + added_files_size - deleted_files_size
+        total_data_size = prev_total_data_size + added_data_size - deleted_data_size
         total_records = prev_total_records + added_records - deleted_records
 
         summary = {
             "added-data-files": added_data_files,
             "added-files-size": added_files_size,
+            "added-data-size": added_data_size,
             "added-records": added_records,
             "deleted-data-files": deleted_data_files,
             "deleted-files-size": deleted_files_size,
+            "deleted-data-size": deleted_data_size,
             "deleted-records": deleted_records,
             "total-data-files": total_data_files,
             "total-files-size": total_files_size,
+            "total-data-size": total_data_size,
             "total-records": total_records,
         }
 
         # Sequence number
         try:
-            max_seq = 0
-            for s in self.metadata.snapshots:
-                seq = getattr(s, "sequence_number", None)
-                if seq is None:
-                    continue
-                try:
-                    ival = int(seq)
-                except Exception:
-                    continue
-                if ival > max_seq:
-                    max_seq = ival
-            next_seq = max_seq + 1
+            next_seq = self._next_sequence_number()
         except Exception:
             next_seq = 1
 
@@ -897,6 +755,7 @@ class SimpleDataset(Dataset):
                     ncols = pf.metadata.num_columns
                     mins = [None] * ncols
                     maxs = [None] * ncols
+                    null_counts = [0] * ncols
                     for rg in range(pf.num_row_groups):
                         for ci in range(ncols):
                             col_meta = pf.metadata.row_group(rg).column(ci)
@@ -905,7 +764,8 @@ class SimpleDataset(Dataset):
                                 continue
                             smin = getattr(stats, "min", None)
                             smax = getattr(stats, "max", None)
-                            if smin is None and smax is None:
+                            snull_count = getattr(stats, "null_count", None)
+                            if smin is None and smax is None and snull_count is None:
                                 continue
 
                             def _to_py(v):
@@ -942,6 +802,11 @@ class SimpleDataset(Dataset):
                                             maxs[ci] = sval
                                     except Exception:
                                         pass
+                            if snull_count is not None:
+                                try:
+                                    null_counts[ci] += int(snull_count)
+                                except Exception:
+                                    pass
 
                     min_values = [m for m in mins if m is not None]
                     max_values = [m for m in maxs if m is not None]
@@ -950,20 +815,23 @@ class SimpleDataset(Dataset):
                 record_count = 0
                 min_values = []
                 max_values = []
-
-            new_entries.append(
-                {
-                    "file_path": fp,
-                    "file_format": "parquet",
-                    "record_count": int(record_count),
-                    "file_size_in_bytes": int(file_size),
-                    "min_k_hashes": [],
-                    "histogram_counts": [],
-                    "histogram_bins": 0,
-                    "min_values": min_values,
-                    "max_values": max_values,
-                }
+                null_counts = []
+
+            manifest_entry = ParquetManifestEntry(
+                file_path=fp,
+                file_format="parquet",
+                record_count=int(record_count),
+                null_counts=null_counts,
+                file_size_in_bytes=int(file_size),
+                uncompressed_size_in_bytes=int(file_size),  # Use compressed size as estimate
+                column_uncompressed_sizes_in_bytes=[],
+                min_k_hashes=[],
+                histogram_counts=[],
+                histogram_bins=0,
+                min_values=min_values,
+                max_values=max_values,
             )
+            new_entries.append(manifest_entry.to_dict())
 
         manifest_path = None
         if self.catalog and hasattr(self.catalog, "write_parquet_manifest"):
@@ -974,42 +842,42 @@ class SimpleDataset(Dataset):
         # Build summary: previous entries become deleted
         deleted_data_files = prev_total_files
         deleted_files_size = prev_total_size
+        deleted_data_size = (
+            int(prev.summary.get("total-data-size", 0)) if prev and prev.summary else 0
+        )
         deleted_records = prev_total_records
 
         added_data_files = len(new_entries)
         added_files_size = 0
+        added_data_size = 0
+        # Sum uncompressed sizes from new entries
+        for entry in new_entries:
+            added_data_size += entry.get("uncompressed_size_in_bytes", 0)
         added_records = 0
 
         total_data_files = added_data_files
         total_files_size = added_files_size
+        total_data_size = added_data_size
         total_records = added_records
 
         summary = {
             "added-data-files": added_data_files,
             "added-files-size": added_files_size,
+            "added-data-size": added_data_size,
             "added-records": added_records,
             "deleted-data-files": deleted_data_files,
             "deleted-files-size": deleted_files_size,
+            "deleted-data-size": deleted_data_size,
             "deleted-records": deleted_records,
             "total-data-files": total_data_files,
             "total-files-size": total_files_size,
+            "total-data-size": total_data_size,
             "total-records": total_records,
         }
 
         # Sequence number
         try:
-            max_seq = 0
-            for s in self.metadata.snapshots:
-                seq = getattr(s, "sequence_number", None)
-                if seq is None:
-                    continue
-                try:
-                    ival = int(seq)
-                except Exception:
-                    continue
-                if ival > max_seq:
-                    max_seq = ival
-            next_seq = max_seq + 1
+            next_seq = self._next_sequence_number()
         except Exception:
             next_seq = 1
 
@@ -1042,13 +910,11 @@ class SimpleDataset(Dataset):
             self.catalog.save_dataset_metadata(self.identifier, self.metadata)
 
     def scan(
-        self, row_filter=None, row_limit=None, snapshot_id: Optional[int] = None
+        self, row_filter=None, snapshot_id: Optional[int] = None
     ) -> Iterable[Datafile]:
         """Return Datafile objects for the given snapshot.
 
         - If `snapshot_id` is None, use the current snapshot.
-        - Ignore `row_filter` for now and return all files listed in the
-          snapshot's parquet manifest (if present).
         """
         # Determine snapshot to read using the dataset-level helper which
         # prefers the in-memory current snapshot and otherwise performs a
@@ -1065,8 +931,6 @@ class SimpleDataset(Dataset):
             import pyarrow as pa
             import pyarrow.parquet as pq
 
-            data = None
-
             inp = self.io.new_input(manifest_path)
             with inp.open() as f:
                 data = f.read()
@@ -1076,23 +940,148 @@ class SimpleDataset(Dataset):
 
             table = pq.read_table(pa.BufferReader(data))
             rows = table.to_pylist()
-            cum_rows = 0
             for r in rows:
                 yield Datafile(entry=r)
-                try:
-                    rc = int(r.get("record_count") or 0)
-                except Exception:
-                    rc = 0
-                cum_rows += rc
-                if row_limit is not None and cum_rows >= row_limit:
-                    break
         except FileNotFoundError:
             return iter(())
         except Exception:
             return iter(())
 
+    def refresh_manifest(self, agent: str, author: Optional[str] = None) -> Optional[int]:
+        """Refresh manifest statistics and create a new snapshot.
+
+        - `agent`: identifier for the agent performing the refresh (string)
+        - `author`: optional author to record; if omitted uses current snapshot author
+
+        This recalculates per-file statistics (min/max, record counts, sizes)
+        for every file in the current manifest, writes a new manifest and
+        creates a new snapshot with `user_created=False` and
+        `operation_type='statistics-refresh'`.
+
+        Returns the new `snapshot_id` on success or None on failure.
+        """
+        prev = self.snapshot(None)
+        if prev is None or not getattr(prev, "manifest_list", None):
+            raise ValueError("No current manifest available to refresh")
+
+        # Use same author/commit-timestamp as previous snapshot unless overridden
+        use_author = author if author is not None else getattr(prev, "author", None)
+
+        snapshot_id = int(time.time() * 1000)
+
+        # Rebuild manifest entries by re-reading each data file
+        entries = []
+        try:
+            # Read previous manifest entries
+            inp = self.io.new_input(prev.manifest_list)
+            with inp.open() as f:
+                prev_data = f.read()
+            import pyarrow as pa
+            import pyarrow.parquet as pq
+
+            prev_table = pq.read_table(pa.BufferReader(prev_data))
+            prev_rows = prev_table.to_pylist()
+        except Exception:
+            prev_rows = []
+
+        total_files = 0
+        total_size = 0
+        total_data_size = 0
+        total_records = 0
+
+        for ent in prev_rows:
+            if not isinstance(ent, dict):
+                continue
+            fp = ent.get("file_path")
+            if not fp:
+                continue
+            try:
+                inp = self.io.new_input(fp)
+                with inp.open() as f:
+                    data = f.read()
+                # Full statistics including histograms and k-hashes
+                table = pq.read_table(pa.BufferReader(data))
+                manifest_entry = build_parquet_manifest_entry(table, fp, len(data))
+                dent = manifest_entry.to_dict()
+            except Exception:
+                # Fall back to original entry if re-read fails
+                dent = ent
+
+            entries.append(dent)
+            total_files += 1
+            total_size += int(dent.get("file_size_in_bytes") or 0)
+            total_data_size += int(dent.get("uncompressed_size_in_bytes") or 0)
+            total_records += int(dent.get("record_count") or 0)
+
+        # write new manifest
+        manifest_path = self.catalog.write_parquet_manifest(
+            snapshot_id, entries, self.metadata.location
+        )
+
+        # Build summary
+        summary = {
+            "added-data-files": 0,
+            "added-files-size": 0,
+            "added-data-size": 0,
+            "added-records": 0,
+            "deleted-data-files": 0,
+            "deleted-files-size": 0,
+            "deleted-data-size": 0,
+            "deleted-records": 0,
+            "total-data-files": total_files,
+            "total-files-size": total_size,
+            "total-data-size": total_data_size,
+            "total-records": total_records,
+        }
+
+        # sequence number
+        try:
+            next_seq = self._next_sequence_number()
+        except Exception:
+            next_seq = 1
+
+        parent_id = self.metadata.current_snapshot_id
+
+        # Agent committer metadata
+        agent_meta = {
+            "timestamp": int(time.time() * 1000),
+            "action": "statistics-refresh",
+            "agent": agent,
+        }
+
+        snap = Snapshot(
+            snapshot_id=snapshot_id,
+            timestamp_ms=getattr(prev, "timestamp_ms", snapshot_id),
+            author=use_author,
+            sequence_number=next_seq,
+            user_created=False,
+            operation_type="statistics-refresh",
+            parent_snapshot_id=parent_id,
+            manifest_list=manifest_path,
+            schema_id=self.metadata.current_schema_id,
+            commit_message=getattr(prev, "commit_message", "statistics refresh"),
+            summary=summary,
+        )
+
+        # attach agent metadata under summary
+        if snap.summary is None:
+            snap.summary = {}
+        snap.summary["agent-committer"] = agent_meta
+
+        # update in-memory metadata
+        self.metadata.snapshots.append(snap)
+        self.metadata.current_snapshot_id = snapshot_id
+
+        # persist
+        if self.catalog and hasattr(self.catalog, "save_snapshot"):
+            self.catalog.save_snapshot(self.identifier, snap)
+        if self.catalog and hasattr(self.catalog, "save_dataset_metadata"):
+            self.catalog.save_dataset_metadata(self.identifier, self.metadata)
+
+        return snapshot_id
+
     def truncate(self, author: str = None, commit_message: Optional[str] = None) -> None:
-        """Delete all data files and manifests for this table.
+        """Delete all data files and manifests for this dataset.
 
         This attempts to delete every data file referenced by existing
         Parquet manifests and then delete the manifest files themselves.
@@ -1109,6 +1098,7 @@ class SimpleDataset(Dataset):
         snaps = list(self.metadata.snapshots)
         removed_files = []
         removed_total_size = 0
+        removed_data_size = 0
 
         for snap in snaps:
             manifest_path = getattr(snap, "manifest_list", None)
@@ -1118,31 +1108,34 @@ class SimpleDataset(Dataset):
             # Read manifest via FileIO if available
             rows = []
             try:
-                if hasattr(io, "new_input"):
-                    inp = io.new_input(manifest_path)
-                    with inp.open() as f:
-                        data = f.read()
-                    table = pq.read_table(pa.BufferReader(data))
-                    rows = table.to_pylist()
+                inp = io.new_input(manifest_path)
+                with inp.open() as f:
+                    data = f.read()
+                table = pq.read_table(pa.BufferReader(data))
+                rows = table.to_pylist()
             except Exception:
                 rows = []
 
             for r in rows:
                 fp = None
                 fsize = 0
+                data_size = 0
                 if isinstance(r, dict):
                     fp = r.get("file_path")
                     fsize = int(r.get("file_size_in_bytes") or 0)
+                    data_size = int(r.get("uncompressed_size_in_bytes") or 0)
                     if not fp and "data_file" in r and isinstance(r["data_file"], dict):
                         fp = r["data_file"].get("file_path") or r["data_file"].get("path")
                         fsize = int(r["data_file"].get("file_size_in_bytes") or 0)
+                        data_size = int(r["data_file"].get("uncompressed_size_in_bytes") or 0)
 
                 if fp:
                     removed_files.append(fp)
                     removed_total_size += fsize
+                    removed_data_size += data_size
 
         # Create a new empty Parquet manifest (entries=[]) to represent the
-        # truncated table for the new snapshot. Do not delete objects.
+        # truncated dataset for the new snapshot. Do not delete objects.
         snapshot_id = int(time.time() * 1000)
 
         # Do NOT write an empty Parquet manifest when there are no entries.
@@ -1157,29 +1150,21 @@ class SimpleDataset(Dataset):
         summary = {
             "added-data-files": 0,
             "added-files-size": 0,
+            "added-data-size": 0,
             "added-records": 0,
             "deleted-data-files": deleted_count,
             "deleted-files-size": deleted_size,
+            "deleted-data-size": removed_data_size,
             "deleted-records": 0,
             "total-data-files": 0,
             "total-files-size": 0,
+            "total-data-size": 0,
             "total-records": 0,
         }
 
         # Sequence number
         try:
-            max_seq = 0
-            for s in self.metadata.snapshots:
-                seq = getattr(s, "sequence_number", None)
-                if seq is None:
-                    continue
-                try:
-                    ival = int(seq)
-                except Exception:
-                    continue
-                if ival > max_seq:
-                    max_seq = ival
-            next_seq = max_seq + 1
+            next_seq = self._next_sequence_number()
         except Exception:
             next_seq = 1
 
@@ -1215,7 +1200,4 @@ class SimpleDataset(Dataset):
         self.metadata.current_snapshot_id = snapshot_id
 
         if self.catalog and hasattr(self.catalog, "save_snapshot"):
-            try:
-                self.catalog.save_snapshot(self.identifier, snap)
-            except Exception:
-                pass
+            self.catalog.save_snapshot(self.identifier, snap)