core-lens 0.1.dev191__tar.gz → 0.1.dev194__tar.gz

{core_lens-0.1.dev191 → core_lens-0.1.dev194}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: core-lens
-Version: 0.1.dev191
+Version: 0.1.dev194
 Summary: Query, analyse, and visualise CoreStack's microwatershed and Earth science data through a clean, composable Python API.
 Project-URL: Homepage, https://github.com/ApoorvaKashyap/core-lens
 Project-URL: Issues, https://github.com/ApoorvaKashyap/core-lens/issues

{core_lens-0.1.dev191 → core_lens-0.1.dev194}/src/core_lens/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.1.dev191'
-__version_tuple__ = version_tuple = (0, 1, 'dev191')
+__version__ = version = '0.1.dev194'
+__version_tuple__ = version_tuple = (0, 1, 'dev194')
 
 __commit_id__ = commit_id = None

{core_lens-0.1.dev191 → core_lens-0.1.dev194}/src/core_lens/base/entity.py

@@ -650,6 +650,7 @@ class BaseEntity(ABC):
             aoi_geometry=geometry,
             relationship=relationship,
             threshold=threshold,
+            storage_options=self._storage_options,
         )
 
         entity_name = _entity_name(type(self))

{core_lens-0.1.dev191 → core_lens-0.1.dev194}/src/core_lens/base/view.py

@@ -215,6 +215,7 @@ class View:
             aoi_geometry=geometry,
             relationship=relationship,
             threshold=threshold,
+            storage_options=self.entity._storage_options,
         )
 
         keys_lf = keys_df.lazy()

{core_lens-0.1.dev191 → core_lens-0.1.dev194}/src/core_lens/utils/polars_utils.py

@@ -137,12 +137,25 @@ def scan_with_key_filter(
 
     1. **Key filter** — restricts to entity instances whose key column(s) are
        in ``key_values``.  For a single-column key this is an ``is_in``
-       predicate pushed down to the Parquet reader.  For composite keys each
-       column is filtered independently (over-selects slightly, then pruned
-       by the join at collect time).
+       predicate; for composite keys each column is filtered independently
+       (slight over-select, pruned by the semi-join at collect time).
+
+       **S3 / cloud note:** ``is_in`` pushdown skips row groups only when the
+       Parquet file was written with Bloom filters on the key column (e.g.
+       ``pyarrow.parquet.write_table(..., bloom_filter_columns=[key])``).  Without
+       Bloom filters the Parquet reader downloads all row groups and filters
+       in-memory regardless.  Use the semi-join path (> 50 000 keys) or add
+       Bloom filters at write time to avoid full-file S3 reads.
 
     2. **Time filter** — an optional Polars expression appended with ``&``,
-       also pushed down if the Parquet file carries column statistics.
+       pushed down when the Parquet file carries min/max column statistics.
+
+    ``use_statistics=True`` and ``parallel="columns"`` are set explicitly on
+    the underlying ``pl.scan_parquet`` call so that:
+
+    * Row-group skipping via Parquet min/max statistics is always active.
+    * On wide Parquet files, column chunks within each row group are fetched
+      concurrently instead of serially — reduces round trips on S3.
 
     Args:
         path (str): Absolute path or cloud URI to a Parquet file.
@@ -159,7 +172,13 @@ def scan_with_key_filter(
 
     """
     _so = storage_options or {}
-    lf = pl.scan_parquet(path, hive_partitioning=True, storage_options=_so or None)
+    lf = pl.scan_parquet(
+        path,
+        hive_partitioning=True,
+        use_statistics=True,
+        parallel="columns",
+        storage_options=_so or None,
+    )
 
     # Use an inner join to filter the parquet file down to the exact requested keys.
     # This avoids building a massive literal expression tree (which consumes gigabytes
@@ -169,9 +188,10 @@ def scan_with_key_filter(
 
     kv_df = key_values.collect() if isinstance(key_values, pl.LazyFrame) else key_values
     if len(key_cols) == 1 and kv_df.height < 50000:
-        # Fast path: For reasonably sized AoIs, `is_in` enables perfect row-group
-        # predicate pushdown in the Parquet reader, avoiding a full table scan.
-        # The literal tree RAM issue only occurs with hundreds of thousands of keys.
+        # Fast path: `is_in` keeps the plan simple and enables row-group skipping
+        # via Parquet Bloom filters when the key column was written with them.
+        # Without Bloom filters on S3, all row groups are downloaded regardless;
+        # see docstring note above.
         key = key_cols[0]
         lf = lf.filter(pl.col(key).is_in(kv_df[key].to_list()))
     else:

{core_lens-0.1.dev191 → core_lens-0.1.dev194}/src/core_lens/utils/spatial.py

@@ -25,9 +25,16 @@ if TYPE_CHECKING:
 # dataset (not once per process).  Subsequent runs load the index directly
 # from the sidecar — no geometry decoding required.
 #
-# Sidecar location priority:
+# Local sidecar location priority:
 #   1. Alongside the static parquet directory/file (requires write access).
 #   2. ~/.cache/core_lens/ (fallback for read-only data roots).
+#
+# Cloud (S3 / GCS / ABFS) sidecar:
+#   Always stored in ~/.cache/core_lens/ keyed on a SHA-256 hash of the URI.
+#   A companion <fingerprint>.etag file records the ETag returned by the last
+#   HeadObject call.  On cache hit the ETag is re-fetched (one cheap metadata
+#   call) and compared; a mismatch triggers a full rebuild.  This approach
+#   avoids streaming the entire GeoParquet from S3 on every cold process start.
 # ---------------------------------------------------------------------------
 
 _SIDECAR_SUFFIX = ".bbox_index.parquet"
@@ -41,6 +48,8 @@ def _bbox_sidecar_path(static_path: str) -> pathlib.Path | None:
     not writable (e.g. read-only data root), falls back to
     ``~/.cache/core_lens/``.
 
+    For cloud URIs use :func:`_cloud_sidecar_path` instead.
+
     Args:
         static_path (str): Absolute local path to the static parquet file/directory.
 
@@ -66,6 +75,136 @@ def _bbox_sidecar_path(static_path: str) -> pathlib.Path | None:
     return sidecar
 
 
+def _cloud_sidecar_path(static_path: str) -> pathlib.Path:
+    """Return the ``~/.cache/core_lens/`` sidecar path for a cloud URI.
+
+    The path is keyed on a SHA-256 hash of the full URI so different buckets,
+    prefixes, or regions never collide.  The companion ``.etag`` file (same
+    stem, ``.etag`` suffix) stores the last-seen ETag for staleness detection.
+
+    Args:
+        static_path (str): Cloud URI (e.g. ``s3://bucket/prefix/``).
+
+    Returns:
+        pathlib.Path: Local cache path for the sidecar Parquet.
+
+    """
+    import hashlib
+
+    fingerprint = hashlib.sha256(static_path.encode()).hexdigest()[:24]
+    _CACHE_DIR.mkdir(parents=True, exist_ok=True)
+    return _CACHE_DIR / (fingerprint + _SIDECAR_SUFFIX)
+
+
+def _get_cloud_etag(
+    static_path: str,
+    storage_options: dict[str, Any],
+) -> str | None:
+    """Return the ETag (or size-based fingerprint) for a cloud object, or ``None``.
+
+    Performs a single metadata call (``HeadObject`` on S3) — no data is
+    transferred.  The ETag is used as the staleness key for the cloud bbox
+    sidecar: if the ETag matches the stored value the sidecar is fresh.
+
+    Falls back to the object size when the ETag is not exposed by the
+    filesystem implementation, and returns ``None`` on any error so the caller
+    can safely skip the cache.
+
+    Args:
+        static_path (str): Cloud URI to the static GeoParquet file or directory.
+        storage_options (dict[str, Any]): Cloud credential / configuration options.
+
+    Returns:
+        str | None: An opaque fingerprint string, or ``None`` on failure.
+
+    """
+    try:
+        import pyarrow.fs as pafs
+
+        fs, arrow_path = resolve_fs_and_path(static_path)
+        # For a directory (partitioned dataset) inspect the first child file.
+        info = fs.get_file_info(arrow_path)
+        if info.type == pafs.FileType.Directory:
+            selector = pafs.FileSelector(arrow_path, recursive=False)
+            children = fs.get_file_info(selector)
+            parquet_children = [c for c in children if c.base_name.endswith(".parquet")]
+            if not parquet_children:
+                return None
+            info = parquet_children[0]
+
+        # pyarrow S3FileSystem exposes etag via the metadata dict.
+        if hasattr(info, "metadata") and info.metadata:
+            etag = info.metadata.get("ETag") or info.metadata.get("etag")
+            if etag:
+                return str(etag)
+
+        # Fallback: use mtime + size as a combined fingerprint.
+        if info.size is not None and info.size > 0:
+            mtime = getattr(info, "mtime", None)
+            return f"{info.size}-{mtime}"
+
+        return None
+    except Exception as exc:
+        logger.debug(
+            "_get_cloud_etag: could not fetch ETag for {!r}: {}", static_path, exc
+        )
+        return None
+
+
+def _read_cloud_bbox_sidecar(
+    sidecar: pathlib.Path,
+    etag: str,
+) -> pl.DataFrame | None:
+    """Read the cloud bbox sidecar if it exists and its stored ETag matches *etag*.
+
+    Args:
+        sidecar (pathlib.Path): Local cache path returned by :func:`_cloud_sidecar_path`.
+        etag (str): Current ETag / fingerprint of the remote object.
+
+    Returns:
+        pl.DataFrame | None: Cached index DataFrame, or ``None`` if absent or stale.
+
+    """
+    etag_file = sidecar.with_suffix(".etag")
+    if not sidecar.exists() or not etag_file.exists():
+        return None
+    try:
+        stored_etag = etag_file.read_text(encoding="utf-8").strip()
+        if stored_etag != etag:
+            logger.debug(
+                "Cloud bbox sidecar {} is stale (ETag mismatch), rebuilding.",
+                sidecar,
+            )
+            return None
+        logger.debug("Loading cloud bbox index from sidecar: {}", sidecar)
+        return pl.read_parquet(str(sidecar))
+    except Exception as exc:
+        logger.warning("Failed to read cloud bbox sidecar {}: {}", sidecar, exc)
+        return None
+
+
+def _write_cloud_bbox_sidecar(
+    df: pl.DataFrame,
+    sidecar: pathlib.Path,
+    etag: str,
+) -> None:
+    """Persist ``df`` and ``etag`` to the cloud bbox sidecar, silently ignoring errors.
+
+    Args:
+        df (pl.DataFrame): The bbox index DataFrame to persist.
+        sidecar (pathlib.Path): Destination path for the Parquet sidecar.
+        etag (str): ETag / fingerprint to store alongside the sidecar.
+
+    """
+    try:
+        sidecar.parent.mkdir(parents=True, exist_ok=True)
+        df.write_parquet(str(sidecar))
+        sidecar.with_suffix(".etag").write_text(etag, encoding="utf-8")
+        logger.debug("Wrote cloud bbox index sidecar: {}", sidecar)
+    except Exception as exc:
+        logger.debug("Could not write cloud bbox index sidecar {}: {}", sidecar, exc)
+
+
 def _read_bbox_sidecar(sidecar: pathlib.Path, static_path: str) -> pl.DataFrame | None:
     """Read the sidecar if it exists and is newer than the static data.
 
@@ -212,10 +351,20 @@ def build_bbox_index(
             + "Cannot compute bounds from separate lat/lon columns without bbox hints."
         )
 
-    # --- Sidecar fast path (local paths only) --------------------------------
+    # --- Sidecar fast path ---------------------------------------------------
     # Check for a pre-built index sidecar before doing the expensive WKB loop.
+    # Local paths use mtime-based invalidation; cloud URIs use ETag.
     sidecar: pathlib.Path | None = None
-    if not is_cloud_uri(static_path):
+    cloud_etag: str | None = None
+    if is_cloud_uri(static_path):
+        # One cheap HeadObject call to get the ETag — no data transferred.
+        cloud_etag = _get_cloud_etag(static_path, _so)
+        if cloud_etag is not None:
+            sidecar = _cloud_sidecar_path(static_path)
+            cached = _read_cloud_bbox_sidecar(sidecar, cloud_etag)
+            if cached is not None:
+                return cached
+    else:
         sidecar = _bbox_sidecar_path(static_path)
         if sidecar is not None:
             cached = _read_bbox_sidecar(sidecar, static_path)
@@ -266,9 +415,14 @@ def build_bbox_index(
             pl.Series("maxy", [], dtype=pl.Float64),
         )
 
-    # Persist the sidecar for future runs (local paths only, errors silenced).
+    # Persist the sidecar for future runs (errors silenced in both helpers).
     if sidecar is not None:
-        _write_bbox_sidecar(result, sidecar)
+        if cloud_etag is not None:
+            # Cloud path — store alongside the ETag fingerprint.
+            _write_cloud_bbox_sidecar(result, sidecar, cloud_etag)
+        else:
+            # Local path — plain mtime-checked sidecar.
+            _write_bbox_sidecar(result, sidecar)
 
     return result
 
@@ -310,6 +464,7 @@ def exact_spatial_filter(
     aoi_geometry: "shapely.Geometry",
     relationship: str = "centroid",
     threshold: float = 0.5,
+    storage_options: dict[str, Any] | None = None,
 ) -> pl.DataFrame:
     """Refine a bbox candidate set to rows that match ``aoi_geometry``.
 
@@ -319,7 +474,7 @@ def exact_spatial_filter(
     Args:
         candidates (pl.DataFrame): The DataFrame from :func:`bbox_intersects_geometry` —
             only the key columns are used here; the bbox columns are ignored.
-        static_path (str): Absolute path to the static GeoParquet file.
+        static_path (str): Absolute path or cloud URI to the static GeoParquet file.
         key_cols (list[str]): Column name(s) that form the entity's unique key.
         geometry_col (str): Name of the geometry column in the static file.
         geometry_type (str): One of ``"wkb"`` or ``"wkt"``.
@@ -333,6 +488,10 @@ def exact_spatial_filter(
 
         threshold (float, optional): Minimum intersection-to-entity area ratio used in
             ``"area"`` mode.  Ignored in ``"centroid"`` mode.  Default 0.5.
+        storage_options (dict[str, Any] | None, optional): Cloud credential /
+            configuration options forwarded to ``pl.scan_parquet``.  Required
+            when ``static_path`` is a cloud URI; ``None`` uses ambient
+            credentials.
 
     Returns:
         pl.DataFrame: A ``pl.DataFrame`` containing only the key columns for entities that
@@ -361,8 +520,12 @@ def exact_spatial_filter(
         return candidates.select(key_cols)
 
     # Build a lazy scan and push down an inner join to load only required geometries.
+    _so = storage_options or {}
     full_df = (
-        pl.scan_parquet(parquet_scan_path(static_path))
+        pl.scan_parquet(
+            parquet_scan_path(static_path),
+            storage_options=_so or None,
+        )
         .select(key_cols + [geometry_col])
         .join(candidates.select(key_cols).lazy(), on=key_cols, how="semi")
         .collect()