core-lens 0.1.dev172__tar.gz → 0.1.dev175__tar.gz

{core_lens-0.1.dev172 → core_lens-0.1.dev175}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: core-lens
-Version: 0.1.dev172
+Version: 0.1.dev175
 Summary: Query, analyse, and visualise CoreStack's microwatershed and Earth science data through a clean, composable Python API.
 Project-URL: Homepage, https://github.com/ApoorvaKashyap/core-lens
 Project-URL: Issues, https://github.com/ApoorvaKashyap/core-lens/issues

{core_lens-0.1.dev172 → core_lens-0.1.dev175}/benchmarks/README.md

@@ -59,7 +59,6 @@ uv run python benchmarks/bench_spatial.py
 | `bench_aoi.py` | `AoI.register`, `AoI.__init__` (bbox/geometry/named/seasons), `__getattr__`, `current_season/year`, `registered_entities`, `deregister` |
 | `bench_entity.py` | `_resolve` (local/cloud), `where` (single/composite/list/error), `spatial_filter` (bbox/geometry/area/error), `spatial_join`, `schema_profile` cache, `_index` cache, `geometry_lazy`, `_so_key`, `_entity_name`, `_cached_detect`, `EntityValidationError` |
 | `bench_spatial.py` | `build_bbox_index`, `bbox_intersects_geometry`, `exact_spatial_filter` (centroid/area/all-India), `execute_spatial_join`, `_bbox_sidecar_path`, `_read_bbox_sidecar`, error path |
-| `bench_paths.py` | `is_cloud_uri` (positive/negative), `resolve_fs_and_path` (local/cloud), `path_exists` (exists/absent), `join_uri` (local/cloud) |
 | `bench_view.py` | `where` (single/composite), `spatial_filter` (bbox), `between` (dates/season/CURRENT/year-range/errors), `spatial_join` (spec/double-error), `static`/`annual`/`sub_annual` materialisation |
 | `bench_result.py` | `df`, `lazy`, `gdf` (WKB decode), `materialise`, `derive` (single/chain), `with_geometry` (noop/annual), `aggregate` (no-by/year/month/season/year_month/season_year/errors), `gdf` TypeError guard, `stats`/`plot` accessors |
 | `bench_season.py` | `SeasonConfig.__post_init__` (default/custom), `season_for`, `resolve_time_filter` (date-range/kharif/rabi/year-range), `add_temporal_columns` (first/second noop call), Polars filter execution |

{core_lens-0.1.dev172 → core_lens-0.1.dev175}/benchmarks/bench_entity.py

@@ -38,7 +38,6 @@ from core_lens.base.entity import (
     _so_key,
     _entity_name,
     _cached_detect,
-    _cached_build_index,
     EntityValidationError,
 )
 

{core_lens-0.1.dev172 → core_lens-0.1.dev175}/benchmarks/bench_polars_utils.py

@@ -191,7 +191,7 @@ print(
 # ── 7. cached_read_schema() — cold vs cached ──────────────────────────────────
 _section("7. cached_read_schema()  [cold vs functools.cache warm]")
 from core_lens.utils.polars_utils import cached_read_schema  # noqa: E402
-from core_lens.utils.polars_utils import _cached_schema_internal  # type: ignore[attr-defined]  # noqa: E402
+from core_lens.utils.polars_utils import _cached_schema_internal  # noqa: E402
 
 # Warm the cache first (may already be warm from module import chain).
 cached_read_schema(static_path)

{core_lens-0.1.dev172 → core_lens-0.1.dev175}/benchmarks/bench_result.py

@@ -188,7 +188,8 @@ print(
 
 # ── 10. Result.aggregate(by='season') / by='month' / by='year_month' ─────────
 _section("10. Result.aggregate() sub_annual groupings")
-from core_lens.base.view import Season
+from core_lens.base.view import Season  # noqa: E402
+
 result_fn = aoi_small.mws.between(season=Season.KHARIF, year=(2018, 2022)).sub_annual
 
 data_col = result_fn.columns[-1]  # last column (data column)
@@ -197,7 +198,9 @@ for by_key in ("month", "season", "year_month", "season_year", "year"):
     t0 = time.perf_counter()
     agg_r = result_fn.aggregate(pl.first(data_col), by=by_key)
     t1 = time.perf_counter()
-    print(f"aggregate(by={by_key!r:12}) : {(t1 - t0) * 1000:.2f} ms  shape={agg_r.df().shape}")
+    print(
+        f"aggregate(by={by_key!r:12}) : {(t1 - t0) * 1000:.2f} ms  shape={agg_r.df().shape}"
+    )
 
 
 # ── 11. Result.aggregate() error paths ──────────────────────────────────

{core_lens-0.1.dev172 → core_lens-0.1.dev175}/benchmarks/bench_schema.py

@@ -221,7 +221,7 @@ else:
 
 # ── 9. _find_lon_companion() ─────────────────────────────────────────────────
 _section("9. _find_lon_companion()  [lat→lon heuristic — O(candidates)]")
-import polars as pl  # noqa: E401  (already imported; re-stated for Scalene trace clarity)
+import polars as pl  # noqa: E401, E402  (already imported; re-stated for Scalene trace clarity)
 
 # Build a synthetic schema with lat + lon columns to exercise the positive branch.
 synthetic_schema = pl.Schema(
@@ -265,7 +265,9 @@ print("_require_cols (ok)    : no error (expect no error)")
 
 # ── 11. SchemaProfile.from_file() — JSON round-trip ──────────────────────────
 _section("11. SchemaProfile.from_file()  [JSON round-trip]")
-import json, tempfile, pathlib
+import json  # noqa: E402
+import tempfile  # noqa: E402
+import pathlib  # noqa: E402
 
 profile2 = entity2.schema_profile
 profile_dict = profile2.model_dump()

{core_lens-0.1.dev172 → core_lens-0.1.dev175}/benchmarks/bench_spatial.py

@@ -193,7 +193,6 @@ print(f"spatial_filter (geom) : {(t1 - t0) * 1000:.2f} ms")
 
 # ── 9. execute_spatial_join() — cross-entity agg join ────────────────────────
 _section("9. execute_spatial_join()  [cross-entity agg on small bbox]")
-import polars as pl  # already imported at module top — re-imported for clarity
 from core_lens.utils.spatial import execute_spatial_join  # noqa: E402
 
 # Use the small-AoI static data as primary and the same entity as secondary.

{core_lens-0.1.dev172 → core_lens-0.1.dev175}/benchmarks/bench_view.py

@@ -2,7 +2,7 @@
 =======================================================
 Targets:
   - View.where()                attribute filter (Polars semi-join on static file)
-  - View.spatial_filter()       STRtree filter narrowing existing keys
+  - View.spatial_filter()       vectorized centroid test (or STRtree for area)
   - View.between()              time-filter dict construction (no I/O)
   - View.between(season=CURRENT)  current-season resolution
   - View.between(year=tuple)    year-range season filter
@@ -147,7 +147,9 @@ print(f"Shape            : {result_annual.df().shape}")
 # ── 11. sub_annual materialisation ───────────────────────────────────────────
 _section("11. aoi_small.mws.sub_annual  [sub_annual materialisation]")
 t0 = time.perf_counter()
-result_fn = aoi_small.mws.between(season=Season.KHARIF, year=2022).sub_annual.materialise()
+result_fn = aoi_small.mws.between(
+    season=Season.KHARIF, year=2022
+).sub_annual.materialise()
 t1 = time.perf_counter()
 print(f"sub_annual      : {(t1 - t0) * 1000:.2f} ms")
 print(f"Shape            : {result_fn.df().shape}")
@@ -179,7 +181,6 @@ print(
 
 # ── 14. View.between() error paths ───────────────────────────────────────────
 _section("14. View.between() error paths  [guard branches — no I/O]")
-import traceback
 
 errors_caught = 0
 
@@ -218,7 +219,7 @@ print(f"Error guards raised  : {errors_caught}/5 (expect 5)")
 
 # ── 15. View.spatial_join() — deferred cross-entity join spec ─────────────────
 _section("15. View.spatial_join()  [deferred spec — no I/O]")
-from core_lens.entities.mws import MWSEntity as _MWS2
+from core_lens.entities.mws import MWSEntity as _MWS2  # noqa: E402
 
 other_entity = _MWS2(data_root=DATA_ROOT)
 REPS_SJ = 10_000
@@ -234,7 +235,9 @@ print(
 # 15b. Double spatial_join raises ValueError
 duplicate_raised = False
 try:
-    view_all.spatial_join(other_entity, {"x": "sum"}).spatial_join(other_entity, {"y": "mean"})
+    view_all.spatial_join(other_entity, {"x": "sum"}).spatial_join(
+        other_entity, {"y": "mean"}
+    )
 except ValueError:
     duplicate_raised = True
 print(f"Double join guard    : {duplicate_raised} (expect True)")

{core_lens-0.1.dev172 → core_lens-0.1.dev175}/benchmarks/run_all.sh

@@ -19,7 +19,7 @@ BENCHMARKS=(
     "bench_aoi"
     "bench_entity"
     "bench_spatial"
-    "bench_paths"
+    # "bench_paths"
     "bench_view"
     "bench_result"
     "bench_season"

{core_lens-0.1.dev172 → core_lens-0.1.dev175}/src/core_lens/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.1.dev172'
-__version_tuple__ = version_tuple = (0, 1, 'dev172')
+__version__ = version = '0.1.dev175'
+__version_tuple__ = version_tuple = (0, 1, 'dev175')
 
 __commit_id__ = commit_id = None

{core_lens-0.1.dev172 → core_lens-0.1.dev175}/src/core_lens/base/namespaces/plot.py

@@ -731,31 +731,45 @@ class PlotNamespace:
             )
 
             entities = df.get_column(key_col).unique().to_list()
-            for i, entity in enumerate(entities):
-                row = df.filter(pl.col(key_col) == entity)
-
-                # Replace None with NaN for numeric columns to prevent Bokeh JS rendering crashes
-                source_data = {}
-                for c in hover_cols:
-                    if c in row.columns:
-                        lst = row.get_column(c).to_list()
-                        if row.schema[c].is_numeric():
-                            lst = [float("nan") if v is None else v for v in lst]
-                        source_data[c] = lst
-
-                source = ColumnDataSource(source_data)  # type: ignore[arg-type]
+
+            if len(entities) > 20:
+                # Fast path: single dataset, single colour, no legend
+                pdf = df.drop_nulls(subset=[x, y_col]).select(hover_cols).to_pandas()
+                source = ColumnDataSource(pdf)
                 fig.scatter(
                     x=x,
                     y=y_col,
                     source=source,
-                    color=_color_for(i),
-                    legend_label=str(entity),
+                    color=_color_for(0),
                     size=8,
                     alpha=0.7,
                 )
-            if fig.legend:
-                fig.legend.click_policy = "hide"
-                fig.add_layout(fig.legend[0], "right")
+            else:
+                for i, entity in enumerate(entities):
+                    row = df.filter(pl.col(key_col) == entity)
+
+                    # Replace None with NaN for numeric columns to prevent Bokeh JS rendering crashes
+                    source_data = {}
+                    for c in hover_cols:
+                        if c in row.columns:
+                            lst = row.get_column(c).to_list()
+                            if row.schema[c].is_numeric():
+                                lst = [float("nan") if v is None else v for v in lst]
+                            source_data[c] = lst
+
+                    source = ColumnDataSource(source_data)  # type: ignore[arg-type]
+                    fig.scatter(
+                        x=x,
+                        y=y_col,
+                        source=source,
+                        color=_color_for(i),
+                        legend_label=str(entity),
+                        size=8,
+                        alpha=0.7,
+                    )
+                if fig.legend:
+                    fig.legend.click_policy = "hide"
+                    fig.add_layout(fig.legend[0], "right")
             _apply_theme(fig, self.result, "Scatter Distribution")
             return fig
 

{core_lens-0.1.dev172 → core_lens-0.1.dev175}/src/core_lens/base/result.py

@@ -279,12 +279,23 @@ class Result:
             )
 
         if by is None:
-            new_data = self.data.group_by(self.key_cols).agg(*exprs)
+            group_cols = self.key_cols
         else:
             # Temporal grouping columns are expected to already exist on the
             # frame (added by the materialisation layer from the time column).
             group_cols = self.key_cols + [by]
-            new_data = self.data.group_by(group_cols).agg(*exprs)
+
+        # Polars group_by throws DuplicateError if an aggregation expression
+        # outputs a column name that is already present in the grouping keys.
+        # Filter them out to prevent crashes when users iterate over all columns.
+        safe_exprs = [
+            expr
+            for expr in exprs
+            if getattr(expr, "meta", None) is None
+            or expr.meta.output_name() not in group_cols
+        ]
+
+        new_data = self.data.group_by(group_cols).agg(*safe_exprs)
 
         return self._replace(data=new_data)
 

core_lens-0.1.dev172/benchmarks/bench_paths.py

@@ -1,217 +0,0 @@
-"""bench_paths.py — Benchmark for core_lens.utils.paths
-======================================================
-Targets:
-  - is_cloud_uri()              URI scheme prefix check (hot path helper)
-  - resolve_fs_and_path()       URI → (FileSystem, path) resolution
-  - path_exists()               local filesystem existence check
-  - join_uri()                  URI-agnostic path join (local + cloud)
-
-Scalene focuses on:
-  - string prefix scanning in is_cloud_uri (Python vs C in str.startswith)
-  - pyarrow.fs.FileSystem.from_uri overhead for cloud URIs
-  - pathlib.Path normalisation cost in resolve_fs_and_path / join_uri
-  - pyarrow.fs.get_file_info cost for path_exists on local paths
-
-Usage:
-    uv run scalene run --html --outfile benchmarks/profiles/bench_paths.html benchmarks/bench_paths.py
-    uv run python benchmarks/bench_paths.py
-"""
-
-from __future__ import annotations
-
-import pathlib
-import time
-
-from core_lens.utils.paths import (
-    is_cloud_uri,
-    join_uri,
-    path_exists,
-    resolve_fs_and_path,
-)
-
-# ── Config ────────────────────────────────────────────────────────────────────
-DATA_ROOT = "data/"
-LOCAL_PATH = str(pathlib.Path(DATA_ROOT).resolve())
-
-CLOUD_URIS = [
-    "s3://geo-benchmark-data/converted/mws/static",
-]
-LOCAL_URIS = [
-    LOCAL_PATH,
-    "/tmp/nonexistent_path",
-    "relative/path",
-]
-
-
-def _section(title: str) -> None:
-    print(f"\n{'─' * 60}")
-    print(f"  {title}")
-    print("─" * 60)
-
-
-# ── 1. is_cloud_uri() — positive path (cloud URIs) ────────────────────────────
-_section("1. is_cloud_uri()  [cloud URIs — positive match]")
-REPS = 1_000_000
-
-t0 = time.perf_counter()
-for _ in range(REPS):
-    for uri in CLOUD_URIS:
-        is_cloud_uri(uri)
-t1 = time.perf_counter()
-per_call = (t1 - t0) / (REPS * len(CLOUD_URIS))
-print(
-    f"is_cloud_uri (positive) ×{REPS * len(CLOUD_URIS)}: "
-    f"{(t1 - t0) * 1000:.2f} ms total  ({per_call * 1e9:.1f} ns/call)"
-)
-for uri in CLOUD_URIS:
-    print(f"  {uri[:40]!r:42} → {is_cloud_uri(uri)}")
-
-
-# ── 2. is_cloud_uri() — negative path (local paths) ──────────────────────────
-_section("2. is_cloud_uri()  [local paths — negative match]")
-t0 = time.perf_counter()
-for _ in range(REPS):
-    for uri in LOCAL_URIS:
-        is_cloud_uri(uri)
-t1 = time.perf_counter()
-per_call = (t1 - t0) / (REPS * len(LOCAL_URIS))
-print(
-    f"is_cloud_uri (negative) ×{REPS * len(LOCAL_URIS)}: "
-    f"{(t1 - t0) * 1000:.2f} ms total  ({per_call * 1e9:.1f} ns/call)"
-)
-for uri in LOCAL_URIS:
-    print(f"  {uri!r:42} → {is_cloud_uri(uri)}")
-
-
-# ── 3. resolve_fs_and_path() — local path ─────────────────────────────────────
-_section("3. resolve_fs_and_path()  [local path — pathlib normalise]")
-REPS_FS = 10_000
-t0 = time.perf_counter()
-for _ in range(REPS_FS):
-    resolve_fs_and_path(LOCAL_PATH)
-t1 = time.perf_counter()
-print(
-    f"resolve_fs (local) ×{REPS_FS}: {(t1 - t0) * 1000:.2f} ms total  "
-    f"({(t1 - t0) / REPS_FS * 1e6:.2f} µs/call)"
-)
-fs, path = resolve_fs_and_path(LOCAL_PATH)
-print(f"  fs type : {type(fs).__name__}")
-print(f"  path    : {path!r}")
-
-
-# ── 4. resolve_fs_and_path() — cloud URI ──────────────────────────────────────
-_section("4. resolve_fs_and_path()  [cloud URI — lru_cache cold vs warm]")
-from core_lens.utils.paths import _resolve_fs_and_path_cached  # noqa: E402
-
-cloud_uri = CLOUD_URIS[0]  # s3:// URI
-
-# Cold: flush cache, measure the first call (pafs.FileSystem.from_uri cost).
-_resolve_fs_and_path_cached.cache_clear()
-t0 = time.perf_counter()
-resolve_fs_and_path(cloud_uri)
-t1 = time.perf_counter()
-t_cold_us = (t1 - t0) * 1e6
-print(f"resolve_fs cloud (cold)  : {t_cold_us:.2f} µs  ← pafs.FileSystem.from_uri")
-
-# Warm: same URI, now served from lru_cache.
-REPS_CFS = 100_000
-t0 = time.perf_counter()
-for _ in range(REPS_CFS):
-    resolve_fs_and_path(cloud_uri)
-t1 = time.perf_counter()
-t_warm_us = (t1 - t0) / REPS_CFS * 1e6
-print(
-    f"resolve_fs cloud (warm) ×{REPS_CFS}: {(t1 - t0) * 1000:.2f} ms total  "
-    f"({t_warm_us:.3f} µs/call)"
-)
-print(f"Speedup (cold→warm)      : {t_cold_us / t_warm_us:.0f}×")
-print(f"Cache info               : {_resolve_fs_and_path_cached.cache_info()}")
-
-fs_c, path_c = resolve_fs_and_path(cloud_uri)
-print(f"  fs type : {type(fs_c).__name__}")
-print(f"  path    : {path_c!r}")
-
-# All four cloud URI schemes — each is a separate cache entry.
-_section("4b. All cloud URI schemes  [each scheme = distinct cache entry]")
-for uri in CLOUD_URIS:
-    _resolve_fs_and_path_cached.cache_clear()
-    t0 = time.perf_counter()
-    resolve_fs_and_path(uri)
-    t1 = time.perf_counter()
-    print(f"  cold {uri[:45]!r:47} : {(t1 - t0) * 1e6:.1f} µs")
-print(f"Cache info (after 4 URIs): {_resolve_fs_and_path_cached.cache_info()}")
-
-
-# ── 5. path_exists() — local path (exists) ────────────────────────────────────
-_section("5. path_exists()  [local — directory exists]")
-REPS_PE = 1_000
-t0 = time.perf_counter()
-for _ in range(REPS_PE):
-    path_exists(LOCAL_PATH)
-t1 = time.perf_counter()
-print(
-    f"path_exists (exists) ×{REPS_PE}: {(t1 - t0) * 1000:.2f} ms total  "
-    f"({(t1 - t0) / REPS_PE * 1e6:.2f} µs/call)"
-)
-print(f"  exists: {path_exists(LOCAL_PATH)!r}")
-
-
-# ── 6. path_exists() — local path (absent) ────────────────────────────────────
-_section("6. path_exists()  [local — path does not exist]")
-t0 = time.perf_counter()
-for _ in range(REPS_PE):
-    path_exists("/tmp/core_lens_nonexistent_xyz_123")
-t1 = time.perf_counter()
-print(
-    f"path_exists (absent) ×{REPS_PE}: {(t1 - t0) * 1000:.2f} ms total  "
-    f"({(t1 - t0) / REPS_PE * 1e6:.2f} µs/call)"
-)
-print(f"  exists: {path_exists('/tmp/core_lens_nonexistent_xyz_123')!r}")
-
-
-# ── 7. join_uri() — local root ────────────────────────────────────────────────
-_section("7. join_uri()  [local root — pathlib join]")
-REPS_JU = 1_000_000
-t0 = time.perf_counter()
-for _ in range(REPS_JU):
-    join_uri(LOCAL_PATH, "mws/static")
-t1 = time.perf_counter()
-print(
-    f"join_uri (local) ×{REPS_JU}: {(t1 - t0) * 1000:.2f} ms total  "
-    f"({(t1 - t0) / REPS_JU * 1e9:.1f} ns/call)"
-)
-print(f"  result: {join_uri(LOCAL_PATH, 'mws/static')!r}")
-
-
-# ── 8. join_uri() — cloud root ────────────────────────────────────────────────
-_section("8. join_uri()  [cloud root — string concat]")
-cloud_root = "s3://my-bucket/data"
-t0 = time.perf_counter()
-for _ in range(REPS_JU):
-    join_uri(cloud_root, "mws/static")
-t1 = time.perf_counter()
-print(
-    f"join_uri (cloud) ×{REPS_JU}: {(t1 - t0) * 1000:.2f} ms total  "
-    f"({(t1 - t0) / REPS_JU * 1e9:.1f} ns/call)"
-)
-print(f"  result: {join_uri(cloud_root, 'mws/static')!r}")
-
-# Head-to-head: local vs cloud join.
-t_local = time.perf_counter()
-for _ in range(REPS_JU):
-    join_uri(LOCAL_PATH, "mws/static")
-t_local = time.perf_counter() - t_local
-
-t_cloud = time.perf_counter()
-for _ in range(REPS_JU):
-    join_uri(cloud_root, "mws/static")
-t_cloud = time.perf_counter() - t_cloud
-
-print(
-    f"\nHead-to-head ×{REPS_JU}: local={t_local * 1000:.2f} ms  "
-    f"cloud={t_cloud * 1000:.2f} ms  "
-    f"(cloud is {t_cloud / t_local:.2f}× {'faster' if t_cloud < t_local else 'slower'})"
-)
-
-
-print("\n✓ bench_paths.py complete")

core_lens-0.1.dev172/parse_scalene.py

@@ -1,50 +0,0 @@
-import json
-import glob
-import os
-
-profiles = glob.glob('/home/akashyap54/workstation/Projects/benchmarks/profiles/*_profile.json')
-
-for profile in profiles:
-    name = os.path.basename(profile).replace('_profile.json', '')
-    try:
-        d = json.load(open(profile))
-    except Exception as e:
-        print(f"Error loading {name}: {e}")
-        continue
-    
-    print(f"\n# {name.upper()}")
-    print("-" * 40)
-    
-    line_stats = []
-    
-    for filename, f_data in d.get('files', {}).items():
-        short_filename = os.path.basename(filename)
-        for line_data in f_data.get('lines', []):
-            cpu_percent = line_data.get('n_cpu_percent_c', 0) + line_data.get('n_cpu_percent_python', 0)
-            malloc_mb = line_data.get('n_malloc_mb', 0)
-            peak_mb = line_data.get('n_peak_mb', 0)
-            line_str = line_data.get('line', '').strip()
-            lineno = line_data.get('lineno', 0)
-            
-            if cpu_percent > 0 or malloc_mb > 0 or peak_mb > 0:
-                line_stats.append({
-                    'file': short_filename,
-                    'lineno': lineno,
-                    'line': line_str[:60] + ('...' if len(line_str) > 60 else ''),
-                    'cpu': cpu_percent,
-                    'malloc_mb': malloc_mb,
-                    'peak_mb': peak_mb
-                })
-    
-    # Top 5 by CPU
-    print(f"Top 5 lines by CPU %:")
-    top_cpu = sorted(line_stats, key=lambda x: x['cpu'], reverse=True)[:5]
-    for x in top_cpu:
-        print(f"  {x['file']}:{x['lineno']} | CPU: {x['cpu']:.1f}% | Line: {x['line']}")
-        
-    # Top 5 by Malloc
-    print(f"\nTop 5 lines by Memory Allocation (MB):")
-    top_mem = sorted(line_stats, key=lambda x: x['malloc_mb'], reverse=True)[:5]
-    for x in top_mem:
-        if x['malloc_mb'] > 0:
-            print(f"  {x['file']}:{x['lineno']} | Malloc: {x['malloc_mb']:.1f} MB | Line: {x['line']}")