core-lens 0.1.dev135__tar.gz → 0.1.dev136__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/PKG-INFO +1 -1
  2. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/_version.py +2 -2
  3. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/aoi.py +221 -125
  4. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/utils/spatial.py +119 -19
  5. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/.github/ISSUE_TEMPLATE/blank-proposal.yaml +0 -0
  6. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/.github/ISSUE_TEMPLATE/bug-report.yaml +0 -0
  7. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/.github/ISSUE_TEMPLATE/feature-request.yaml +0 -0
  8. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/.github/pull_request_template.md +0 -0
  9. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/.github/workflows/ci.yml +0 -0
  10. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/.github/workflows/gh-pages.yml +0 -0
  11. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/.github/workflows/pre-release.yml +0 -0
  12. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/.github/workflows/release.yml +0 -0
  13. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/.gitignore +0 -0
  14. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/.gitmessage +0 -0
  15. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/.pre-commit-config.yaml +0 -0
  16. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/.python-version +0 -0
  17. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/CONTRIBUTING.md +0 -0
  18. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/LICENSE +0 -0
  19. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/README.md +0 -0
  20. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/SKILLS.md +0 -0
  21. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/benchmarks/README.md +0 -0
  22. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/benchmarks/bench_aoi.py +0 -0
  23. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/benchmarks/bench_export.py +0 -0
  24. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/benchmarks/bench_polars_utils.py +0 -0
  25. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/benchmarks/bench_result.py +0 -0
  26. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/benchmarks/bench_schema.py +0 -0
  27. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/benchmarks/bench_season.py +0 -0
  28. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/benchmarks/bench_spatial.py +0 -0
  29. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/benchmarks/bench_view.py +0 -0
  30. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/benchmarks/run_all.sh +0 -0
  31. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/docs/Makefile +0 -0
  32. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/docs/make.bat +0 -0
  33. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/docs/source/concepts.md +0 -0
  34. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/docs/source/conf.py +0 -0
  35. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/docs/source/export.md +0 -0
  36. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/docs/source/index.rst +0 -0
  37. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/docs/source/intro.md +0 -0
  38. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/docs/source/logging.md +0 -0
  39. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/docs/source/plots.md +0 -0
  40. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/docs/source/plugins.md +0 -0
  41. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/docs/source/queries.md +0 -0
  42. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/docs/source/quickstart.md +0 -0
  43. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/docs/source/stats.md +0 -0
  44. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/examples/demo_mws.py +0 -0
  45. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/examples/demo_tehsil.py +0 -0
  46. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/hooks/mypy.sh +0 -0
  47. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/hooks/no-parquet-outside-fixtures.sh +0 -0
  48. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/hooks/pytest.sh +0 -0
  49. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/pyproject.toml +0 -0
  50. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/__init__.py +0 -0
  51. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/__main__.py +0 -0
  52. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/base/__init__.py +0 -0
  53. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/base/entity.py +0 -0
  54. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/base/namespaces/__init__.py +0 -0
  55. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/base/namespaces/plot.py +0 -0
  56. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/base/namespaces/stats.py +0 -0
  57. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/base/result.py +0 -0
  58. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/base/view.py +0 -0
  59. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/entities/__init__.py +0 -0
  60. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/entities/mws.py +0 -0
  61. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/entities/tehsil.py +0 -0
  62. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/entities/waterbody.py +0 -0
  63. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/export/__init__.py +0 -0
  64. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/export/formats.py +0 -0
  65. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/py.typed +0 -0
  66. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/schema/__init__.py +0 -0
  67. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/schema/detection.py +0 -0
  68. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/schema/profile.py +0 -0
  69. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/utils/__init__.py +0 -0
  70. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/utils/paths.py +0 -0
  71. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/utils/polars_utils.py +0 -0
  72. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/src/core_lens/utils/season.py +0 -0
  73. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/tests/fixtures/generate_fixtures.py +0 -0
  74. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/tests/unit/conftest.py +0 -0
  75. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/tests/unit/test_aoi.py +0 -0
  76. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/tests/unit/test_entities.py +0 -0
  77. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/tests/unit/test_entity.py +0 -0
  78. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/tests/unit/test_export.py +0 -0
  79. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/tests/unit/test_main.py +0 -0
  80. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/tests/unit/test_plot.py +0 -0
  81. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/tests/unit/test_polars_utils.py +0 -0
  82. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/tests/unit/test_profile.py +0 -0
  83. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/tests/unit/test_result.py +0 -0
  84. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/tests/unit/test_schema_detection.py +0 -0
  85. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/tests/unit/test_schema_profile.py +0 -0
  86. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/tests/unit/test_season.py +0 -0
  87. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/tests/unit/test_season_config.py +0 -0
  88. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/tests/unit/test_spatial.py +0 -0
  89. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/tests/unit/test_stats.py +0 -0
  90. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/tests/unit/test_view.py +0 -0
  91. {core_lens-0.1.dev135 → core_lens-0.1.dev136}/uv.lock +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: core-lens
3
- Version: 0.1.dev135
3
+ Version: 0.1.dev136
4
4
  Summary: Query, analyse, and visualise CoreStack's microwatershed and Earth science data through a clean, composable Python API.
5
5
  Project-URL: Homepage, https://github.com/ApoorvaKashyap/core-lens
6
6
  Project-URL: Issues, https://github.com/ApoorvaKashyap/core-lens/issues
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '0.1.dev135'
22
- __version_tuple__ = version_tuple = (0, 1, 'dev135')
21
+ __version__ = version = '0.1.dev136'
22
+ __version_tuple__ = version_tuple = (0, 1, 'dev136')
23
23
 
24
24
  __commit_id__ = commit_id = None
@@ -2,6 +2,7 @@
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
+ import functools
5
6
  import os
6
7
  import pathlib
7
8
  from dataclasses import dataclass
@@ -12,13 +13,15 @@ if TYPE_CHECKING:
12
13
  from core_lens.base.result import Result
13
14
 
14
15
  import polars as pl
16
+ import shapely
17
+ import shapely.geometry as sgeom
18
+ import shapely.ops as sops
15
19
  from loguru import logger
16
20
 
17
21
  from core_lens.base.entity import BaseEntity, EntityValidationError
18
22
  from core_lens.utils.paths import is_cloud_uri
19
23
 
20
24
  if TYPE_CHECKING:
21
- import shapely
22
25
  from core_lens.base.view import View
23
26
 
24
27
 
@@ -125,6 +128,121 @@ def _default_season_config() -> SeasonConfig:
125
128
  _REGISTRY: dict[str, type[BaseEntity]] = {}
126
129
 
127
130
 
131
+ # ---------------------------------------------------------------------------
132
+ # Process-level boundary cache — shared across all AoI instances.
133
+ #
134
+ # Keyed on (data_root, storage_options_key, entity_kwargs_key).
135
+ # Returns a (geometry_wkb, entity_name, key_rows_tuple) triple so the
136
+ # expensive scan + WKB decode + unary_union is only paid once per process
137
+ # per unique boundary specification.
138
+ #
139
+ # Cache diagnostics: _cached_resolve_boundary.cache_info()
140
+ # Cache reset (tests): _cached_resolve_boundary.cache_clear()
141
+ # ---------------------------------------------------------------------------
142
+
143
+
144
+ @functools.cache
145
+ def _cached_resolve_boundary(
146
+ data_root: str,
147
+ storage_options_key: tuple[tuple[str, Any], ...],
148
+ entity_kwargs_key: tuple[tuple[str, Any], ...],
149
+ ) -> tuple[bytes, str, tuple[tuple[Any, ...], ...]]:
150
+ """Cached boundary resolution: return ``(geometry_wkb, entity_name, key_rows)``.
151
+
152
+ All arguments must be hashable. ``entity_kwargs`` must be converted to a
153
+ sorted tuple of ``(key, value)`` pairs before calling this function.
154
+
155
+ The returned triple contains:
156
+
157
+ * **geometry_wkb** — WKB bytes of the unioned boundary geometry.
158
+ * **entity_name** — name of the entity that defined the boundary
159
+ (e.g. ``"mws"`` or ``"tehsil"``).
160
+ * **key_rows** — tuple of key-value tuples for the matched rows
161
+ (e.g. ``(("mws_id", "13_551"),)`` for a single-row result).
162
+
163
+ Args:
164
+ data_root (str): Resolved data root path or cloud URI.
165
+ storage_options_key (tuple): Sorted ``storage_options`` items (hashable).
166
+ entity_kwargs_key (tuple): Sorted ``entity_kwargs`` items (hashable).
167
+
168
+ Returns:
169
+ tuple: ``(geometry_wkb, entity_name, key_rows)``.
170
+ """
171
+ from core_lens.base.entity import _entity_name as _ename
172
+
173
+ entity_kwargs: dict[str, Any] = dict(entity_kwargs_key)
174
+ storage_options: dict[str, Any] = dict(storage_options_key)
175
+
176
+ # Instantiate a throw-away AoI-like context to run the resolution logic.
177
+ # We cannot use a live AoI instance here (would be circular), so we
178
+ # inline the logic from _resolve_named_boundary.
179
+ candidate: BaseEntity | None = None
180
+
181
+ # First pass: find entity whose schema columns match the kwargs.
182
+ for ename in _REGISTRY:
183
+ entity_cls = _REGISTRY[ename]
184
+ entity = entity_cls(
185
+ data_root=data_root,
186
+ storage_options=storage_options or None,
187
+ )
188
+ schema = entity.schema_profile # uses _cached_detect — I/O only once
189
+ if any(
190
+ k in schema.key_cols or k in schema.extra_static_cols for k in entity_kwargs
191
+ ):
192
+ candidate = entity
193
+ break
194
+
195
+ # Fallback: entity name matches a kwarg key.
196
+ if candidate is None:
197
+ for ename in _REGISTRY:
198
+ if ename in entity_kwargs:
199
+ entity_cls = _REGISTRY[ename]
200
+ candidate = entity_cls(
201
+ data_root=data_root,
202
+ storage_options=storage_options or None,
203
+ )
204
+ break
205
+
206
+ if candidate is None:
207
+ raise EntityValidationError(
208
+ f"No registered entity can satisfy the filters {entity_kwargs}. "
209
+ f"Registered entities: {sorted(_REGISTRY)}."
210
+ )
211
+
212
+ schema = candidate.schema_profile
213
+ geom_col = schema.geometry_col
214
+ static_path = candidate._resolve(candidate.static_path)
215
+
216
+ lf = pl.scan_parquet(static_path, storage_options=storage_options or None)
217
+ filter_expr = pl.lit(True)
218
+ for col, val in entity_kwargs.items():
219
+ if col in schema.key_cols or col in schema.extra_static_cols:
220
+ if isinstance(val, list):
221
+ lf = lf.filter(pl.col(col).is_in(val))
222
+ else:
223
+ filter_expr = filter_expr & (pl.col(col) == val)
224
+
225
+ df = lf.filter(filter_expr).select(candidate.key_cols + [geom_col]).collect()
226
+
227
+ if df.is_empty():
228
+ raise ValueError(
229
+ f"No rows matched the filters {entity_kwargs} in {candidate.static_path!r}."
230
+ )
231
+
232
+ geoms = shapely.from_wkb(df[geom_col].to_numpy())
233
+ unified = sops.unary_union(geoms) if len(geoms) > 1 else geoms[0]
234
+ geom_wkb: bytes = shapely.to_wkb(unified)
235
+
236
+ entity_name = _ename(type(candidate))
237
+ # Serialise key DataFrame as a tuple-of-tuples for the cache return value.
238
+ key_rows = tuple(
239
+ tuple(row[c] for c in candidate.key_cols)
240
+ for row in df.select(candidate.key_cols).to_dicts()
241
+ )
242
+
243
+ return geom_wkb, entity_name, key_rows
244
+
245
+
128
246
  class AoI:
129
247
  """Area of Interest — the primary entry point for querying geospatial data.
130
248
 
@@ -365,7 +483,9 @@ class AoI:
365
483
  data_root=self.data_root,
366
484
  storage_options=self._storage_options or None,
367
485
  )
368
- _validate_entity(entity, name)
486
+ # Only validate path existence here — schema detection (parquet
487
+ # footer reads) is deferred to first data access via schema_profile.
488
+ _validate_entity_paths(entity, name)
369
489
  self._entity_instances[name] = entity
370
490
  return self._entity_instances[name]
371
491
 
@@ -427,9 +547,11 @@ class AoI:
427
547
  ) -> "shapely.Geometry":
428
548
  """Resolve a set of named attribute filters to a Shapely geometry.
429
549
 
430
- The entity whose key column matches one of the kwargs is queried.
431
- Multiple kwargs act as AND-filters (e.g. tehsil + district narrows to
432
- the unique matching row).
550
+ Delegates to the process-level :func:`_cached_resolve_boundary` cache
551
+ so that the expensive Parquet scan + WKB decode + ``unary_union`` is
552
+ only paid once per unique ``(data_root, entity_kwargs)`` combination
553
+ within a Python process. Subsequent ``AoI()`` calls with the same
554
+ boundary specification reconstruct the geometry from cached WKB bytes.
433
555
 
434
556
  Args:
435
557
  entity_kwargs (dict[str, str | list[str]]): Column–value pairs used to identify the boundary.
@@ -441,85 +563,47 @@ class AoI:
441
563
  :class:`~core_lens.base.EntityValidationError`: If no registered entity can satisfy the filters.
442
564
  ValueError: If the filters match zero rows.
443
565
  """
444
- import shapely.ops as sops
445
-
446
566
  logger.debug("Resolving named boundary using kwargs: {}", entity_kwargs)
447
567
 
448
- # Find the registered entity whose key_col or known attribute column
449
- # matches one of the filter keys. Entities are lazily instantiated
450
- # via _get_entity() only those inspected during the search are built.
451
- candidate: BaseEntity | None = None
452
- for name in _REGISTRY:
453
- entity = self._get_entity(name)
454
- schema = entity.schema_profile
455
- if any(
456
- k in schema.key_cols or k in schema.extra_static_cols
457
- for k in entity_kwargs
458
- ):
459
- candidate = entity
460
- break
461
-
462
- # Fall back: look for an entity whose name matches a kwarg key
463
- # (e.g. tehsil="Pangi" → TehsilEntity if registered as "tehsil").
464
- if candidate is None:
465
- logger.debug(
466
- "No direct column match found for kwargs, attempting entity name fallback"
467
- )
468
- for name in _REGISTRY:
469
- if name in entity_kwargs:
470
- candidate = self._get_entity(name)
471
- break
472
-
473
- if candidate is None:
474
- logger.error(
475
- "Boundary resolution failed: no registered entity matched filters {}",
476
- entity_kwargs,
477
- )
478
- raise EntityValidationError(
479
- f"No registered entity can satisfy the filters {entity_kwargs}. "
480
- f"Registered entities: {sorted(_REGISTRY)}."
481
- )
482
-
483
- schema = candidate.schema_profile
484
- geom_col = schema.geometry_col
485
-
486
- # Build a lazy frame to push filters down into the Parquet reader.
487
- # This avoids loading the entire geometry column into memory.
488
- lf = pl.scan_parquet(candidate._resolve(candidate.static_path))
489
-
490
- filter_expr = pl.lit(True)
491
- for col, val in entity_kwargs.items():
492
- if (
493
- col in candidate.schema_profile.key_cols
494
- or col in candidate.schema_profile.extra_static_cols
495
- ):
496
- if isinstance(val, list):
497
- lf = lf.filter(pl.col(col) == val)
498
- else:
499
- filter_expr = filter_expr & (pl.col(col) == val)
500
-
501
- df = lf.filter(filter_expr).select(candidate.key_cols + [geom_col]).collect()
502
-
503
- # Cache the boundary-defining entity's name and its exact key rows so
504
- # that __getattr__ can short-circuit the spatial_filter for this one
505
- # entity — the keys are already known, no need to rebuild the bbox
506
- # index or re-run a geometric predicate.
507
- self._boundary_entity_name = _entity_name(type(candidate))
508
- self._boundary_keys = df.select(candidate.key_cols)
568
+ from core_lens.base.entity import _so_key
569
+
570
+ # Build hashable cache key from entity_kwargs.
571
+ # list values are converted to tuples so they are hashable.
572
+ kwargs_hashable: dict[str, Any] = {
573
+ k: tuple(v) if isinstance(v, list) else v for k, v in entity_kwargs.items()
574
+ }
575
+ entity_kwargs_key = tuple(sorted(kwargs_hashable.items()))
576
+ data_root_str = str(self.data_root)
577
+ so_key = _so_key(self._storage_options)
578
+
579
+ # Delegate to the process-level cache. On a cache miss this runs the
580
+ # full scan + WKB decode + unary_union; on a hit it returns instantly.
581
+ geom_wkb, boundary_entity_name, key_rows = _cached_resolve_boundary(
582
+ data_root_str, so_key, entity_kwargs_key
583
+ )
509
584
 
510
- if df.is_empty():
511
- logger.error(
512
- "Boundary resolution failed: no rows matched filters {}", entity_kwargs
585
+ # Reconstruct the boundary geometry from cached WKB bytes (fast).
586
+ geometry: shapely.Geometry = shapely.from_wkb(geom_wkb)
587
+
588
+ # Reconstruct _boundary_keys DataFrame from cached key_rows.
589
+ # We need the key column names — get them from the boundary entity.
590
+ boundary_entity_cls = _REGISTRY.get(boundary_entity_name)
591
+ if boundary_entity_cls is not None:
592
+ key_cols = boundary_entity_cls().key_cols
593
+ self._boundary_entity_name = boundary_entity_name
594
+ self._boundary_keys = pl.DataFrame(
595
+ {col: [row[i] for row in key_rows] for i, col in enumerate(key_cols)}
513
596
  )
514
- raise ValueError(
515
- f"No rows matched the filters {entity_kwargs} "
516
- f"in {candidate.static_path!r}."
597
+ else:
598
+ # Registry changed since cache was populated — defensive fallback.
599
+ logger.warning(
600
+ "Boundary cache returned entity name '{}' not found in current registry.",
601
+ boundary_entity_name,
517
602
  )
603
+ self._boundary_entity_name = None
604
+ self._boundary_keys = None
518
605
 
519
- import shapely
520
-
521
- geoms = shapely.from_wkb(df[geom_col].to_numpy())
522
- return sops.unary_union(geoms) if len(geoms) > 1 else geoms[0]
606
+ return geometry
523
607
 
524
608
  @classmethod
525
609
  def register(cls, entity_cls: type[BaseEntity]) -> None:
@@ -588,27 +672,29 @@ def _entity_name(entity_cls: type[BaseEntity]) -> str:
588
672
  def _bbox_to_polygon(
589
673
  bbox: tuple[float, float, float, float],
590
674
  ) -> "shapely.Geometry":
591
- import shapely.geometry as sgeom
592
-
593
675
  minx, miny, maxx, maxy = bbox
594
676
  return sgeom.box(minx, miny, maxx, maxy)
595
677
 
596
678
 
597
- def _validate_entity(entity: BaseEntity, name: str) -> None:
598
- """Validate an entity's paths and schema.
679
+ def _validate_entity_paths(entity: BaseEntity, name: str) -> None:
680
+ """Validate an entity's path existence only — no Parquet I/O.
599
681
 
600
- File-existence checks are performed eagerly for local paths. Schema
601
- validation (key columns, geometry column) is delegated to
602
- :attr:`BaseEntity.schema_profile`, which routes through the process-level
603
- :func:`_cached_detect` cache so the second ``AoI()`` pointed at the
604
- same ``data_root`` pays zero Parquet footer reads for validation.
682
+ Checks that ``static_path``, ``annual_path``, and ``fortnightly_path``
683
+ resolve to existing local files. Schema validation (key columns,
684
+ geometry column, geometry type) is intentionally deferred to first data
685
+ access via :attr:`BaseEntity.schema_profile`, which routes through the
686
+ process-level :func:`~core_lens.base.entity._cached_detect` cache.
687
+
688
+ This is the fast variant called from :meth:`AoI._get_entity`. Use
689
+ :func:`_validate_entity` for the full eager check (e.g. in
690
+ :meth:`AoI.validate` or for absolute-path entities at register time).
605
691
 
606
692
  Args:
607
693
  entity (BaseEntity): The entity instance to validate.
608
694
  name (str): Human-readable entity name for error messages.
609
695
 
610
696
  Raises:
611
- EntityValidationError: If any validation check fails.
697
+ EntityValidationError: If any path does not exist.
612
698
  """
613
699
  # --- Static path existence check ----------------------------------------
614
700
  try:
@@ -623,9 +709,6 @@ def _validate_entity(entity: BaseEntity, name: str) -> None:
623
709
  f"Entity {name!r}: static_path {entity.static_path!r} does not exist."
624
710
  )
625
711
 
626
- # For local paths perform an eager existence check; for cloud paths we rely
627
- # on the schema-read below to surface a missing-file error (avoids an extra
628
- # HeadObject call per entity at startup).
629
712
  if not is_cloud_uri(static) and not os.path.exists(static):
630
713
  logger.error(
631
714
  "Validation failed for entity {}: static path '{}' does not exist.",
@@ -636,26 +719,7 @@ def _validate_entity(entity: BaseEntity, name: str) -> None:
636
719
  f"Entity {name!r}: static_path {static!r} does not exist."
637
720
  )
638
721
 
639
- # --- Schema validation (key_cols, geometry_col, geometry_type) -----------
640
- # Delegate to schema_profile which routes through _cached_detect.
641
- # detect() internally calls _require_cols for key_cols and geometry_col,
642
- # and infers geometry_type — so this covers the same checks that the old
643
- # _validate_entity did with a raw pl.scan_parquet().collect_schema(), but
644
- # without a redundant footer read.
645
- try:
646
- _ = entity.schema_profile
647
- except Exception as exc:
648
- logger.error(
649
- "Validation failed for entity {}: could not read schema from '{}': {}",
650
- name,
651
- static,
652
- exc,
653
- )
654
- raise EntityValidationError(
655
- f"Entity {name!r}: could not read schema from {static!r}: {exc}"
656
- ) from exc
657
-
658
- # --- Temporal path existence checks -------------------------------------
722
+ # --- Temporal path existence checks (no schema read) --------------------
659
723
  for attr, label in [("annual_path", "annual"), ("fortnightly_path", "fortnightly")]:
660
724
  path = getattr(entity, attr)
661
725
  if path is not None:
@@ -663,10 +727,8 @@ def _validate_entity(entity: BaseEntity, name: str) -> None:
663
727
  abs_path = entity._resolve(path)
664
728
  except FileNotFoundError:
665
729
  abs_path = None
666
- if (
667
- abs_path is not None
668
- and not is_cloud_uri(abs_path)
669
- and not os.path.exists(abs_path)
730
+ if abs_path is None or (
731
+ not is_cloud_uri(abs_path) and not os.path.exists(abs_path)
670
732
  ):
671
733
  logger.error(
672
734
  "Validation failed for entity {}: {} path '{}' does not exist.",
@@ -677,13 +739,47 @@ def _validate_entity(entity: BaseEntity, name: str) -> None:
677
739
  raise EntityValidationError(
678
740
  f"Entity {name!r}: {label}_path {path!r} does not exist."
679
741
  )
680
- elif abs_path is None:
681
- logger.error(
682
- "Validation failed for entity {}: {} path '{}' does not exist.",
683
- name,
684
- label,
685
- path,
686
- )
687
- raise EntityValidationError(
688
- f"Entity {name!r}: {label}_path {path!r} does not exist."
689
- )
742
+
743
+
744
+ def _validate_entity(entity: BaseEntity, name: str) -> None:
745
+ """Validate an entity's paths and schema (full eager check).
746
+
747
+ Runs path existence checks via :func:`_validate_entity_paths`, then
748
+ triggers schema detection (Parquet footer reads) by accessing
749
+ :attr:`BaseEntity.schema_profile`. The result is cached by
750
+ :func:`~core_lens.base.entity._cached_detect`, so the second call for the
751
+ same ``data_root`` pays no I/O.
752
+
753
+ Called by :meth:`AoI.validate` and by :meth:`AoI.register` for
754
+ entities with absolute paths. Normal lazy instantiation via
755
+ :meth:`AoI._get_entity` uses the cheaper :func:`_validate_entity_paths`.
756
+
757
+ Args:
758
+ entity (BaseEntity): The entity instance to validate.
759
+ name (str): Human-readable entity name for error messages.
760
+
761
+ Raises:
762
+ EntityValidationError: If any validation check fails.
763
+ """
764
+ # Fast path-only check first.
765
+ _validate_entity_paths(entity, name)
766
+
767
+ # --- Schema validation (key_cols, geometry_col, geometry_type) -----------
768
+ # Accessing schema_profile triggers _cached_detect, which reads Parquet
769
+ # footer metadata. Subsequent calls for the same data_root are free.
770
+ try:
771
+ static = entity._resolve(entity.static_path)
772
+ _ = entity.schema_profile
773
+ except EntityValidationError:
774
+ raise
775
+ except Exception as exc:
776
+ static = entity.static_path # best-effort for the error message
777
+ logger.error(
778
+ "Validation failed for entity {}: could not read schema from '{}': {}",
779
+ name,
780
+ static,
781
+ exc,
782
+ )
783
+ raise EntityValidationError(
784
+ f"Entity {name!r}: could not read schema from {static!r}: {exc}"
785
+ ) from exc
@@ -5,7 +5,9 @@ from __future__ import annotations
5
5
  import pathlib
6
6
  from typing import TYPE_CHECKING, Any
7
7
 
8
+ import numpy as np
8
9
  import polars as pl
10
+ import pyarrow.dataset as ds
9
11
  import shapely
10
12
  from loguru import logger
11
13
 
@@ -14,6 +16,98 @@ from core_lens.utils.paths import is_cloud_uri, path_exists, resolve_fs_and_path
14
16
  if TYPE_CHECKING:
15
17
  pass
16
18
 
19
+ # ---------------------------------------------------------------------------
20
+ # Bbox index sidecar helpers
21
+ #
22
+ # A sidecar Parquet file caches the pre-computed bounding-box index so that
23
+ # the expensive WKB-decode loop in build_bbox_index is only paid once per
24
+ # dataset (not once per process). Subsequent runs load the index directly
25
+ # from the sidecar — no geometry decoding required.
26
+ #
27
+ # Sidecar location priority:
28
+ # 1. Alongside the static parquet directory/file (requires write access).
29
+ # 2. ~/.cache/core_lens/ (fallback for read-only data roots).
30
+ # ---------------------------------------------------------------------------
31
+
32
+ _SIDECAR_SUFFIX = ".bbox_index.parquet"
33
+ _CACHE_DIR = pathlib.Path.home() / ".cache" / "core_lens"
34
+
35
+
36
+ def _bbox_sidecar_path(static_path: str) -> pathlib.Path | None:
37
+ """Return the preferred path for the bbox index sidecar, or ``None`` for cloud URIs.
38
+
39
+ Tries the directory alongside ``static_path`` first; if that directory is
40
+ not writable (e.g. read-only data root), falls back to
41
+ ``~/.cache/core_lens/``.
42
+
43
+ Args:
44
+ static_path (str): Absolute local path to the static parquet file/directory.
45
+
46
+ Returns:
47
+ pathlib.Path | None: Sidecar path, or ``None`` if ``static_path`` is a cloud URI.
48
+ """
49
+ if is_cloud_uri(static_path):
50
+ return None
51
+ p = pathlib.Path(static_path)
52
+ # For a partitioned directory dataset, the sidecar lives inside the directory.
53
+ # For a single file, it lives next to it.
54
+ parent = p if p.is_dir() else p.parent
55
+ sidecar = parent / (p.name + _SIDECAR_SUFFIX)
56
+ # Check if the parent directory is writable.
57
+ if not parent.exists() or not parent.stat().st_mode & 0o200:
58
+ # Fallback to user cache dir.
59
+ import hashlib
60
+
61
+ fingerprint = hashlib.sha1(static_path.encode()).hexdigest()[:16]
62
+ _CACHE_DIR.mkdir(parents=True, exist_ok=True)
63
+ sidecar = _CACHE_DIR / (fingerprint + _SIDECAR_SUFFIX)
64
+ return sidecar
65
+
66
+
67
+ def _read_bbox_sidecar(sidecar: pathlib.Path, static_path: str) -> pl.DataFrame | None:
68
+ """Read the sidecar if it exists and is newer than the static data.
69
+
70
+ Args:
71
+ sidecar (pathlib.Path): Candidate sidecar path.
72
+ static_path (str): Absolute local path to the static parquet file/directory.
73
+
74
+ Returns:
75
+ pl.DataFrame | None: The cached index, or ``None`` if the sidecar is
76
+ absent or stale.
77
+ """
78
+ if not sidecar.exists():
79
+ return None
80
+ try:
81
+ static_mtime = pathlib.Path(static_path).stat().st_mtime
82
+ sidecar_mtime = sidecar.stat().st_mtime
83
+ if sidecar_mtime < static_mtime:
84
+ logger.debug(
85
+ "Bbox sidecar {} is stale (older than {}), rebuilding.",
86
+ sidecar,
87
+ static_path,
88
+ )
89
+ return None
90
+ logger.debug("Loading bbox index from sidecar: {}", sidecar)
91
+ return pl.read_parquet(str(sidecar))
92
+ except Exception as exc:
93
+ logger.warning("Failed to read bbox sidecar {}: {}", sidecar, exc)
94
+ return None
95
+
96
+
97
+ def _write_bbox_sidecar(df: pl.DataFrame, sidecar: pathlib.Path) -> None:
98
+ """Write ``df`` to the sidecar file, silently ignoring any write errors.
99
+
100
+ Args:
101
+ df (pl.DataFrame): The bbox index DataFrame to persist.
102
+ sidecar (pathlib.Path): Destination sidecar path.
103
+ """
104
+ try:
105
+ sidecar.parent.mkdir(parents=True, exist_ok=True)
106
+ df.write_parquet(str(sidecar))
107
+ logger.debug("Wrote bbox index sidecar: {}", sidecar)
108
+ except Exception as exc:
109
+ logger.debug("Could not write bbox index sidecar {}: {}", sidecar, exc)
110
+
17
111
 
18
112
  def resolve_path(path: str) -> str:
19
113
  """Return an absolute path or cloud URI string, resolving local relative paths against cwd.
@@ -99,7 +193,15 @@ def build_bbox_index(
99
193
  + "Cannot compute bounds from separate lat/lon columns without bbox hints."
100
194
  )
101
195
 
102
- import pyarrow.dataset as ds
196
+ # --- Sidecar fast path (local paths only) --------------------------------
197
+ # Check for a pre-built index sidecar before doing the expensive WKB loop.
198
+ sidecar: pathlib.Path | None = None
199
+ if not is_cloud_uri(static_path):
200
+ sidecar = _bbox_sidecar_path(static_path)
201
+ if sidecar is not None:
202
+ cached = _read_bbox_sidecar(sidecar, static_path)
203
+ if cached is not None:
204
+ return cached
103
205
 
104
206
  logger.debug(
105
207
  "No pre-computed bbox found in {}; falling back to Shapely decoding",
@@ -133,17 +235,23 @@ def build_bbox_index(
133
235
  chunks.append(chunk)
134
236
 
135
237
  if chunks:
136
- return pl.concat(chunks)
238
+ result = pl.concat(chunks)
239
+ else:
240
+ empty_df = pl.read_parquet(
241
+ static_path, columns=key_cols, storage_options=_so or None
242
+ )
243
+ result = empty_df.with_columns(
244
+ pl.Series("minx", [], dtype=pl.Float64),
245
+ pl.Series("miny", [], dtype=pl.Float64),
246
+ pl.Series("maxx", [], dtype=pl.Float64),
247
+ pl.Series("maxy", [], dtype=pl.Float64),
248
+ )
137
249
 
138
- empty_df = pl.read_parquet(
139
- static_path, columns=key_cols, storage_options=_so or None
140
- )
141
- return empty_df.with_columns(
142
- pl.Series("minx", [], dtype=pl.Float64),
143
- pl.Series("miny", [], dtype=pl.Float64),
144
- pl.Series("maxx", [], dtype=pl.Float64),
145
- pl.Series("maxy", [], dtype=pl.Float64),
146
- )
250
+ # Persist the sidecar for future runs (local paths only, errors silenced).
251
+ if sidecar is not None:
252
+ _write_bbox_sidecar(result, sidecar)
253
+
254
+ return result
147
255
 
148
256
 
149
257
  def bbox_intersects_geometry(
@@ -213,8 +321,6 @@ def exact_spatial_filter(
213
321
  Raises:
214
322
  ValueError: If ``relationship`` is not one of the valid options.
215
323
  """
216
- import shapely
217
-
218
324
  logger.debug(
219
325
  "Refining {} candidates with exact spatial filter (relationship='{}')",
220
326
  len(candidates),
@@ -247,8 +353,6 @@ def exact_spatial_filter(
247
353
  else:
248
354
  geoms = shapely.from_wkt(geom_array)
249
355
 
250
- import numpy as np
251
-
252
356
  if relationship == "centroid":
253
357
  # Centroid mode: entity centroid must lie within the AoI geometry.
254
358
  # STRtree is overkill for a single-query containment test — building
@@ -320,8 +424,6 @@ def execute_spatial_join(
320
424
  pl.DataFrame: ``primary_df`` with additional columns
321
425
  ``{other_entity_name}_{col}`` appended for each ``agg`` entry.
322
426
  """
323
- import shapely
324
-
325
427
  logger.info(
326
428
  "Starting execute_spatial_join for other_entity_name={}", other_entity_name
327
429
  )
@@ -353,8 +455,6 @@ def execute_spatial_join(
353
455
  return primary_df
354
456
 
355
457
  # Compute bounding box of all primary geometries
356
- import numpy as np
357
-
358
458
  bnds = shapely.bounds(primary_geoms)
359
459
  valid_bnds = bnds[~np.isnan(bnds).any(axis=1)]
360
460
  if len(valid_bnds) > 0:
File without changes
File without changes
File without changes
File without changes