core-lens 0.1.dev171__tar.gz → 0.1.dev174__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/PKG-INFO +1 -1
  2. core_lens-0.1.dev174/benchmarks/README.md +76 -0
  3. core_lens-0.1.dev174/benchmarks/bench_entity.py +307 -0
  4. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/benchmarks/bench_export.py +65 -2
  5. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/benchmarks/bench_polars_utils.py +28 -0
  6. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/benchmarks/bench_result.py +95 -3
  7. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/benchmarks/bench_schema.py +104 -0
  8. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/benchmarks/bench_spatial.py +68 -0
  9. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/benchmarks/bench_view.py +99 -5
  10. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/benchmarks/run_all.sh +2 -0
  11. core_lens-0.1.dev174/parse_scalene.py +50 -0
  12. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/_version.py +2 -2
  13. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/base/result.py +13 -2
  14. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/export/formats.py +17 -1
  15. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/utils/paths.py +72 -8
  16. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/utils/season.py +23 -6
  17. core_lens-0.1.dev174/tests/unit/test_season.py +251 -0
  18. core_lens-0.1.dev171/benchmarks/README.md +0 -60
  19. core_lens-0.1.dev171/tests/unit/test_season.py +0 -130
  20. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/.github/ISSUE_TEMPLATE/blank-proposal.yaml +0 -0
  21. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/.github/ISSUE_TEMPLATE/bug-report.yaml +0 -0
  22. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/.github/ISSUE_TEMPLATE/feature-request.yaml +0 -0
  23. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/.github/pull_request_template.md +0 -0
  24. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/.github/workflows/ci.yml +0 -0
  25. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/.github/workflows/gh-pages.yml +0 -0
  26. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/.github/workflows/pre-release.yml +0 -0
  27. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/.github/workflows/release.yml +0 -0
  28. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/.gitignore +0 -0
  29. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/.gitmessage +0 -0
  30. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/.pre-commit-config.yaml +0 -0
  31. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/.python-version +0 -0
  32. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/CONTRIBUTING.md +0 -0
  33. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/LICENSE +0 -0
  34. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/README.md +0 -0
  35. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/SKILLS.md +0 -0
  36. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/benchmarks/bench_aoi.py +0 -0
  37. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/benchmarks/bench_season.py +0 -0
  38. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/docs/Makefile +0 -0
  39. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/docs/make.bat +0 -0
  40. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/docs/source/concepts.md +0 -0
  41. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/docs/source/conf.py +0 -0
  42. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/docs/source/export.md +0 -0
  43. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/docs/source/index.rst +0 -0
  44. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/docs/source/intro.md +0 -0
  45. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/docs/source/logging.md +0 -0
  46. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/docs/source/plots.md +0 -0
  47. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/docs/source/plugins.md +0 -0
  48. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/docs/source/queries.md +0 -0
  49. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/docs/source/quickstart.md +0 -0
  50. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/docs/source/stats.md +0 -0
  51. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/examples/demo_mws.py +0 -0
  52. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/examples/demo_tehsil.py +0 -0
  53. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/hooks/mypy.sh +0 -0
  54. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/hooks/no-parquet-outside-fixtures.sh +0 -0
  55. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/hooks/pytest.sh +0 -0
  56. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/pyproject.toml +0 -0
  57. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/__init__.py +0 -0
  58. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/__main__.py +0 -0
  59. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/aoi.py +0 -0
  60. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/base/__init__.py +0 -0
  61. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/base/entity.py +0 -0
  62. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/base/namespaces/__init__.py +0 -0
  63. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/base/namespaces/plot.py +0 -0
  64. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/base/namespaces/stats.py +0 -0
  65. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/base/view.py +0 -0
  66. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/entities/__init__.py +0 -0
  67. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/entities/mws.py +0 -0
  68. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/entities/tehsil.py +0 -0
  69. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/entities/waterbody.py +0 -0
  70. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/export/__init__.py +0 -0
  71. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/py.typed +0 -0
  72. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/schema/__init__.py +0 -0
  73. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/schema/detection.py +0 -0
  74. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/schema/profile.py +0 -0
  75. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/utils/__init__.py +0 -0
  76. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/utils/polars_utils.py +0 -0
  77. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/src/core_lens/utils/spatial.py +0 -0
  78. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/tests/fixtures/generate_fixtures.py +0 -0
  79. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/tests/unit/conftest.py +0 -0
  80. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/tests/unit/test_aoi.py +0 -0
  81. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/tests/unit/test_entities.py +0 -0
  82. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/tests/unit/test_entity.py +0 -0
  83. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/tests/unit/test_export.py +0 -0
  84. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/tests/unit/test_main.py +0 -0
  85. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/tests/unit/test_plot.py +0 -0
  86. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/tests/unit/test_polars_utils.py +0 -0
  87. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/tests/unit/test_profile.py +0 -0
  88. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/tests/unit/test_result.py +0 -0
  89. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/tests/unit/test_schema_detection.py +0 -0
  90. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/tests/unit/test_schema_profile.py +0 -0
  91. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/tests/unit/test_season_config.py +0 -0
  92. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/tests/unit/test_spatial.py +0 -0
  93. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/tests/unit/test_stats.py +0 -0
  94. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/tests/unit/test_view.py +0 -0
  95. {core_lens-0.1.dev171 → core_lens-0.1.dev174}/uv.lock +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: core-lens
3
- Version: 0.1.dev171
3
+ Version: 0.1.dev174
4
4
  Summary: Query, analyse, and visualise CoreStack's microwatershed and Earth science data through a clean, composable Python API.
5
5
  Project-URL: Homepage, https://github.com/ApoorvaKashyap/core-lens
6
6
  Project-URL: Issues, https://github.com/ApoorvaKashyap/core-lens/issues
@@ -0,0 +1,76 @@
1
+ # core-lens Benchmarks
2
+
3
+ Focused Scalene profiling scripts — one per module/subsystem.
4
+
5
+ ## Structure
6
+
7
+ ```
8
+ benchmarks/
9
+ ├── README.md
10
+ ├── run_all.sh # Run all benchmarks in sequence
11
+ ├── bench_aoi.py # AoI init, register, named-boundary resolution, current_season/year
12
+ ├── bench_entity.py # BaseEntity._resolve, where, spatial_filter, spatial_join, caches [NEW]
13
+ ├── bench_spatial.py # build_bbox_index, bbox_intersects_geometry, exact_spatial_filter, execute_spatial_join
14
+ ├── bench_paths.py # is_cloud_uri, resolve_fs_and_path, path_exists, join_uri [NEW]
15
+ ├── bench_view.py # View.where, spatial_filter, between (all modes + errors), spatial_join, materialise
16
+ ├── bench_result.py # Result.derive, aggregate (all groupings + errors), with_geometry, df/gdf, stats/plot
17
+ ├── bench_season.py # resolve_time_filter, add_temporal_columns, SeasonConfig.season_for
18
+ ├── bench_schema.py # schema detection (_infer_* helpers, _is_year_col, _find_lon_companion, from_file)
19
+ ├── bench_polars_utils.py # scan_with_key_filter, collect_lf, cached_read_schema (CPU path)
20
+ └── bench_export.py # parquet, json, csv, geoparquet, geojson, GeoJSONSeq, error paths
21
+ ```
22
+
23
+ ## Running
24
+
25
+ ### Single benchmark
26
+
27
+ ```bash
28
+ uv run scalene --html --outfile benchmarks/profiles/bench_spatial.html benchmarks/bench_spatial.py
29
+ ```
30
+
31
+ ### All benchmarks (sequential, JSON output via Scalene)
32
+
33
+ ```bash
34
+ chmod +x benchmarks/run_all.sh
35
+ ./benchmarks/run_all.sh
36
+ ```
37
+
38
+ Profiles land in `benchmarks/profiles/`.
39
+
40
+ ### Quick timing-only run (no Scalene overhead)
41
+
42
+ ```bash
43
+ uv run python benchmarks/bench_spatial.py
44
+ ```
45
+
46
+ ## What to look for in Scalene output
47
+
48
+ | Signal | Meaning |
49
+ |---|---|
50
+ | High **CPU %** on line | Compute bottleneck — vectorise or push into Polars expr |
51
+ | High **memory** spike | Possible copy-on-write or large intermediate frame |
52
+ | `[native]` vs `[Python]` | Native = C ext (fast). Python = pure Python (optimise) |
53
+ | **GPU %** column | cudf path hit or not |
54
+
55
+ ## Coverage map
56
+
57
+ | File | Functions / paths covered |
58
+ |---|---|
59
+ | `bench_aoi.py` | `AoI.register`, `AoI.__init__` (bbox/geometry/named/seasons), `__getattr__`, `current_season/year`, `registered_entities`, `deregister` |
60
+ | `bench_entity.py` | `_resolve` (local/cloud), `where` (single/composite/list/error), `spatial_filter` (bbox/geometry/area/error), `spatial_join`, `schema_profile` cache, `_index` cache, `geometry_lazy`, `_so_key`, `_entity_name`, `_cached_detect`, `EntityValidationError` |
61
+ | `bench_spatial.py` | `build_bbox_index`, `bbox_intersects_geometry`, `exact_spatial_filter` (centroid/area/all-India), `execute_spatial_join`, `_bbox_sidecar_path`, `_read_bbox_sidecar`, error path |
62
+ | `bench_view.py` | `where` (single/composite), `spatial_filter` (bbox), `between` (dates/season/CURRENT/year-range/errors), `spatial_join` (spec/double-error), `static`/`annual`/`sub_annual` materialisation |
63
+ | `bench_result.py` | `df`, `lazy`, `gdf` (WKB decode), `materialise`, `derive` (single/chain), `with_geometry` (noop/annual), `aggregate` (no-by/year/month/season/year_month/season_year/errors), `gdf` TypeError guard, `stats`/`plot` accessors |
64
+ | `bench_season.py` | `SeasonConfig.__post_init__` (default/custom), `season_for`, `resolve_time_filter` (date-range/kharif/rabi/year-range), `add_temporal_columns` (first/second noop call), Polars filter execution |
65
+ | `bench_schema.py` | `_read_schema`, `_infer_geometry_type`, `_infer_bbox_cols`, `_infer_time_col`, `_is_year_col_from_schema`, `_find_lon_companion`, `_require_cols` (ok/error), `detect` (static/full), `SchemaProfile.from_file`, `entity.schema_profile` (cold/warm) |
66
+ | `bench_polars_utils.py` | `_gpu_available`, `scan_with_key_filter` (build/+time_expr), `collect_lf` (1-key/small/all), `cached_read_schema` (warm/cold) |
67
+ | `bench_export.py` | `_get_duckdb_conn` (cold/cached), `parquet`/`json`/`csv`/`geoparquet`/`geojson` (small/all), `geojson(GeoJSONSeq)`, all error paths, format size comparison |
68
+
69
+ ## Prerequisites
70
+
71
+ ```bash
72
+ uv sync --group dev
73
+ ```
74
+
75
+ Each script needs `DATA_ROOT` pointing at a directory with the expected entity Parquet files.
76
+ Edit the `DATA_ROOT` constant at the top of each file to match your local data path.
@@ -0,0 +1,307 @@
1
+ """bench_entity.py — Benchmark for core_lens.base.entity.BaseEntity
2
+ ===================================================================
3
+ Targets:
4
+ - BaseEntity._resolve() absolute / relative / cloud URI resolution
5
+ - BaseEntity.where() attribute filter (single col, composite, list val)
6
+ - BaseEntity.where() entity-kwarg spatial lookup path
7
+ - BaseEntity.spatial_filter() full pipeline (bbox + geometry mode)
8
+ - BaseEntity.spatial_join() deferred join spec construction
9
+ - BaseEntity.schema_profile instance-level cache (hasattr guard)
10
+ - BaseEntity._index process-level cache (_cached_build_index)
11
+ - BaseEntity.geometry_lazy lazy geometry scan construction
12
+ - _so_key() storage_options → hashable tuple
13
+ - _entity_name() entity class name → string
14
+ - _cached_detect() process-level schema cache (functools.cache)
15
+ - _cached_build_index() process-level bbox index cache
16
+ - EntityValidationError exception class (import overhead)
17
+
18
+ Scalene focuses on:
19
+ - _resolve() pathlib.Path ops vs cloud URI fast-path
20
+ - cached_read_schema() in where() — hot path schema lookup
21
+ - bbox_intersects_geometry() → exact_spatial_filter() inside where()
22
+ - functools.cache lookup overhead in _cached_detect / _cached_build_index
23
+
24
+ Usage:
25
+ uv run scalene run --html --outfile benchmarks/profiles/bench_entity.html benchmarks/bench_entity.py
26
+ uv run python benchmarks/bench_entity.py
27
+ """
28
+
29
+ from __future__ import annotations
30
+
31
+ import time
32
+
33
+ import shapely.geometry as sgeom
34
+
35
+ from core_lens import AoI
36
+ from core_lens.entities.mws import MWSEntity
37
+ from core_lens.base.entity import (
38
+ _so_key,
39
+ _entity_name,
40
+ _cached_detect,
41
+ EntityValidationError,
42
+ )
43
+
44
+ # ── Config ────────────────────────────────────────────────────────────────────
45
+ DATA_ROOT = "data/"
46
+ INDIA_BBOX: tuple[float, float, float, float] = (68.0, 6.0, 98.0, 38.0)
47
+ SMALL_BBOX: tuple[float, float, float, float] = (86.0, 21.0, 87.5, 22.0)
48
+ MWS_ID = "12_322881" # update to a valid mws_id in your dataset
49
+
50
+ AoI.register(MWSEntity)
51
+ entity = MWSEntity(data_root=DATA_ROOT)
52
+ static_path = entity._resolve(entity.static_path)
53
+
54
+
55
+ def _section(title: str) -> None:
56
+ print(f"\n{'─' * 60}")
57
+ print(f" {title}")
58
+ print("─" * 60)
59
+
60
+
61
+ # ── 1. _so_key() — storage_options → hashable tuple ──────────────────────────
62
+ _section("1. _so_key() [dict → sorted tuple — O(n log n)]")
63
+ REPS = 1_000_000
64
+
65
+ so_empty: dict[str, str] = {}
66
+ so_aws: dict[str, str] = {"region": "ap-south-1", "access_key": "AKIAIOSFODNN7EXAMPLE"}
67
+
68
+ t0 = time.perf_counter()
69
+ for _ in range(REPS):
70
+ _so_key(so_empty)
71
+ t1 = time.perf_counter()
72
+ print(
73
+ f"_so_key (empty) ×{REPS}: {(t1 - t0) * 1000:.2f} ms total "
74
+ f"({(t1 - t0) / REPS * 1e9:.1f} ns/call)"
75
+ )
76
+
77
+ t0 = time.perf_counter()
78
+ for _ in range(REPS):
79
+ _so_key(so_aws)
80
+ t1 = time.perf_counter()
81
+ print(
82
+ f"_so_key (2 keys) ×{REPS}: {(t1 - t0) * 1000:.2f} ms total "
83
+ f"({(t1 - t0) / REPS * 1e9:.1f} ns/call)"
84
+ )
85
+ print(f" _so_key(aws): {_so_key(so_aws)}")
86
+
87
+
88
+ # ── 2. _entity_name() — class → string ───────────────────────────────────────
89
+ _section("2. _entity_name() [class → name string, 'Entity' suffix strip]")
90
+ REPS_EN = 1_000_000
91
+ t0 = time.perf_counter()
92
+ for _ in range(REPS_EN):
93
+ _entity_name(MWSEntity)
94
+ t1 = time.perf_counter()
95
+ print(
96
+ f"_entity_name ×{REPS_EN}: {(t1 - t0) * 1000:.2f} ms total "
97
+ f"({(t1 - t0) / REPS_EN * 1e9:.1f} ns/call)"
98
+ )
99
+ print(f" _entity_name(MWSEntity): {_entity_name(MWSEntity)!r} (expect 'mws')")
100
+
101
+
102
+ # ── 3. BaseEntity._resolve() — local relative path ───────────────────────────
103
+ _section("3. BaseEntity._resolve() [relative → absolute local path]")
104
+ REPS_R = 100_000
105
+ t0 = time.perf_counter()
106
+ for _ in range(REPS_R):
107
+ entity._resolve(entity.static_path)
108
+ t1 = time.perf_counter()
109
+ print(
110
+ f"_resolve (relative) ×{REPS_R}: {(t1 - t0) * 1000:.2f} ms total "
111
+ f"({(t1 - t0) / REPS_R * 1e6:.2f} µs/call)"
112
+ )
113
+ print(f" resolved: {entity._resolve(entity.static_path)!r}")
114
+
115
+ # Cloud URI fast-path (no pathlib.Path, no existence check).
116
+ entity_cloud_sim = MWSEntity.__new__(MWSEntity) # bypass __init__ for testing
117
+ entity_cloud_sim._data_root = "s3://my-bucket/data"
118
+ entity_cloud_sim._storage_options = {}
119
+ REPS_CR = 100_000
120
+ t0 = time.perf_counter()
121
+ for _ in range(REPS_CR):
122
+ entity_cloud_sim._resolve("s3://my-bucket/data/mws/static/")
123
+ t1 = time.perf_counter()
124
+ print(
125
+ f"_resolve (cloud abs) ×{REPS_CR}: {(t1 - t0) * 1000:.2f} ms total "
126
+ f"({(t1 - t0) / REPS_CR * 1e6:.2f} µs/call)"
127
+ )
128
+
129
+
130
+ # ── 4. BaseEntity.schema_profile — instance cache ────────────────────────────
131
+ _section("4. BaseEntity.schema_profile [hasattr guard — instance-level cache]")
132
+ entity2 = MWSEntity(data_root=DATA_ROOT)
133
+ t0 = time.perf_counter()
134
+ entity2.schema_profile # cold — triggers _cached_detect
135
+ t1 = time.perf_counter()
136
+ print(f"schema_profile (cold) : {(t1 - t0) * 1000:.2f} ms")
137
+
138
+ REPS_SP = 100_000
139
+ t0 = time.perf_counter()
140
+ for _ in range(REPS_SP):
141
+ entity2.schema_profile # warm — hasattr guard short-circuits
142
+ t1 = time.perf_counter()
143
+ print(
144
+ f"schema_profile (warm) ×{REPS_SP}: {(t1 - t0) * 1000:.2f} ms total "
145
+ f"({(t1 - t0) / REPS_SP * 1e6:.3f} µs/call)"
146
+ )
147
+
148
+
149
+ # ── 5. _cached_detect() — process-level functools.cache ──────────────────────
150
+ _section("5. _cached_detect() [process-level functools.cache lookup]")
151
+ profile = entity.schema_profile
152
+ key = (
153
+ static_path,
154
+ tuple(entity.key_cols),
155
+ entity.geometry_col,
156
+ entity._resolve(entity.annual_path) if entity.annual_path else None,
157
+ entity._resolve(entity.sub_annual_path) if entity.sub_annual_path else None,
158
+ _so_key(entity._storage_options),
159
+ )
160
+ # Prime the cache.
161
+ _cached_detect(*key)
162
+
163
+ REPS_CD = 100_000
164
+ t0 = time.perf_counter()
165
+ for _ in range(REPS_CD):
166
+ _cached_detect(*key)
167
+ t1 = time.perf_counter()
168
+ print(
169
+ f"_cached_detect (warm) ×{REPS_CD}: {(t1 - t0) * 1000:.2f} ms total "
170
+ f"({(t1 - t0) / REPS_CD * 1e6:.3f} µs/call)"
171
+ )
172
+ print(f"Cache info: {_cached_detect.cache_info()}")
173
+
174
+
175
+ # ── 6. BaseEntity._index — process-level bbox cache ──────────────────────────
176
+ _section("6. BaseEntity._index [process-level bbox index cache]")
177
+ t0 = time.perf_counter()
178
+ idx = entity._index # may be warm from earlier modules
179
+ t1 = time.perf_counter()
180
+ print(f"_index (first access) : {(t1 - t0) * 1000:.2f} ms")
181
+ print(f"Index shape : {idx.shape}")
182
+
183
+ REPS_IX = 100_000
184
+ t0 = time.perf_counter()
185
+ for _ in range(REPS_IX):
186
+ entity._index # warm — hasattr guard short-circuits
187
+ t1 = time.perf_counter()
188
+ print(
189
+ f"_index (cached) ×{REPS_IX}: {(t1 - t0) * 1000:.2f} ms total "
190
+ f"({(t1 - t0) / REPS_IX * 1e6:.3f} µs/call)"
191
+ )
192
+
193
+
194
+ # ── 7. BaseEntity.geometry_lazy — lazy geometry LazyFrame ────────────────────
195
+ _section("7. BaseEntity.geometry_lazy [lazy scan construction + hasattr cache]")
196
+ entity3 = MWSEntity(data_root=DATA_ROOT)
197
+ t0 = time.perf_counter()
198
+ lf = entity3.geometry_lazy # cold — builds pl.scan_parquet plan
199
+ t1 = time.perf_counter()
200
+ print(f"geometry_lazy (cold) : {(t1 - t0) * 1000:.2f} ms")
201
+
202
+ REPS_GL = 100_000
203
+ t0 = time.perf_counter()
204
+ for _ in range(REPS_GL):
205
+ entity3.geometry_lazy # warm — hasattr guard short-circuits
206
+ t1 = time.perf_counter()
207
+ print(
208
+ f"geometry_lazy (warm) ×{REPS_GL}: {(t1 - t0) * 1000:.2f} ms total "
209
+ f"({(t1 - t0) / REPS_GL * 1e6:.3f} µs/call)"
210
+ )
211
+
212
+
213
+ # ── 8. BaseEntity.where() — single attribute filter ──────────────────────────
214
+ _section("8. BaseEntity.where(state='Bihar') [single attribute filter]")
215
+ t0 = time.perf_counter()
216
+ view_attr = entity.where(state="Bihar")
217
+ t1 = time.perf_counter()
218
+ print(f"where(state) : {(t1 - t0) * 1000:.2f} ms")
219
+ print(f"Bihar keys : {view_attr.keys.collect().shape}")
220
+
221
+
222
+ # ── 9. BaseEntity.where() — composite attribute filter ───────────────────────
223
+ _section("9. BaseEntity.where(state=..., district=..., tehsil=...) [composite AND]")
224
+ t0 = time.perf_counter()
225
+ view_composite = entity.where(state="Bihar", district="Banka", tehsil="Banka")
226
+ t1 = time.perf_counter()
227
+ print(f"where(3 cols) : {(t1 - t0) * 1000:.2f} ms")
228
+ print(f"Banka keys : {view_composite.keys.collect().shape}")
229
+
230
+
231
+ # ── 10. BaseEntity.where() — list value (is_in) ──────────────────────────────
232
+ _section("10. BaseEntity.where(state=['Bihar', 'Jharkhand']) [list value → is_in]")
233
+ t0 = time.perf_counter()
234
+ view_list = entity.where(state=["Bihar", "Jharkhand"])
235
+ t1 = time.perf_counter()
236
+ print(f"where(list) : {(t1 - t0) * 1000:.2f} ms")
237
+ print(f"Bihar+Jharkhand keys : {view_list.keys.collect().shape}")
238
+
239
+
240
+ # ── 11. BaseEntity.where() — unknown kwarg error ─────────────────────────────
241
+ _section("11. BaseEntity.where() [unknown kwarg → ValueError guard]")
242
+ err_raised = False
243
+ try:
244
+ entity.where(totally_unknown_column="xyz")
245
+ except ValueError:
246
+ err_raised = True
247
+ print(f"ValueError guard : {err_raised} (expect True)")
248
+
249
+
250
+ # ── 12. BaseEntity.spatial_filter() — bbox mode ──────────────────────────────
251
+ _section("12. BaseEntity.spatial_filter(bbox=SMALL_BBOX) [full pipeline]")
252
+ t0 = time.perf_counter()
253
+ view_sf = entity.spatial_filter(bbox=SMALL_BBOX)
254
+ t1 = time.perf_counter()
255
+ print(f"spatial_filter (bbox): {(t1 - t0) * 1000:.2f} ms")
256
+ print(f"Keys : {view_sf.keys.collect().shape}")
257
+
258
+ # geometry mode
259
+ small_poly = sgeom.box(*SMALL_BBOX)
260
+ t0 = time.perf_counter()
261
+ view_sf2 = entity.spatial_filter(geometry=small_poly)
262
+ t1 = time.perf_counter()
263
+ print(f"spatial_filter (geom): {(t1 - t0) * 1000:.2f} ms")
264
+
265
+ # area relationship
266
+ t0 = time.perf_counter()
267
+ view_sf_area = entity.spatial_filter(
268
+ bbox=SMALL_BBOX, relationship="area", threshold=0.3
269
+ )
270
+ t1 = time.perf_counter()
271
+ print(
272
+ f"spatial_filter (area): {(t1 - t0) * 1000:.2f} ms keys={view_sf_area.keys.collect().shape}"
273
+ )
274
+
275
+
276
+ # ── 13. BaseEntity.spatial_join() — deferred spec ────────────────────────────
277
+ _section("13. BaseEntity.spatial_join() [deferred spec construction — no I/O]")
278
+ REPS_SJ = 10_000
279
+ t0 = time.perf_counter()
280
+ for _ in range(REPS_SJ):
281
+ entity.spatial_join(other=entity, agg={"area_in_ha": "sum"})
282
+ t1 = time.perf_counter()
283
+ print(
284
+ f"spatial_join (spec) ×{REPS_SJ}: {(t1 - t0) * 1000:.2f} ms total "
285
+ f"({(t1 - t0) / REPS_SJ * 1e6:.2f} µs/call)"
286
+ )
287
+
288
+
289
+ # ── 14. EntityValidationError — import / raise overhead ──────────────────────
290
+ _section("14. EntityValidationError [raise + catch overhead]")
291
+ REPS_EV = 100_000
292
+ t0 = time.perf_counter()
293
+ for _ in range(REPS_EV):
294
+ try:
295
+ raise EntityValidationError("bench test")
296
+ except EntityValidationError:
297
+ pass
298
+ t1 = time.perf_counter()
299
+ print(
300
+ f"raise+catch ×{REPS_EV}: {(t1 - t0) * 1000:.2f} ms total "
301
+ f"({(t1 - t0) / REPS_EV * 1e6:.2f} µs/call)"
302
+ )
303
+
304
+
305
+ # ── Cleanup ───────────────────────────────────────────────────────────────────
306
+ AoI.deregister(MWSEntity)
307
+ print("\n✓ bench_entity.py complete")
@@ -42,8 +42,8 @@ AoI.register(MWSEntity)
42
42
  aoi_all = AoI(DATA_ROOT, bbox=INDIA_BBOX)
43
43
  aoi_small = AoI(DATA_ROOT, bbox=SMALL_BBOX)
44
44
 
45
- result_all = aoi_all.mws.static # full dataset — geometry present
46
- result_small = aoi_small.mws.static # small subset
45
+ result_all = aoi_all.mws.static.materialise() # full dataset — geometry present
46
+ result_small = aoi_small.mws.static.materialise() # small subset
47
47
 
48
48
  print(f"All result : {result_all.df().shape}")
49
49
  print(f"Small result : {result_small.df().shape}")
@@ -205,6 +205,69 @@ with tempfile.TemporaryDirectory() as tmpdir:
205
205
  f"Ratio : geojson is {t_gj / t_gpq:.2f}× {'slower' if t_gj > t_gpq else 'faster'}"
206
206
  )
207
207
 
208
+ # ── 8. Export error paths — geometry guards ───────────────────────────────
209
+ _section("8. Export error paths [TypeError guards — no write, O(1)]")
210
+
211
+ errs = 0
212
+
213
+ # parquet() with geometry raises TypeError.
214
+ try:
215
+ parquet(result_small, tmp / "err.parquet")
216
+ except TypeError:
217
+ errs += 1
218
+
219
+ # json() with geometry raises TypeError.
220
+ try:
221
+ json(result_small, tmp / "err.json")
222
+ except TypeError:
223
+ errs += 1
224
+
225
+ # csv() with geometry raises TypeError.
226
+ try:
227
+ csv(result_small, tmp / "err.csv")
228
+ except TypeError:
229
+ errs += 1
230
+
231
+ # geoparquet() without geometry raises TypeError.
232
+ try:
233
+ geoparquet(result_no_geom, tmp / "err.geoparquet")
234
+ except TypeError:
235
+ errs += 1
236
+
237
+ # geojson() without geometry raises TypeError.
238
+ try:
239
+ geojson(result_no_geom, tmp / "err.geojson")
240
+ except TypeError:
241
+ errs += 1
242
+
243
+ print(f"TypeError guards raised : {errs}/5 (expect 5)")
244
+
245
+ # ── 9. GeoJSONSeq (newline-delimited GeoJSON) driver ─────────────────────
246
+ _section("9. geojson(driver='GeoJSONSeq') [streaming newline-delimited GeoJSON]")
247
+ out_gjseq = tmp / "bench.geojsonl"
248
+ t0 = time.perf_counter()
249
+ for _ in range(REPS_IO):
250
+ geojson(result_small, out_gjseq, driver="GeoJSONSeq")
251
+ t1 = time.perf_counter()
252
+ sz = out_gjseq.stat().st_size
253
+ print(
254
+ f"GeoJSONSeq (small) ×{REPS_IO}: {(t1 - t0) * 1000:.2f} ms total "
255
+ f"({(t1 - t0) / REPS_IO * 1000:.2f} ms/call) → {sz // 1024} KB"
256
+ )
257
+
258
+ # ── 10. Format comparison summary (small dataset) ─────────────────────────
259
+ _section("10. Format size comparison [small dataset]")
260
+ sizes = {
261
+ "parquet": (tmp / "bench.parquet").stat().st_size,
262
+ "json": (tmp / "bench.json").stat().st_size,
263
+ "csv": (tmp / "bench.csv").stat().st_size,
264
+ "geoparquet": (tmp / "bench.geoparquet").stat().st_size,
265
+ "geojson": (tmp / "bench.geojson").stat().st_size,
266
+ "geojsonseq": (tmp / "bench.geojsonl").stat().st_size,
267
+ }
268
+ for fmt, sz_bytes in sorted(sizes.items(), key=lambda x: x[1]):
269
+ print(f" {fmt:12} : {sz_bytes // 1024:6} KB")
270
+
208
271
 
209
272
  # ── Cleanup ───────────────────────────────────────────────────────────────────
210
273
  AoI.deregister(MWSEntity)
@@ -188,6 +188,34 @@ print(
188
188
  )
189
189
 
190
190
 
191
+ # ── 7. cached_read_schema() — cold vs cached ──────────────────────────────────
192
+ _section("7. cached_read_schema() [cold vs functools.cache warm]")
193
+ from core_lens.utils.polars_utils import cached_read_schema # noqa: E402
194
+ from core_lens.utils.polars_utils import _cached_schema_internal # noqa: E402
195
+
196
+ # Warm the cache first (may already be warm from module import chain).
197
+ cached_read_schema(static_path)
198
+
199
+ REPS_CS = 100_000
200
+ t0 = time.perf_counter()
201
+ for _ in range(REPS_CS):
202
+ cached_read_schema(static_path)
203
+ t1 = time.perf_counter()
204
+ print(
205
+ f"cached_read_schema (warm) ×{REPS_CS}: {(t1 - t0) * 1000:.2f} ms total "
206
+ f"({(t1 - t0) / REPS_CS * 1e6:.3f} µs/call)"
207
+ )
208
+ print(f"Schema columns: {list(cached_read_schema(static_path).names())[:5]} ...")
209
+
210
+ # Cold path: clear the underlying cache, measure one miss.
211
+ _cached_schema_internal.cache_clear()
212
+ t0 = time.perf_counter()
213
+ cached_read_schema(static_path)
214
+ t1 = time.perf_counter()
215
+ print(f"cached_read_schema (cold) : {(t1 - t0) * 1000:.2f} ms (Parquet footer read)")
216
+ print(f"Cache info after: {_cached_schema_internal.cache_info()}")
217
+
218
+
191
219
  # ── Cleanup ───────────────────────────────────────────────────────────────────
192
220
  AoI.deregister(MWSEntity)
193
221
  print("\n✓ bench_polars_utils.py complete")
@@ -4,10 +4,14 @@ Targets:
4
4
  - Result.df() zero-copy accessor
5
5
  - Result.gdf() WKB decode + GeoPandas construction
6
6
  - Result.lazy() LazyFrame wrapping
7
+ - Result.materialise() eager evaluate + re-wrap as in-memory lazy
7
8
  - Result.with_geometry() static-join to attach geometry to non-static result
8
9
  - Result.derive() with_columns expression append
9
10
  - Result.aggregate() group_by + agg (no temporal grouping)
10
11
  - Result.aggregate(by=...) temporal grouping (annual / sub_annual)
12
+ - Result.aggregate errors STATIC resolution guard, invalid by, sub_annual-only by
13
+ - Result.gdf errors TypeError when has_geometry=False
14
+ - Result.stats / Result.plot namespace accessor overhead
11
15
 
12
16
  Scalene focuses on:
13
17
  - shapely.wkb.loads() in gdf() — Python loop over WKB bytes
@@ -39,8 +43,8 @@ aoi = AoI(DATA_ROOT, bbox=INDIA_BBOX)
39
43
  aoi_small = AoI(DATA_ROOT, bbox=SMALL_BBOX)
40
44
 
41
45
  # Materialise once — all benchmarks below operate on this result.
42
- result = aoi.mws.static
43
- result_small = aoi_small.mws.static
46
+ result = aoi.mws.static.materialise()
47
+ result_small = aoi_small.mws.static.materialise()
44
48
  print(f"Dataset: {result.df().shape[0]} MWS, {result.df().shape[1]} columns")
45
49
 
46
50
 
@@ -168,6 +172,94 @@ print(f"aggregate(year) : {(t1 - t0) * 1000:.2f} ms")
168
172
  print(f"Shape : {agg_year.df().shape}")
169
173
 
170
174
 
171
- # ── Cleanup ───────────────────────────────────────────────────────────────
175
+ # ── 9. Result.materialise() — eager evaluate (≈ df + wrap) ────────────────────
176
+ _section("9. Result.materialise() [collect + re-wrap in-memory lazy]")
177
+ # Use result (all-India static) — data already in memory so measures wrap cost.
178
+ REPS_M = 100
179
+ t0 = time.perf_counter()
180
+ for _ in range(REPS_M):
181
+ result.materialise()
182
+ t1 = time.perf_counter()
183
+ print(
184
+ f"materialise (— mem) ×{REPS_M}: {(t1 - t0) * 1000:.2f} ms total "
185
+ f"({(t1 - t0) / REPS_M * 1000:.2f} ms/call)"
186
+ )
187
+
188
+
189
+ # ── 10. Result.aggregate(by='season') / by='month' / by='year_month' ─────────
190
+ _section("10. Result.aggregate() sub_annual groupings")
191
+ from core_lens.base.view import Season # noqa: E402
192
+
193
+ result_fn = aoi_small.mws.between(season=Season.KHARIF, year=(2018, 2022)).sub_annual
194
+
195
+ data_col = result_fn.columns[-1] # last column (data column)
196
+
197
+ for by_key in ("month", "season", "year_month", "season_year", "year"):
198
+ t0 = time.perf_counter()
199
+ agg_r = result_fn.aggregate(pl.first(data_col), by=by_key)
200
+ t1 = time.perf_counter()
201
+ print(
202
+ f"aggregate(by={by_key!r:12}) : {(t1 - t0) * 1000:.2f} ms shape={agg_r.df().shape}"
203
+ )
204
+
205
+
206
+ # ── 11. Result.aggregate() error paths ──────────────────────────────────
207
+ _section("11. Result.aggregate() error paths [guard branches — no I/O]")
208
+ errors_caught = 0
209
+
210
+ # 11a. aggregate on STATIC resolution is forbidden
211
+ try:
212
+ result.aggregate(pl.mean(result.columns[-1]))
213
+ except ValueError:
214
+ errors_caught += 1
215
+
216
+ # 11b. unknown by key
217
+ try:
218
+ result_annual.aggregate(pl.mean(result_annual.columns[-1]), by="quarter")
219
+ except ValueError:
220
+ errors_caught += 1
221
+
222
+ # 11c. sub_annual-only grouping on annual resolution
223
+ try:
224
+ result_annual.aggregate(pl.mean(result_annual.columns[-1]), by="season")
225
+ except ValueError:
226
+ errors_caught += 1
227
+
228
+ print(f"Error guards raised : {errors_caught}/3 (expect 3)")
229
+
230
+
231
+ # ── 12. Result.gdf() TypeError when has_geometry=False ───────────────────────
232
+ _section("12. Result.gdf() TypeError [no geometry guard]")
233
+ gdf_err = False
234
+ try:
235
+ result_annual.gdf() # annual result has no geometry
236
+ except TypeError:
237
+ gdf_err = True
238
+ print(f"gdf() TypeError guard: {gdf_err} (expect True)")
239
+
240
+
241
+ # ── 13. Result.stats / Result.plot namespace access ────────────────────────
242
+ _section("13. Result.stats / Result.plot [namespace accessor cost]")
243
+ REPS_NS = 50_000
244
+ t0 = time.perf_counter()
245
+ for _ in range(REPS_NS):
246
+ result_annual.stats
247
+ t1 = time.perf_counter()
248
+ print(
249
+ f"result.stats ×{REPS_NS}: {(t1 - t0) * 1000:.2f} ms total "
250
+ f"({(t1 - t0) / REPS_NS * 1e6:.2f} µs/call)"
251
+ )
252
+
253
+ t0 = time.perf_counter()
254
+ for _ in range(REPS_NS):
255
+ result_annual.plot
256
+ t1 = time.perf_counter()
257
+ print(
258
+ f"result.plot ×{REPS_NS}: {(t1 - t0) * 1000:.2f} ms total "
259
+ f"({(t1 - t0) / REPS_NS * 1e6:.2f} µs/call)"
260
+ )
261
+
262
+
263
+ # ── Cleanup ───────────────────────────────────────────────────────────────────
172
264
  AoI.deregister(MWSEntity)
173
265
  print("\n✓ bench_result.py complete")