core-lens 0.1.dev124__tar.gz → 0.1.dev129__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- core_lens-0.1.dev129/.github/ISSUE_TEMPLATE/blank-proposal.yaml +33 -0
- core_lens-0.1.dev129/.github/ISSUE_TEMPLATE/bug-report.yaml +63 -0
- core_lens-0.1.dev129/.github/ISSUE_TEMPLATE/feature-request.yaml +33 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/PKG-INFO +8 -1
- core_lens-0.1.dev129/benchmarks/README.md +60 -0
- core_lens-0.1.dev129/benchmarks/bench_aoi.py +172 -0
- core_lens-0.1.dev129/benchmarks/bench_export.py +212 -0
- core_lens-0.1.dev129/benchmarks/bench_polars_utils.py +194 -0
- core_lens-0.1.dev129/benchmarks/bench_result.py +174 -0
- core_lens-0.1.dev129/benchmarks/bench_schema.py +189 -0
- core_lens-0.1.dev129/benchmarks/bench_season.py +192 -0
- core_lens-0.1.dev129/benchmarks/bench_spatial.py +194 -0
- core_lens-0.1.dev129/benchmarks/bench_view.py +155 -0
- core_lens-0.1.dev129/benchmarks/run_all.sh +75 -0
- core_lens-0.1.dev129/hooks/mypy.sh +2 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/pyproject.toml +7 -2
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/_version.py +2 -2
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/aoi.py +50 -12
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/base/entity.py +58 -21
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/base/namespaces/stats.py +10 -4
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/base/result.py +2 -1
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/base/view.py +9 -1
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/schema/detection.py +28 -9
- core_lens-0.1.dev129/src/core_lens/utils/paths.py +101 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/utils/polars_utils.py +8 -2
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/utils/spatial.py +107 -41
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/tests/unit/test_aoi.py +1 -1
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/uv.lock +260 -165
- core_lens-0.1.dev124/hooks/mypy.sh +0 -2
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/.github/pull_request_template.md +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/.github/workflows/ci.yml +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/.github/workflows/gh-pages.yml +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/.github/workflows/pre-release.yml +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/.github/workflows/release.yml +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/.gitignore +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/.gitmessage +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/.pre-commit-config.yaml +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/.python-version +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/CONTRIBUTING.md +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/LICENSE +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/README.md +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/SKILLS.md +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/docs/Makefile +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/docs/make.bat +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/docs/source/concepts.md +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/docs/source/conf.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/docs/source/export.md +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/docs/source/index.rst +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/docs/source/intro.md +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/docs/source/logging.md +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/docs/source/plots.md +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/docs/source/plugins.md +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/docs/source/queries.md +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/docs/source/quickstart.md +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/docs/source/stats.md +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/examples/demo_mws.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/examples/demo_tehsil.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/hooks/no-parquet-outside-fixtures.sh +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/hooks/pytest.sh +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/__init__.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/__main__.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/base/__init__.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/base/namespaces/__init__.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/base/namespaces/plot.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/entities/__init__.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/entities/mws.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/entities/tehsil.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/entities/waterbody.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/export/__init__.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/export/formats.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/py.typed +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/schema/__init__.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/schema/profile.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/utils/__init__.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/src/core_lens/utils/season.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/tests/fixtures/generate_fixtures.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/tests/unit/conftest.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/tests/unit/test_entities.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/tests/unit/test_entity.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/tests/unit/test_export.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/tests/unit/test_main.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/tests/unit/test_plot.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/tests/unit/test_polars_utils.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/tests/unit/test_profile.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/tests/unit/test_result.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/tests/unit/test_schema_detection.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/tests/unit/test_schema_profile.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/tests/unit/test_season.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/tests/unit/test_season_config.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/tests/unit/test_spatial.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/tests/unit/test_stats.py +0 -0
- {core_lens-0.1.dev124 → core_lens-0.1.dev129}/tests/unit/test_view.py +0 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
name: Blank Proposal
|
|
2
|
+
description: Use this for all proposals.
|
|
3
|
+
title: "[Proposal]: "
|
|
4
|
+
labels: ["proposal"]
|
|
5
|
+
body:
|
|
6
|
+
- type: textarea
|
|
7
|
+
id: issue
|
|
8
|
+
attributes:
|
|
9
|
+
label: Is the proposal related to an issue you are facing?
|
|
10
|
+
description: If yes, then please give a clear and concise description of the problem.
|
|
11
|
+
validations:
|
|
12
|
+
required: true
|
|
13
|
+
- type: textarea
|
|
14
|
+
id: feature
|
|
15
|
+
attributes:
|
|
16
|
+
label: Describe the proposal you have in mind.
|
|
17
|
+
placeholder: Describe the proposal you are in detail.
|
|
18
|
+
validations:
|
|
19
|
+
required: true
|
|
20
|
+
- type: textarea
|
|
21
|
+
id: alts
|
|
22
|
+
attributes:
|
|
23
|
+
label: Have you considered any alternatives?
|
|
24
|
+
description: If so, please give a clear and concise description.
|
|
25
|
+
validations:
|
|
26
|
+
required: false
|
|
27
|
+
- type: textarea
|
|
28
|
+
id: additions
|
|
29
|
+
attributes:
|
|
30
|
+
label: Any additional context you want to share?
|
|
31
|
+
placeholder: Add any other context or screenshots about the proposal here.
|
|
32
|
+
validations:
|
|
33
|
+
required: false
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
name: Bug Report
|
|
2
|
+
description: File a bug report.
|
|
3
|
+
title: "[Bug]: "
|
|
4
|
+
labels: ["bug"]
|
|
5
|
+
body:
|
|
6
|
+
- type: textarea
|
|
7
|
+
id: what-happened
|
|
8
|
+
attributes:
|
|
9
|
+
label: Please describe issue you faced.
|
|
10
|
+
description: Also tell us, what did you expect to happen?
|
|
11
|
+
placeholder: Describe the issue in detail
|
|
12
|
+
validations:
|
|
13
|
+
required: true
|
|
14
|
+
- type: input
|
|
15
|
+
id: version
|
|
16
|
+
attributes:
|
|
17
|
+
label: Version
|
|
18
|
+
description: What is the version of library you are using?
|
|
19
|
+
placeholder: 0.0.1
|
|
20
|
+
validations:
|
|
21
|
+
required: true
|
|
22
|
+
- type: dropdown
|
|
23
|
+
id: python
|
|
24
|
+
attributes:
|
|
25
|
+
label: What is the python version you are using?
|
|
26
|
+
multiple: true
|
|
27
|
+
options:
|
|
28
|
+
- Python 3.11
|
|
29
|
+
- Python 3.12
|
|
30
|
+
- Python 3.13
|
|
31
|
+
- Python 3.14
|
|
32
|
+
- type: dropdown
|
|
33
|
+
id: package-manager
|
|
34
|
+
attributes:
|
|
35
|
+
label: Which package manager are you using?
|
|
36
|
+
description: If you are unsure select pip.
|
|
37
|
+
options:
|
|
38
|
+
- uv
|
|
39
|
+
- poetry
|
|
40
|
+
- conda
|
|
41
|
+
- pip
|
|
42
|
+
- pixi
|
|
43
|
+
- pipenv
|
|
44
|
+
- type: input
|
|
45
|
+
id: OS
|
|
46
|
+
attributes:
|
|
47
|
+
label: Which OS are you using?
|
|
48
|
+
description: OS Name and version
|
|
49
|
+
placeholder: Ubuntu 22.04
|
|
50
|
+
- type: textarea
|
|
51
|
+
id: logs
|
|
52
|
+
attributes:
|
|
53
|
+
label: Relevant log output
|
|
54
|
+
description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
|
|
55
|
+
render: shell
|
|
56
|
+
- type: upload
|
|
57
|
+
id: screenshots
|
|
58
|
+
attributes:
|
|
59
|
+
label: Upload screenshots
|
|
60
|
+
description: If applicable, add screenshots to help explain your problem.
|
|
61
|
+
validations:
|
|
62
|
+
required: false
|
|
63
|
+
accept: ".png,.jpg,.gif,.log,.txt,.zip"
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
name: Feature Request
|
|
2
|
+
description: File a feature request.
|
|
3
|
+
title: "[Feature]: "
|
|
4
|
+
labels: ["feature"]
|
|
5
|
+
body:
|
|
6
|
+
- type: textarea
|
|
7
|
+
id: issue
|
|
8
|
+
attributes:
|
|
9
|
+
label: Is the feature related to an issue you are facing?
|
|
10
|
+
description: If yes, then please give a clear and concise description of the problem.
|
|
11
|
+
validations:
|
|
12
|
+
required: true
|
|
13
|
+
- type: textarea
|
|
14
|
+
id: feature
|
|
15
|
+
attributes:
|
|
16
|
+
label: Describe the solution you have in mind.
|
|
17
|
+
placeholder: Describe the solution you are proposing in detail
|
|
18
|
+
validations:
|
|
19
|
+
required: true
|
|
20
|
+
- type: textarea
|
|
21
|
+
id: alts
|
|
22
|
+
attributes:
|
|
23
|
+
label: Have you considered any alternatives?
|
|
24
|
+
description: If so, please give a clear and concise description.
|
|
25
|
+
validations:
|
|
26
|
+
required: false
|
|
27
|
+
- type: textarea
|
|
28
|
+
id: additions
|
|
29
|
+
attributes:
|
|
30
|
+
label: Any additional context you want to share?
|
|
31
|
+
placeholder: Add any other context or screenshots about the feature request here.
|
|
32
|
+
validations:
|
|
33
|
+
required: false
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: core-lens
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.dev129
|
|
4
4
|
Summary: Query, analyse, and visualise CoreStack's microwatershed and Earth science data through a clean, composable Python API.
|
|
5
5
|
Project-URL: Homepage, https://github.com/ApoorvaKashyap/core-lens
|
|
6
6
|
Project-URL: Issues, https://github.com/ApoorvaKashyap/core-lens/issues
|
|
@@ -17,11 +17,13 @@ Requires-Python: >=3.13
|
|
|
17
17
|
Requires-Dist: core-lens[core]
|
|
18
18
|
Requires-Dist: core-lens[spatial]
|
|
19
19
|
Provides-Extra: core
|
|
20
|
+
Requires-Dist: awscrt>=0.23.0; extra == 'core'
|
|
20
21
|
Requires-Dist: loguru>=0.7.3; extra == 'core'
|
|
21
22
|
Requires-Dist: polars<2,>=1.39.0; extra == 'core'
|
|
22
23
|
Requires-Dist: pyarrow<25,>=23.0.0; extra == 'core'
|
|
23
24
|
Requires-Dist: pydantic<3,>=2.13.3; extra == 'core'
|
|
24
25
|
Provides-Extra: full
|
|
26
|
+
Requires-Dist: awscrt>=0.23.0; extra == 'full'
|
|
25
27
|
Requires-Dist: bokeh<4,>=3.6.0; extra == 'full'
|
|
26
28
|
Requires-Dist: duckdb<2,>=1.5.3; extra == 'full'
|
|
27
29
|
Requires-Dist: geoarrow-rust-core>=0.3.0; extra == 'full'
|
|
@@ -37,6 +39,7 @@ Requires-Dist: scipy<2,>=1.17.1; extra == 'full'
|
|
|
37
39
|
Requires-Dist: shapely<3,>=2.1.2; extra == 'full'
|
|
38
40
|
Requires-Dist: statsmodels<1,>=0.14.6; extra == 'full'
|
|
39
41
|
Provides-Extra: full-gpu
|
|
42
|
+
Requires-Dist: awscrt>=0.23.0; extra == 'full-gpu'
|
|
40
43
|
Requires-Dist: bokeh<4,>=3.6.0; extra == 'full-gpu'
|
|
41
44
|
Requires-Dist: cudf-cu13==26.6.*; extra == 'full-gpu'
|
|
42
45
|
Requires-Dist: cudf-polars-cu13==26.6.*; extra == 'full-gpu'
|
|
@@ -54,6 +57,7 @@ Requires-Dist: scipy<2,>=1.17.1; extra == 'full-gpu'
|
|
|
54
57
|
Requires-Dist: shapely<3,>=2.1.2; extra == 'full-gpu'
|
|
55
58
|
Requires-Dist: statsmodels<1,>=0.14.6; extra == 'full-gpu'
|
|
56
59
|
Provides-Extra: gpu
|
|
60
|
+
Requires-Dist: awscrt>=0.23.0; extra == 'gpu'
|
|
57
61
|
Requires-Dist: cudf-cu13==26.6.*; extra == 'gpu'
|
|
58
62
|
Requires-Dist: cudf-polars-cu13==26.6.*; extra == 'gpu'
|
|
59
63
|
Requires-Dist: loguru>=0.7.3; extra == 'gpu'
|
|
@@ -61,6 +65,7 @@ Requires-Dist: polars<2,>=1.39.0; extra == 'gpu'
|
|
|
61
65
|
Requires-Dist: pyarrow<25,>=23.0.0; extra == 'gpu'
|
|
62
66
|
Requires-Dist: pydantic<3,>=2.13.3; extra == 'gpu'
|
|
63
67
|
Provides-Extra: spatial
|
|
68
|
+
Requires-Dist: awscrt>=0.23.0; extra == 'spatial'
|
|
64
69
|
Requires-Dist: duckdb<2,>=1.5.3; extra == 'spatial'
|
|
65
70
|
Requires-Dist: geopandas<2,>=1.1.3; extra == 'spatial'
|
|
66
71
|
Requires-Dist: loguru>=0.7.3; extra == 'spatial'
|
|
@@ -70,6 +75,7 @@ Requires-Dist: pydantic<3,>=2.13.3; extra == 'spatial'
|
|
|
70
75
|
Requires-Dist: pyproj<4,>=3.7.2; extra == 'spatial'
|
|
71
76
|
Requires-Dist: shapely<3,>=2.1.2; extra == 'spatial'
|
|
72
77
|
Provides-Extra: stats
|
|
78
|
+
Requires-Dist: awscrt>=0.23.0; extra == 'stats'
|
|
73
79
|
Requires-Dist: loguru>=0.7.3; extra == 'stats'
|
|
74
80
|
Requires-Dist: polars<2,>=1.39.0; extra == 'stats'
|
|
75
81
|
Requires-Dist: pyarrow<25,>=23.0.0; extra == 'stats'
|
|
@@ -77,6 +83,7 @@ Requires-Dist: pydantic<3,>=2.13.3; extra == 'stats'
|
|
|
77
83
|
Requires-Dist: scipy<2,>=1.17.1; extra == 'stats'
|
|
78
84
|
Requires-Dist: statsmodels<1,>=0.14.6; extra == 'stats'
|
|
79
85
|
Provides-Extra: viz
|
|
86
|
+
Requires-Dist: awscrt>=0.23.0; extra == 'viz'
|
|
80
87
|
Requires-Dist: bokeh<4,>=3.6.0; extra == 'viz'
|
|
81
88
|
Requires-Dist: geoarrow-rust-core>=0.3.0; extra == 'viz'
|
|
82
89
|
Requires-Dist: loguru>=0.7.3; extra == 'viz'
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# core-lens Benchmarks
|
|
2
|
+
|
|
3
|
+
Focused Scalene profiling scripts — one per module/subsystem.
|
|
4
|
+
|
|
5
|
+
## Structure
|
|
6
|
+
|
|
7
|
+
```
|
|
8
|
+
benchmarks/
|
|
9
|
+
├── README.md
|
|
10
|
+
├── run_all.sh # Run all benchmarks in sequence
|
|
11
|
+
├── bench_aoi.py # AoI init, register, named-boundary resolution
|
|
12
|
+
├── bench_spatial.py # build_bbox_index, bbox_intersects_geometry, exact_spatial_filter
|
|
13
|
+
├── bench_view.py # View.where, spatial_filter, between, materialise (static/annual/fortnightly)
|
|
14
|
+
├── bench_result.py # Result.derive, aggregate, with_geometry, df/gdf
|
|
15
|
+
├── bench_season.py # resolve_time_filter, add_temporal_columns, SeasonConfig.season_for
|
|
16
|
+
├── bench_schema.py # schema detection (detect, _infer_* helpers)
|
|
17
|
+
├── bench_polars_utils.py # scan_with_key_filter, collect_lf (CPU path)
|
|
18
|
+
└── bench_export.py # parquet, json, csv, geoparquet, geojson
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
## Running
|
|
22
|
+
|
|
23
|
+
### Single benchmark
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
uv run scalene --html --outfile benchmarks/profiles/bench_spatial.html benchmarks/bench_spatial.py
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
### All benchmarks (sequential, HTML output)
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
chmod +x benchmarks/run_all.sh
|
|
33
|
+
./benchmarks/run_all.sh
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Profiles land in `benchmarks/profiles/`.
|
|
37
|
+
|
|
38
|
+
### Quick timing-only run (no Scalene overhead)
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
uv run python benchmarks/bench_spatial.py
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## What to look for in Scalene output
|
|
45
|
+
|
|
46
|
+
| Signal | Meaning |
|
|
47
|
+
|---|---|
|
|
48
|
+
| High **CPU %** on line | Compute bottleneck — vectorise or push into Polars expr |
|
|
49
|
+
| High **memory** spike | Possible copy-on-write or large intermediate frame |
|
|
50
|
+
| `[native]` vs `[Python]` | Native = C ext (fast). Python = pure Python (optimise) |
|
|
51
|
+
| **GPU %** column | cudf path hit or not |
|
|
52
|
+
|
|
53
|
+
## Prerequisites
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
uv sync --group dev
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Each script needs `DATA_ROOT` pointing at a directory with the expected entity Parquet files.
|
|
60
|
+
Edit the `DATA_ROOT` constant at the top of each file to match your local data path.
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"""
|
|
2
|
+
bench_aoi.py — Benchmark for core_lens.aoi
|
|
3
|
+
===========================================
|
|
4
|
+
Targets:
|
|
5
|
+
- AoI.register() entity registration + path probe
|
|
6
|
+
- AoI.__init__(bbox=) bbox boundary mode
|
|
7
|
+
- AoI.__init__(geometry=) raw Shapely geometry mode
|
|
8
|
+
- AoI.__init__(**kwargs) named-boundary resolution (Parquet scan + unary_union)
|
|
9
|
+
- AoI.__getattr__ lazy View creation + first spatial_filter call
|
|
10
|
+
- AoI.deregister() cleanup
|
|
11
|
+
|
|
12
|
+
Scalene focuses on:
|
|
13
|
+
- _validate_entity() schema read at init time
|
|
14
|
+
- _resolve_named_boundary() scan_parquet + shapely.from_wkb + unary_union
|
|
15
|
+
|
|
16
|
+
Usage:
|
|
17
|
+
uv run scalene run --html --outfile benchmarks/profiles/bench_aoi.html benchmarks/bench_aoi.py
|
|
18
|
+
uv run python benchmarks/bench_aoi.py # timing only
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import time
|
|
24
|
+
|
|
25
|
+
import shapely.geometry as sgeom
|
|
26
|
+
|
|
27
|
+
from core_lens import AoI, SeasonConfig
|
|
28
|
+
from core_lens.entities.mws import MWSEntity
|
|
29
|
+
|
|
30
|
+
# ── Config ────────────────────────────────────────────────────────────────────
|
|
31
|
+
DATA_ROOT = "data/"
|
|
32
|
+
|
|
33
|
+
# A mws_id known to exist in your dataset.
|
|
34
|
+
MWS_ID = "12_322881" # update to a valid mws_id in your data
|
|
35
|
+
|
|
36
|
+
# Bounding box covering all of India (WGS-84).
|
|
37
|
+
INDIA_BBOX: tuple[float, float, float, float] = (68.0, 6.0, 98.0, 38.0)
|
|
38
|
+
|
|
39
|
+
# A small bbox for the named-boundary AoI sanity check.
|
|
40
|
+
SMALL_BBOX: tuple[float, float, float, float] = (86.0, 21.0, 87.5, 22.0)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _section(title: str) -> None:
|
|
44
|
+
print(f"\n{'─' * 60}")
|
|
45
|
+
print(f" {title}")
|
|
46
|
+
print("─" * 60)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# ── 1. Registration ───────────────────────────────────────────────────────────
|
|
50
|
+
_section("1. AoI.register(MWSEntity)")
|
|
51
|
+
t0 = time.perf_counter()
|
|
52
|
+
|
|
53
|
+
AoI.register(MWSEntity)
|
|
54
|
+
|
|
55
|
+
t1 = time.perf_counter()
|
|
56
|
+
print(f"register() : {(t1 - t0) * 1000:.2f} ms")
|
|
57
|
+
print(f"Registered : {AoI.registered_entities()}")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
# ── 2. AoI init — bbox mode ───────────────────────────────────────────────────
|
|
61
|
+
_section("2. AoI.__init__(bbox=INDIA_BBOX) [validates all entities]")
|
|
62
|
+
t0 = time.perf_counter()
|
|
63
|
+
|
|
64
|
+
aoi_bbox = AoI(DATA_ROOT, bbox=INDIA_BBOX)
|
|
65
|
+
|
|
66
|
+
t1 = time.perf_counter()
|
|
67
|
+
print(f"AoI(bbox) : {(t1 - t0) * 1000:.2f} ms")
|
|
68
|
+
print(f"Geometry type : {type(aoi_bbox.geometry).__name__}")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
# ── 3. AoI init — raw geometry mode ──────────────────────────────────────────
|
|
72
|
+
_section("3. AoI.__init__(geometry=shapely_poly)")
|
|
73
|
+
poly = sgeom.box(*SMALL_BBOX)
|
|
74
|
+
t0 = time.perf_counter()
|
|
75
|
+
|
|
76
|
+
aoi_geom = AoI(DATA_ROOT, geometry=poly)
|
|
77
|
+
|
|
78
|
+
t1 = time.perf_counter()
|
|
79
|
+
print(f"AoI(geometry) : {(t1 - t0) * 1000:.2f} ms")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
# ── 4. AoI init — named boundary (most expensive) ────────────────────────────
|
|
83
|
+
_section(f"4. AoI.__init__(mws_id={MWS_ID!r})")
|
|
84
|
+
t0 = time.perf_counter()
|
|
85
|
+
|
|
86
|
+
aoi_named = AoI(DATA_ROOT, mws_id=MWS_ID)
|
|
87
|
+
|
|
88
|
+
t1 = time.perf_counter()
|
|
89
|
+
print(f"AoI(named) : {(t1 - t0) * 1000:.2f} ms")
|
|
90
|
+
print(f"Geometry type : {type(aoi_named.geometry).__name__}")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
# ── 5. AoI init — custom SeasonConfig ────────────────────────────────────────
|
|
94
|
+
_section("5. AoI.__init__ with custom SeasonConfig")
|
|
95
|
+
t0 = time.perf_counter()
|
|
96
|
+
|
|
97
|
+
aoi_season = AoI(
|
|
98
|
+
DATA_ROOT,
|
|
99
|
+
bbox=INDIA_BBOX,
|
|
100
|
+
seasons=SeasonConfig(
|
|
101
|
+
kharif=("06-15", "10-15"),
|
|
102
|
+
rabi=("10-16", "02-28"),
|
|
103
|
+
zaid=("03-01", "06-14"),
|
|
104
|
+
),
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
t1 = time.perf_counter()
|
|
108
|
+
print(f"AoI(seasons) : {(t1 - t0) * 1000:.2f} ms")
|
|
109
|
+
print(f"Current season : {aoi_season.current_season}")
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
# ── 6. View access via __getattr__ ───────────────────────────────────────────
|
|
113
|
+
_section("6. aoi.mws [first __getattr__ call — runs spatial_filter]")
|
|
114
|
+
t0 = time.perf_counter()
|
|
115
|
+
|
|
116
|
+
view = aoi_bbox.mws
|
|
117
|
+
|
|
118
|
+
t1 = time.perf_counter()
|
|
119
|
+
print(f"First access : {(t1 - t0) * 1000:.2f} ms")
|
|
120
|
+
print(f"Keys shape : {view.keys.shape}")
|
|
121
|
+
|
|
122
|
+
# Second access — should hit _scoped cache (near-zero cost).
|
|
123
|
+
t0 = time.perf_counter()
|
|
124
|
+
view2 = aoi_bbox.mws
|
|
125
|
+
t1 = time.perf_counter()
|
|
126
|
+
print(f"Cached access : {(t1 - t0) * 1000:.4f} ms (expect ~0 ms)")
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
# ── 7. current_season / current_year (property overhead) ─────────────────────
|
|
130
|
+
_section("7. AoI.current_season / current_year")
|
|
131
|
+
REPS = 10_000
|
|
132
|
+
t0 = time.perf_counter()
|
|
133
|
+
for _ in range(REPS):
|
|
134
|
+
aoi_bbox.current_season
|
|
135
|
+
t1 = time.perf_counter()
|
|
136
|
+
print(
|
|
137
|
+
f"current_season ×{REPS}: {(t1 - t0) * 1000:.2f} ms total "
|
|
138
|
+
f"({(t1 - t0) / REPS * 1e6:.2f} µs/call)"
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
t0 = time.perf_counter()
|
|
142
|
+
for _ in range(REPS):
|
|
143
|
+
aoi_bbox.current_year
|
|
144
|
+
t1 = time.perf_counter()
|
|
145
|
+
print(
|
|
146
|
+
f"current_year ×{REPS}: {(t1 - t0) * 1000:.2f} ms total "
|
|
147
|
+
f"({(t1 - t0) / REPS * 1e6:.2f} µs/call)"
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
# ── 8. registered_entities() ─────────────────────────────────────────────────
|
|
152
|
+
_section("8. AoI.registered_entities()")
|
|
153
|
+
REPS = 50_000
|
|
154
|
+
t0 = time.perf_counter()
|
|
155
|
+
for _ in range(REPS):
|
|
156
|
+
AoI.registered_entities()
|
|
157
|
+
t1 = time.perf_counter()
|
|
158
|
+
print(
|
|
159
|
+
f"registered_entities ×{REPS}: {(t1 - t0) * 1000:.2f} ms total "
|
|
160
|
+
f"({(t1 - t0) / REPS * 1e6:.2f} µs/call)"
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
# ── 9. deregister ─────────────────────────────────────────────────────────────
|
|
165
|
+
_section("9. AoI.deregister(MWSEntity)")
|
|
166
|
+
t0 = time.perf_counter()
|
|
167
|
+
AoI.deregister(MWSEntity)
|
|
168
|
+
t1 = time.perf_counter()
|
|
169
|
+
print(f"deregister() : {(t1 - t0) * 1000:.4f} ms")
|
|
170
|
+
print(f"Remaining : {AoI.registered_entities()}")
|
|
171
|
+
|
|
172
|
+
print("\n✓ bench_aoi.py complete")
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
"""
|
|
2
|
+
bench_export.py — Benchmark for core_lens.export.formats
|
|
3
|
+
=========================================================
|
|
4
|
+
Targets:
|
|
5
|
+
- export.parquet() Polars write_parquet (no geometry)
|
|
6
|
+
- export.json() Polars write_json (no geometry)
|
|
7
|
+
- export.csv() Polars write_csv (no geometry)
|
|
8
|
+
- export.geoparquet() DuckDB spatial COPY → Parquet (WKB → geometry)
|
|
9
|
+
- export.geojson() DuckDB spatial COPY → GeoJSON (WKB → GeoJSON)
|
|
10
|
+
- _get_duckdb_conn() singleton connection + INSTALL/LOAD spatial
|
|
11
|
+
|
|
12
|
+
Scalene focuses on:
|
|
13
|
+
- DuckDB spatial extension load overhead (first call)
|
|
14
|
+
- write_parquet / write_json / write_csv serialisation speed
|
|
15
|
+
- DuckDB ST_GeomFromWKB + COPY overhead vs Polars native writers
|
|
16
|
+
- Memory copies between Polars DataFrame → DuckDB
|
|
17
|
+
|
|
18
|
+
Usage:
|
|
19
|
+
uv run scalene run --html --outfile benchmarks/profiles/bench_export.html benchmarks/bench_export.py
|
|
20
|
+
uv run python benchmarks/bench_export.py
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import tempfile
|
|
26
|
+
import time
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
from core_lens import AoI
|
|
31
|
+
from core_lens.entities.mws import MWSEntity
|
|
32
|
+
from core_lens.export import csv, geoparquet, geojson, json, parquet
|
|
33
|
+
from core_lens.export.formats import _get_duckdb_conn
|
|
34
|
+
|
|
35
|
+
# ── Config ────────────────────────────────────────────────────────────────────
|
|
36
|
+
DATA_ROOT = "data/"
|
|
37
|
+
INDIA_BBOX: tuple[float, float, float, float] = (68.0, 6.0, 98.0, 38.0)
|
|
38
|
+
SMALL_BBOX: tuple[float, float, float, float] = (86.0, 21.0, 87.5, 22.0)
|
|
39
|
+
|
|
40
|
+
AoI.register(MWSEntity)
|
|
41
|
+
|
|
42
|
+
# Materialise two result sizes for comparison.
|
|
43
|
+
aoi_all = AoI(DATA_ROOT, bbox=INDIA_BBOX)
|
|
44
|
+
aoi_small = AoI(DATA_ROOT, bbox=SMALL_BBOX)
|
|
45
|
+
|
|
46
|
+
result_all = aoi_all.mws.static # full dataset — geometry present
|
|
47
|
+
result_small = aoi_small.mws.static # small subset
|
|
48
|
+
|
|
49
|
+
print(f"All result : {result_all.data.shape}")
|
|
50
|
+
print(f"Small result : {result_small.data.shape}")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _section(title: str) -> None:
|
|
54
|
+
print(f"\n{'─' * 60}")
|
|
55
|
+
print(f" {title}")
|
|
56
|
+
print("─" * 60)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
60
|
+
tmp = Path(tmpdir)
|
|
61
|
+
|
|
62
|
+
# ── 1. DuckDB singleton init (first call installs spatial extension) ──────
|
|
63
|
+
_section("1. _get_duckdb_conn() [INSTALL spatial; LOAD spatial — cold]")
|
|
64
|
+
t0 = time.perf_counter()
|
|
65
|
+
conn = _get_duckdb_conn()
|
|
66
|
+
t1 = time.perf_counter()
|
|
67
|
+
print(f"DuckDB init (cold) : {(t1 - t0) * 1000:.2f} ms")
|
|
68
|
+
|
|
69
|
+
REPS = 10_000
|
|
70
|
+
t0 = time.perf_counter()
|
|
71
|
+
for _ in range(REPS):
|
|
72
|
+
_ = _get_duckdb_conn()
|
|
73
|
+
t1 = time.perf_counter()
|
|
74
|
+
print(
|
|
75
|
+
f"_get_duckdb_conn ×{REPS}: {(t1 - t0) * 1000:.2f} ms total "
|
|
76
|
+
f"({(t1 - t0) / REPS * 1e6:.3f} µs/call) (cached)"
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
# ── 2. export.parquet() — no geometry (strip geom col first)
|
|
80
|
+
_section("2. export.parquet() [Polars write_parquet, no geometry]")
|
|
81
|
+
|
|
82
|
+
# Strip geometry (and nested bbox) to pass has_geometry=False and allow CSV export.
|
|
83
|
+
geom_col = result_small.entity.geometry_col
|
|
84
|
+
drop_cols = [geom_col]
|
|
85
|
+
if "bbox" in result_small.data.columns:
|
|
86
|
+
drop_cols.append("bbox")
|
|
87
|
+
df_no_geom = result_small.data.drop(drop_cols)
|
|
88
|
+
result_no_geom = result_small._replace(data=df_no_geom, has_geometry=False)
|
|
89
|
+
|
|
90
|
+
REPS_IO = 5
|
|
91
|
+
out = tmp / "bench.parquet"
|
|
92
|
+
t0 = time.perf_counter()
|
|
93
|
+
for _ in range(REPS_IO):
|
|
94
|
+
parquet(result_no_geom, out)
|
|
95
|
+
t1 = time.perf_counter()
|
|
96
|
+
sz = out.stat().st_size
|
|
97
|
+
print(
|
|
98
|
+
f"parquet (small) ×{REPS_IO}: {(t1 - t0) * 1000:.2f} ms total "
|
|
99
|
+
f"({(t1 - t0) / REPS_IO * 1000:.2f} ms/call) → {sz // 1024} KB"
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
# All dataset.
|
|
103
|
+
df_all_no_geom = result_all.data.drop(geom_col)
|
|
104
|
+
result_all_no_geom = result_all._replace(data=df_all_no_geom, has_geometry=False)
|
|
105
|
+
out_all = tmp / "bench_all.parquet"
|
|
106
|
+
t0 = time.perf_counter()
|
|
107
|
+
for _ in range(REPS_IO):
|
|
108
|
+
parquet(result_all_no_geom, out_all)
|
|
109
|
+
t1 = time.perf_counter()
|
|
110
|
+
sz_all = out_all.stat().st_size
|
|
111
|
+
print(
|
|
112
|
+
f"parquet (all) ×{REPS_IO}: {(t1 - t0) * 1000:.2f} ms total "
|
|
113
|
+
f"({(t1 - t0) / REPS_IO * 1000:.2f} ms/call) → {sz_all // 1024} KB"
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
# ── 3. export.json() ──────────────────────────────────────────────────────
|
|
117
|
+
_section("3. export.json() [Polars write_json]")
|
|
118
|
+
out_json = tmp / "bench.json"
|
|
119
|
+
t0 = time.perf_counter()
|
|
120
|
+
for _ in range(REPS_IO):
|
|
121
|
+
json(result_no_geom, out_json)
|
|
122
|
+
t1 = time.perf_counter()
|
|
123
|
+
sz = out_json.stat().st_size
|
|
124
|
+
print(
|
|
125
|
+
f"json (small) ×{REPS_IO}: {(t1 - t0) * 1000:.2f} ms total "
|
|
126
|
+
f"({(t1 - t0) / REPS_IO * 1000:.2f} ms/call) → {sz // 1024} KB"
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
# ── 4. export.csv() ───────────────────────────────────────────────────────
|
|
130
|
+
_section("4. export.csv() [Polars write_csv]")
|
|
131
|
+
out_csv = tmp / "bench.csv"
|
|
132
|
+
t0 = time.perf_counter()
|
|
133
|
+
for _ in range(REPS_IO):
|
|
134
|
+
csv(result_no_geom, out_csv)
|
|
135
|
+
t1 = time.perf_counter()
|
|
136
|
+
sz = out_csv.stat().st_size
|
|
137
|
+
print(
|
|
138
|
+
f"csv (small) ×{REPS_IO}: {(t1 - t0) * 1000:.2f} ms total "
|
|
139
|
+
f"({(t1 - t0) / REPS_IO * 1000:.2f} ms/call) → {sz // 1024} KB"
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
# ── 5. export.geoparquet() — DuckDB spatial (small) ──────────────────────
|
|
143
|
+
_section("5. export.geoparquet() [DuckDB ST_GeomFromWKB + COPY]")
|
|
144
|
+
out_gpq = tmp / "bench.geoparquet"
|
|
145
|
+
t0 = time.perf_counter()
|
|
146
|
+
for _ in range(REPS_IO):
|
|
147
|
+
geoparquet(result_small, out_gpq)
|
|
148
|
+
t1 = time.perf_counter()
|
|
149
|
+
sz = out_gpq.stat().st_size
|
|
150
|
+
print(
|
|
151
|
+
f"geoparquet (small) ×{REPS_IO}: {(t1 - t0) * 1000:.2f} ms total "
|
|
152
|
+
f"({(t1 - t0) / REPS_IO * 1000:.2f} ms/call) → {sz // 1024} KB"
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
out_gpq_all = tmp / "bench_all.geoparquet"
|
|
156
|
+
t0 = time.perf_counter()
|
|
157
|
+
geoparquet(result_all, out_gpq_all)
|
|
158
|
+
t1 = time.perf_counter()
|
|
159
|
+
sz_all = out_gpq_all.stat().st_size
|
|
160
|
+
print(
|
|
161
|
+
f"geoparquet (all) ×1: {(t1 - t0) * 1000:.2f} ms total "
|
|
162
|
+
f"({(t1 - t0) * 1000:.2f} ms/call) → {sz_all // 1024} KB"
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
# ── 6. export.geojson() — DuckDB spatial ──────────────────────────────────
|
|
166
|
+
_section("6. export.geojson() [DuckDB ST_GeomFromWKB + COPY GDAL GeoJSON]")
|
|
167
|
+
out_gj = tmp / "bench.geojson"
|
|
168
|
+
t0 = time.perf_counter()
|
|
169
|
+
for _ in range(REPS_IO):
|
|
170
|
+
geojson(result_small, out_gj)
|
|
171
|
+
t1 = time.perf_counter()
|
|
172
|
+
sz = out_gj.stat().st_size
|
|
173
|
+
print(
|
|
174
|
+
f"geojson (small) ×{REPS_IO}: {(t1 - t0) * 1000:.2f} ms total "
|
|
175
|
+
f"({(t1 - t0) / REPS_IO * 1000:.2f} ms/call) → {sz // 1024} KB"
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
out_gj_all = tmp / "bench_all.geojson"
|
|
179
|
+
t0 = time.perf_counter()
|
|
180
|
+
geojson(result_all, out_gj_all)
|
|
181
|
+
t1 = time.perf_counter()
|
|
182
|
+
sz_all = out_gj_all.stat().st_size
|
|
183
|
+
print(
|
|
184
|
+
f"geojson (all) ×1: {(t1 - t0) * 1000:.2f} ms total "
|
|
185
|
+
f"({(t1 - t0) * 1000:.2f} ms/call) → {sz_all // 1024} KB"
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
# ── 7. Head-to-head: geoparquet vs geojson ────────────────────────────────
|
|
189
|
+
_section("7. Head-to-head: geoparquet vs geojson (all dataset)")
|
|
190
|
+
out_a = tmp / "hth.geoparquet"
|
|
191
|
+
out_b = tmp / "hth.geojson"
|
|
192
|
+
|
|
193
|
+
t0 = time.perf_counter()
|
|
194
|
+
geoparquet(result_all, out_a)
|
|
195
|
+
t1 = time.perf_counter()
|
|
196
|
+
t_gpq = (t1 - t0) * 1000
|
|
197
|
+
|
|
198
|
+
t0 = time.perf_counter()
|
|
199
|
+
geojson(result_all, out_b)
|
|
200
|
+
t1 = time.perf_counter()
|
|
201
|
+
t_gj = (t1 - t0) * 1000
|
|
202
|
+
|
|
203
|
+
print(f"geoparquet : {t_gpq:.2f} ms ({out_a.stat().st_size // 1024} KB)")
|
|
204
|
+
print(f"geojson : {t_gj:.2f} ms ({out_b.stat().st_size // 1024} KB)")
|
|
205
|
+
print(
|
|
206
|
+
f"Ratio : geojson is {t_gj / t_gpq:.2f}× {'slower' if t_gj > t_gpq else 'faster'}"
|
|
207
|
+
)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
# ── Cleanup ───────────────────────────────────────────────────────────────────
|
|
211
|
+
AoI.deregister(MWSEntity)
|
|
212
|
+
print("\n✓ bench_export.py complete")
|