pycopdem 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
pycopdem-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Borevitz Lab, Australian National University, and contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,155 @@
1
+ Metadata-Version: 2.4
2
+ Name: pycopdem
3
+ Version: 0.1.0
4
+ Summary: Cached Copernicus 30 m DEM windows with on-read terrain derivatives — download once per chunk, never twice
5
+ Author: Borevitz Lab, Australian National University
6
+ Author-email: Yasar Adeel Ansari <u6737670@anu.edu.au>
7
+ License: MIT
8
+ Project-URL: Homepage, https://github.com/thestochasticman/pycopdem
9
+ Project-URL: Repository, https://github.com/thestochasticman/pycopdem
10
+ Project-URL: Issues, https://github.com/thestochasticman/pycopdem/issues
11
+ Keywords: dem,copernicus,elevation,terrain,topography,twi,geospatial
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Topic :: Scientific/Engineering :: GIS
20
+ Requires-Python: >=3.11
21
+ Description-Content-Type: text/markdown
22
+ License-File: LICENSE
23
+ Requires-Dist: attrs
24
+ Requires-Dist: typing_extensions
25
+ Requires-Dist: numpy
26
+ Requires-Dist: xarray
27
+ Requires-Dist: zarr
28
+ Requires-Dist: rasterio
29
+ Requires-Dist: affine
30
+ Requires-Dist: pysheds
31
+ Dynamic: license-file
32
+
33
+ # pycopdem
34
+
35
+ **Cached [Copernicus GLO-30](https://registry.opendata.aws/copernicus-dem/)
36
+ elevation with on-read terrain derivatives — download once per chunk,
37
+ never twice.** Every elevation pixel this machine ever downloads lands
38
+ in one sparse, chunk-indexed store on the DEM's native 1-arc-second
39
+ grid; slope, aspect, flow accumulation, TWI and Heat Load Index are
40
+ computed on read and never stored. Part of the
41
+ [Borevitz Lab](https://borevitzlab.anu.edu.au/) ecosystem.
42
+
43
+ ## How it works
44
+
45
+ ```
46
+ {data_root}/copdem_store/
47
+ ├── index.db # SQLite ledger: which chunks are populated
48
+ └── dem.zarr # global sparse array; only written 1200×1200-px chunks exist
49
+ ```
50
+
51
+ - The GLO-30 DEM is served as one COG per 1° × 1° cell on a 1-arc-second
52
+ EPSG:4326 lattice. The store uses the same lattice globally, chunked
53
+ at 1200 px (1/3°) — so **3 × 3 chunks nest exactly inside every
54
+ tile**, and fetching a chunk is a single integer-aligned windowed
55
+ read from one COG. No resampling, ever.
56
+ - Any bbox maps deterministically to a set of chunk ids.
57
+ `Store.get_ds(bbox)` diffs them against the ledger and downloads
58
+ **only the missing chunks**. Elevation is time-invariant, so there's
59
+ no time axis and no date bookkeeping.
60
+ - A 1° tile absent from S3 is genuinely all ocean: its chunks are
61
+ stored as nodata and marked complete.
62
+ - Derivatives are a read-time transform (like everything derived in
63
+ this ecosystem): request them per call, pay compute not disk.
64
+
65
+ ## Usage
66
+
67
+ The core API is **troi-agnostic** — just a bbox:
68
+
69
+ ```python
70
+ from pycopdem.store import Store
71
+
72
+ store = Store()
73
+ bbox = [148.36265, -33.52606, 148.38265, -33.50606] # [W, S, E, N]
74
+
75
+ ds = store.get_ds(bbox) # elevation (lat, lon)
76
+ ds = store.get_ds(bbox, derivatives=('slope', 'aspect', 'twi', 'hli'))
77
+
78
+ store.fill(bbox) # → 0: already local
79
+ ```
80
+
81
+ Derivatives: `slope` and `aspect` (degrees), `accumulation` (pysheds
82
+ fill-pits → fill-depressions → resolve-flats → flowdir → accumulation),
83
+ `twi` (ln(accumulation / tan slope)), `hli` (McCune & Keon 2002).
84
+ Dependencies resolve automatically — asking for `twi` computes slope
85
+ and accumulation internally.
86
+
87
+ Pipelines that speak the shared `troi.troi.Troi` use the
88
+ adapters (dates on the troi are ignored — elevation doesn't change):
89
+
90
+ ```python
91
+ ds = store.get_ds_troi(troi, derivatives=('slope',))
92
+ ```
93
+
94
+ `download_terrain(troi)` remains as a thin wrapper.
95
+
96
+ ## Performance
97
+
98
+ Live measurements against the Copernicus S3 bucket — a ~2 × 2 km AOI
99
+ (one *chunk* = 1200 × 1200 px ≈ 37 × 30 km):
100
+
101
+ | Scenario | Downloaded | Time |
102
+ |---|---|---|
103
+ | Cold fill | 1 chunk | 10.3 s |
104
+ | Same request again | nothing | **0.0 s** |
105
+ | AOI shifted ~28 km east | 1 new chunk | 4.2 s |
106
+ | Read cached window (1200²) | — | 0.4 s |
107
+ | Read + all 5 derivatives | — | 3.2 s (pysheds dominates) |
108
+
109
+ Store footprint: ~10 MB for two chunks (~2 200 km² of elevation).
110
+ Absolute times vary with network; the zero is the point — it's a
111
+ ledger lookup, no network involved.
112
+
113
+ ## Install
114
+
115
+ ### pip
116
+
117
+ ```bash
118
+ pip install git+https://github.com/thestochasticman/pycopdem.git
119
+ ```
120
+
121
+ Dependencies (the `troi` core included, pulled from GitHub) are
122
+ declared in `pyproject.toml` and installed automatically.
123
+
124
+ ### From source
125
+
126
+ ```bash
127
+ git clone https://github.com/thestochasticman/pycopdem.git
128
+ cd pycopdem
129
+ pip install -e .
130
+ ```
131
+
132
+ Package design (shared across the lab's packages — no inheritance,
133
+ composition only):
134
+
135
+ - **`Troi`** (from `troi`) — identity: what region.
136
+ - **`CopernicusDEM`** (`pycopdem.copdem`) — config: tile source.
137
+ - **`Paths`** (`pycopdem.paths`) — derived locations of the store for a
138
+ given `Config`.
139
+ - **`grid`** — the fixed 1-arc-second grid and tile/chunk nesting
140
+ (pure, offline-testable math).
141
+ - **`derive`** — the terrain derivatives (pure array math + pysheds).
142
+ - **`Store`** (`pycopdem.store`) — ties them together.
143
+
144
+ ## Test
145
+
146
+ ```bash
147
+ # offline (pure math + synthetic store):
148
+ python pycopdem/grid.py # True
149
+ python pycopdem/paths.py # True
150
+ python pycopdem/derive.py # True
151
+ python pycopdem/store.py # True
152
+
153
+ # live (small real reads from the Copernicus S3 bucket):
154
+ python pycopdem/download_terrain.py # True
155
+ ```
@@ -0,0 +1,123 @@
1
+ # pycopdem
2
+
3
+ **Cached [Copernicus GLO-30](https://registry.opendata.aws/copernicus-dem/)
4
+ elevation with on-read terrain derivatives — download once per chunk,
5
+ never twice.** Every elevation pixel this machine ever downloads lands
6
+ in one sparse, chunk-indexed store on the DEM's native 1-arc-second
7
+ grid; slope, aspect, flow accumulation, TWI and Heat Load Index are
8
+ computed on read and never stored. Part of the
9
+ [Borevitz Lab](https://borevitzlab.anu.edu.au/) ecosystem.
10
+
11
+ ## How it works
12
+
13
+ ```
14
+ {data_root}/copdem_store/
15
+ ├── index.db # SQLite ledger: which chunks are populated
16
+ └── dem.zarr # global sparse array; only written 1200×1200-px chunks exist
17
+ ```
18
+
19
+ - The GLO-30 DEM is served as one COG per 1° × 1° cell on a 1-arc-second
20
+ EPSG:4326 lattice. The store uses the same lattice globally, chunked
21
+ at 1200 px (1/3°) — so **3 × 3 chunks nest exactly inside every
22
+ tile**, and fetching a chunk is a single integer-aligned windowed
23
+ read from one COG. No resampling, ever.
24
+ - Any bbox maps deterministically to a set of chunk ids.
25
+ `Store.get_ds(bbox)` diffs them against the ledger and downloads
26
+ **only the missing chunks**. Elevation is time-invariant, so there's
27
+ no time axis and no date bookkeeping.
28
+ - A 1° tile absent from S3 is genuinely all ocean: its chunks are
29
+ stored as nodata and marked complete.
30
+ - Derivatives are a read-time transform (like everything derived in
31
+ this ecosystem): request them per call, pay compute not disk.
32
+
33
+ ## Usage
34
+
35
+ The core API is **troi-agnostic** — just a bbox:
36
+
37
+ ```python
38
+ from pycopdem.store import Store
39
+
40
+ store = Store()
41
+ bbox = [148.36265, -33.52606, 148.38265, -33.50606] # [W, S, E, N]
42
+
43
+ ds = store.get_ds(bbox) # elevation (lat, lon)
44
+ ds = store.get_ds(bbox, derivatives=('slope', 'aspect', 'twi', 'hli'))
45
+
46
+ store.fill(bbox) # → 0: already local
47
+ ```
48
+
49
+ Derivatives: `slope` and `aspect` (degrees), `accumulation` (pysheds
50
+ fill-pits → fill-depressions → resolve-flats → flowdir → accumulation),
51
+ `twi` (ln(accumulation / tan slope)), `hli` (McCune & Keon 2002).
52
+ Dependencies resolve automatically — asking for `twi` computes slope
53
+ and accumulation internally.
54
+
55
+ Pipelines that speak the shared `troi.troi.Troi` use the
56
+ adapters (dates on the troi are ignored — elevation doesn't change):
57
+
58
+ ```python
59
+ ds = store.get_ds_troi(troi, derivatives=('slope',))
60
+ ```
61
+
62
+ `download_terrain(troi)` remains as a thin wrapper.
63
+
64
+ ## Performance
65
+
66
+ Live measurements against the Copernicus S3 bucket — a ~2 × 2 km AOI
67
+ (one *chunk* = 1200 × 1200 px ≈ 37 × 30 km):
68
+
69
+ | Scenario | Downloaded | Time |
70
+ |---|---|---|
71
+ | Cold fill | 1 chunk | 10.3 s |
72
+ | Same request again | nothing | **0.0 s** |
73
+ | AOI shifted ~28 km east | 1 new chunk | 4.2 s |
74
+ | Read cached window (1200²) | — | 0.4 s |
75
+ | Read + all 5 derivatives | — | 3.2 s (pysheds dominates) |
76
+
77
+ Store footprint: ~10 MB for two chunks (~2 200 km² of elevation).
78
+ Absolute times vary with network; the zero is the point — it's a
79
+ ledger lookup, no network involved.
80
+
81
+ ## Install
82
+
83
+ ### pip
84
+
85
+ ```bash
86
+ pip install git+https://github.com/thestochasticman/pycopdem.git
87
+ ```
88
+
89
+ Dependencies (the `troi` core included, pulled from GitHub) are
90
+ declared in `pyproject.toml` and installed automatically.
91
+
92
+ ### From source
93
+
94
+ ```bash
95
+ git clone https://github.com/thestochasticman/pycopdem.git
96
+ cd pycopdem
97
+ pip install -e .
98
+ ```
99
+
100
+ Package design (shared across the lab's packages — no inheritance,
101
+ composition only):
102
+
103
+ - **`Troi`** (from `troi`) — identity: what region.
104
+ - **`CopernicusDEM`** (`pycopdem.copdem`) — config: tile source.
105
+ - **`Paths`** (`pycopdem.paths`) — derived locations of the store for a
106
+ given `Config`.
107
+ - **`grid`** — the fixed 1-arc-second grid and tile/chunk nesting
108
+ (pure, offline-testable math).
109
+ - **`derive`** — the terrain derivatives (pure array math + pysheds).
110
+ - **`Store`** (`pycopdem.store`) — ties them together.
111
+
112
+ ## Test
113
+
114
+ ```bash
115
+ # offline (pure math + synthetic store):
116
+ python pycopdem/grid.py # True
117
+ python pycopdem/paths.py # True
118
+ python pycopdem/derive.py # True
119
+ python pycopdem/store.py # True
120
+
121
+ # live (small real reads from the Copernicus S3 bucket):
122
+ python pycopdem/download_terrain.py # True
123
+ ```
@@ -0,0 +1,11 @@
1
+ # Light-weight exports only: pycopdem.store (and the download_terrain
2
+ # wrapper) pull in rasterio/zarr/pysheds, so those stay behind explicit
3
+ # submodule imports.
4
+ from pycopdem.copdem import CopernicusDEM, defaultcopdem
5
+ from pycopdem.paths import Paths
6
+
7
+ __all__ = [
8
+ 'CopernicusDEM',
9
+ 'defaultcopdem',
10
+ 'Paths',
11
+ ]
@@ -0,0 +1,18 @@
1
+ from attrs import frozen
2
+
3
+ @frozen
4
+ class CopernicusDEM:
5
+ """Endpoint configuration for Copernicus GLO-30 DEM tiles on AWS.
6
+
7
+ The DEM is global, 30 m (1 arc-second), served as public
8
+ Cloud-Optimised GeoTIFFs — one per 1° x 1° cell. Tiles that would
9
+ be entirely ocean do not exist; the store treats a missing tile as
10
+ all-nodata.
11
+ """
12
+
13
+ base_url: str = 'https://copernicus-dem-30m.s3.amazonaws.com'
14
+
15
+ def tile_url(self, tile: str) -> str:
16
+ return f'{self.base_url}/{tile}/{tile}.tif'
17
+
18
+ defaultcopdem = CopernicusDEM()
@@ -0,0 +1,121 @@
1
+ """Topographic derivatives, computed on read — never stored.
2
+
3
+ Slope, aspect, and Heat Load Index are cheap array math; flow
4
+ accumulation and TWI run the pysheds conditioning chain (fill pits →
5
+ fill depressions → resolve flats → flow direction → accumulation).
6
+ Everything operates on an elevation window plus its pixel size, so the
7
+ functions stay pure of store concerns.
8
+ """
9
+ import numpy as np
10
+
11
+ # pysheds 0.5 calls np.in1d which was removed in NumPy 2.0. Alias to np.isin
12
+ # (the documented replacement, same semantics) before pysheds is imported.
13
+ if not hasattr(np, 'in1d'):
14
+ np.in1d = np.isin
15
+
16
+
17
+ def slope(dem: np.ndarray, xres: float, yres: float) -> np.ndarray:
18
+ """Slope in degrees from an elevation array and pixel sizes (metres)."""
19
+ gy, gx = np.gradient(dem, yres, xres)
20
+ return np.arctan(np.sqrt(gx ** 2 + gy ** 2)) * (180 / np.pi)
21
+
22
+
23
+ def aspect(dem: np.ndarray, xres: float, yres: float) -> np.ndarray:
24
+ """Aspect (direction of steepest descent) in degrees, 0/360=N 90=E 180=S 270=W."""
25
+ gy, gx = np.gradient(dem, yres, xres)
26
+ deg = np.degrees(np.arctan2(-gx, gy))
27
+ return np.where(deg < 0, deg + 360, deg)
28
+
29
+
30
+ def accumulation(dem: np.ndarray, transform, nodata: float = -9999.0) -> np.ndarray:
31
+ """Flow accumulation (cells) via pysheds on an in-memory window.
32
+
33
+ The window is written to a temporary GeoTIFF because pysheds'
34
+ conditioning chain reads from raster files.
35
+ """
36
+ import os
37
+ import tempfile
38
+ import rasterio
39
+ from pysheds.grid import Grid
40
+
41
+ filled = np.where(np.isnan(dem), nodata, dem).astype('float32')
42
+ fd, path = tempfile.mkstemp(suffix='.tif')
43
+ os.close(fd)
44
+ try:
45
+ with rasterio.open(
46
+ path, 'w', driver='GTiff', height=dem.shape[0], width=dem.shape[1],
47
+ count=1, dtype='float32', transform=transform, crs='EPSG:4326',
48
+ nodata=nodata,
49
+ ) as dst:
50
+ dst.write(filled, 1)
51
+ grid = Grid.from_raster(path, nodata=nodata)
52
+ raster = grid.read_raster(path, nodata=nodata)
53
+ inflated = grid.resolve_flats(grid.fill_depressions(grid.fill_pits(raster)))
54
+ fdir = grid.flowdir(inflated, nodata_out=0)
55
+ return np.asarray(grid.accumulation(fdir, nodata_out=0))
56
+ finally:
57
+ os.remove(path)
58
+
59
+
60
+ def twi(acc: np.ndarray, slope_deg: np.ndarray) -> np.ndarray:
61
+ """Topographic wetness index: ln(accumulation / tan(slope))."""
62
+ ratio = acc / np.tan(np.radians(slope_deg))
63
+ ratio[ratio <= 0] = 1
64
+ return np.log(ratio)
65
+
66
+
67
+ def hli(slope_deg: np.ndarray, aspect_deg: np.ndarray, latitude: float) -> np.ndarray:
68
+ """Heat Load Index (McCune & Keon 2002), ~0-1; higher = more insolation."""
69
+ slope_rad = np.radians(slope_deg)
70
+ lat_rad = np.radians(latitude)
71
+ folded = np.radians(np.abs(180 - np.abs(aspect_deg - 225)))
72
+ out = np.exp(
73
+ -1.467
74
+ + 1.582 * np.cos(lat_rad) * np.cos(slope_rad)
75
+ - 1.5 * np.cos(folded) * np.sin(slope_rad) * np.sin(lat_rad)
76
+ - 0.262 * np.sin(lat_rad) * np.sin(slope_rad)
77
+ + 0.607 * np.sin(folded) * np.sin(slope_rad)
78
+ )
79
+ return np.clip(out, 0, 1)
80
+
81
+
82
+ def test_slope_of_inclined_plane():
83
+ """A plane rising 1 m per metre east has slope 45° everywhere inside."""
84
+ x = np.arange(50, dtype='float64')
85
+ dem = np.tile(x, (50, 1)) # elevation == x coordinate (1 m pixels)
86
+ s = slope(dem, 1.0, 1.0)
87
+ return abs(float(s[25, 25]) - 45.0) < 1e-6
88
+
89
+
90
+ def test_aspect_points_downhill():
91
+ """Rising to the east -> steepest descent faces west (270°)."""
92
+ x = np.arange(50, dtype='float64')
93
+ dem = np.tile(x, (50, 1))
94
+ a = aspect(dem, 1.0, 1.0)
95
+ return abs(float(a[25, 25]) - 270.0) < 1e-6
96
+
97
+
98
+ def test_flat_ground_hli_matches_formula():
99
+ h = hli(np.zeros((5, 5)), np.zeros((5, 5)), -33.5)
100
+ expected = np.exp(-1.467 + 1.582 * np.cos(np.radians(-33.5)))
101
+ return abs(float(h[2, 2]) - min(expected, 1.0)) < 1e-9
102
+
103
+
104
+ def test_twi_monotonic_in_accumulation():
105
+ s = np.full((3, 3), 10.0)
106
+ low = twi(np.full((3, 3), 10.0), s)
107
+ high = twi(np.full((3, 3), 1000.0), s)
108
+ return float(high[1, 1]) > float(low[1, 1])
109
+
110
+
111
+ def test():
112
+ return all([
113
+ test_slope_of_inclined_plane(),
114
+ test_aspect_points_downhill(),
115
+ test_flat_ground_hli_matches_formula(),
116
+ test_twi_monotonic_in_accumulation(),
117
+ ])
118
+
119
+
120
+ if __name__ == '__main__':
121
+ print(test())
@@ -0,0 +1,78 @@
1
+ """Fetch the elevation window (+ derivatives) for a troi — via the
2
+ machine-wide store.
3
+
4
+ Thin compatibility wrapper: the heavy lifting (chunk-level dedup,
5
+ windowed COG reads, on-read derivatives) lives in
6
+ :class:`pycopdem.store.Store`. Kept as a module so the familiar
7
+ ``download_terrain(troi)`` entry point survives.
8
+ """
9
+ import xarray as xr
10
+ from troi import Troi
11
+ from pycopdem.copdem import CopernicusDEM, defaultcopdem
12
+
13
+
14
+ def download_terrain(troi: Troi, derivatives: tuple[str, ...] = (),
15
+ copdem: CopernicusDEM = defaultcopdem) -> xr.Dataset:
16
+ """Return the Copernicus 30 m elevation window for ``troi.bbox``.
17
+
18
+ Fetches only the grid chunks no previous request has populated —
19
+ repeat and overlapping queries re-download nothing. Dates on the
20
+ troi are ignored: elevation is time-invariant.
21
+
22
+ Args:
23
+ troi: The :class:`troi.Troi` (bbox is what matters).
24
+ derivatives: Any of ``'slope'``, ``'aspect'``, ``'accumulation'``,
25
+ ``'twi'``, ``'hli'`` — computed on read, never stored.
26
+ copdem: Tile-source configuration; defaults to Copernicus on AWS.
27
+
28
+ Returns:
29
+ xarray.Dataset with dims ``(lat, lon)``: ``elevation`` plus one
30
+ variable per requested derivative.
31
+ """
32
+ from pycopdem.store import Store
33
+ store = Store(config=troi.config, copdem=copdem)
34
+ return store.get_ds_troi(troi, derivatives=derivatives)
35
+
36
+
37
+ def test_live_fetch_and_dedup():
38
+ """Live: cold fetch covers the bbox; repeat and overlapping bboxes
39
+ fetch nothing; derivatives come back finite."""
40
+ import numpy as np
41
+ import tempfile
42
+ from troi import Config
43
+ from pycopdem.store import Store
44
+
45
+ tmpdir = tempfile.mkdtemp(prefix='pycopdem_live_test_')
46
+ cfg = Config(out_dir=tmpdir, tmp_dir=tmpdir)
47
+ store = Store(config=cfg)
48
+ bbox = [148.36265, -33.52606, 148.38265, -33.50606]
49
+
50
+ fetched = store.fill(bbox)
51
+ if fetched < 1:
52
+ return False
53
+ ds = store.get_ds(bbox)
54
+ elev = ds['elevation'].values
55
+ if not np.isfinite(elev).any() or float(np.nanmax(elev)) <= 0:
56
+ return False
57
+ # identical repeat -> nothing
58
+ if store.fill(bbox) != 0:
59
+ return False
60
+ # overlapping bbox shifted ~2 km -> shares chunks, fetches at most the difference
61
+ shifted = [bbox[0] + 0.02, bbox[1], bbox[2] + 0.02, bbox[3]]
62
+ if store.fill(shifted) != 0: # within the same 1/3-degree chunks here
63
+ return False
64
+ # derivatives computed on read
65
+ dsd = store.get_ds(bbox, derivatives=('slope', 'aspect', 'twi', 'hli'))
66
+ return (
67
+ float(dsd['slope'].max()) >= 0
68
+ and np.isfinite(dsd['twi'].values).any()
69
+ and np.isfinite(dsd['hli'].values).any()
70
+ )
71
+
72
+
73
+ def test():
74
+ return test_live_fetch_and_dedup()
75
+
76
+
77
+ if __name__ == '__main__':
78
+ print(test())
@@ -0,0 +1,135 @@
1
+ """The fixed global grid every stored elevation pixel lives on.
2
+
3
+ The Copernicus GLO-30 DEM is served as 1° x 1° COG tiles on a
4
+ 1-arc-second (1/3600°) EPSG:4326 lattice anchored at integer degrees.
5
+ This module defines the same lattice globally, chunked at 1200 x 1200
6
+ pixels (1/3°) — chosen so exactly 3 x 3 chunks nest inside every 1°
7
+ tile, which makes each chunk fetchable with a single integer-aligned
8
+ windowed read from one COG. Any bbox maps deterministically to a set
9
+ of chunk ids, which is what makes the store dedup-able: overlapping
10
+ AOIs resolve to overlapping chunk sets, and a chunk is only ever
11
+ downloaded once.
12
+
13
+ All functions here are pure — no I/O, no store access.
14
+ """
15
+
16
+ PX_PER_DEG = 3600 # 1 arc-second pixels
17
+ CHUNK = 1200 # pixels per chunk edge (1/3 degree)
18
+ LON0, LAT_TOP = -180.0, 90.0 # grid origin: top-left corner
19
+ WIDTH_PX = 360 * PX_PER_DEG
20
+ HEIGHT_PX = 180 * PX_PER_DEG
21
+ RES = 1.0 / PX_PER_DEG
22
+
23
+
24
+ def window_for_bbox(bbox: list[float]) -> tuple[int, int, int, int]:
25
+ """Pixel window ``(row0, row1, col0, col1)`` covering ``bbox``, snapped
26
+ outward to whole chunks. Rows increase southward (row 0 at +90°)."""
27
+ west, south, east, north = bbox
28
+ col0 = int((west - LON0) * PX_PER_DEG // CHUNK) * CHUNK
29
+ col1 = -int(-((east - LON0) * PX_PER_DEG) // CHUNK) * CHUNK
30
+ row0 = int((LAT_TOP - north) * PX_PER_DEG // CHUNK) * CHUNK
31
+ row1 = -int(-((LAT_TOP - south) * PX_PER_DEG) // CHUNK) * CHUNK
32
+ return (row0, row1, col0, col1)
33
+
34
+
35
+ def chunks_in_window(window: tuple[int, int, int, int]) -> list[tuple[int, int]]:
36
+ """All chunk ids ``(cy, cx)`` inside a chunk-aligned pixel window."""
37
+ row0, row1, col0, col1 = window
38
+ return [
39
+ (cy, cx)
40
+ for cy in range(row0 // CHUNK, row1 // CHUNK)
41
+ for cx in range(col0 // CHUNK, col1 // CHUNK)
42
+ ]
43
+
44
+
45
+ def chunk_bounds(cy: int, cx: int) -> tuple[float, float, float, float]:
46
+ """EPSG:4326 ``(west, south, east, north)`` bounds of a chunk."""
47
+ west = LON0 + cx * CHUNK * RES
48
+ north = LAT_TOP - cy * CHUNK * RES
49
+ return (west, north - CHUNK * RES, west + CHUNK * RES, north)
50
+
51
+
52
+ def tile_of_chunk(cy: int, cx: int) -> tuple[int, int]:
53
+ """The 1° DEM tile ``(lat, lon)`` (south-west corner, integer degrees)
54
+ containing this chunk. Chunks nest exactly: one chunk, one tile."""
55
+ west, south, _, _ = chunk_bounds(cy, cx)
56
+ import math
57
+ return (math.floor(south), math.floor(west))
58
+
59
+
60
+ def tile_name(lat: int, lon: int) -> str:
61
+ """Copernicus GLO-30 tile stem for the 1° cell with SW corner (lat, lon)."""
62
+ ns = f'S{abs(lat):02d}_00' if lat < 0 else f'N{lat:02d}_00'
63
+ ew = f'W{abs(lon):03d}_00' if lon < 0 else f'E{lon:03d}_00'
64
+ return f'Copernicus_DSM_COG_10_{ns}_{ew}_DEM'
65
+
66
+
67
+ def chunk_window_in_tile(cy: int, cx: int) -> tuple[int, int, int, int]:
68
+ """Integer pixel window ``(row0, row1, col0, col1)`` of this chunk
69
+ inside its containing tile's 3600 x 3600 raster."""
70
+ r0 = (cy * CHUNK) % PX_PER_DEG
71
+ c0 = (cx * CHUNK) % PX_PER_DEG
72
+ return (r0, r0 + CHUNK, c0, c0 + CHUNK)
73
+
74
+
75
+ def coords_for_window(window: tuple[int, int, int, int]):
76
+ """Pixel-centre coordinate arrays ``(lat, lon)`` for a window
77
+ (lat descending)."""
78
+ import numpy as np
79
+ row0, row1, col0, col1 = window
80
+ lon = LON0 + (np.arange(col0, col1) + 0.5) * RES
81
+ lat = LAT_TOP - (np.arange(row0, row1) + 0.5) * RES
82
+ return lat, lon
83
+
84
+
85
+ _BBOX = [148.36265, -33.52606, 148.38265, -33.50606]
86
+
87
+
88
+ def test_window_is_chunk_aligned():
89
+ w = window_for_bbox(_BBOX)
90
+ return all(v % CHUNK == 0 for v in w) and w[1] > w[0] and w[3] > w[2]
91
+
92
+
93
+ def test_chunks_nest_in_tiles():
94
+ """Every chunk's pixel window inside its tile must be within [0, 3600]
95
+ and aligned to CHUNK."""
96
+ for cy, cx in chunks_in_window(window_for_bbox(_BBOX)):
97
+ r0, r1, c0, c1 = chunk_window_in_tile(cy, cx)
98
+ if not (0 <= r0 < r1 <= PX_PER_DEG and 0 <= c0 < c1 <= PX_PER_DEG):
99
+ return False
100
+ if r0 % CHUNK or c0 % CHUNK:
101
+ return False
102
+ return True
103
+
104
+
105
+ def test_tile_naming():
106
+ return tile_name(-34, 148) == 'Copernicus_DSM_COG_10_S34_00_E148_00_DEM'
107
+
108
+
109
+ def test_chunk_bounds_inside_tile():
110
+ for cy, cx in chunks_in_window(window_for_bbox(_BBOX)):
111
+ tlat, tlon = tile_of_chunk(cy, cx)
112
+ w, s, e, n = chunk_bounds(cy, cx)
113
+ if not (tlon <= w and e <= tlon + 1 and tlat <= s and n <= tlat + 1):
114
+ return False
115
+ return True
116
+
117
+
118
+ def test_overlapping_bboxes_share_chunks():
119
+ a = window_for_bbox(_BBOX)
120
+ b = window_for_bbox([_BBOX[0] + 0.05, _BBOX[1], _BBOX[2] + 0.05, _BBOX[3]])
121
+ return len(set(chunks_in_window(a)) & set(chunks_in_window(b))) > 0
122
+
123
+
124
+ def test():
125
+ return all([
126
+ test_window_is_chunk_aligned(),
127
+ test_chunks_nest_in_tiles(),
128
+ test_tile_naming(),
129
+ test_chunk_bounds_inside_tile(),
130
+ test_overlapping_bboxes_share_chunks(),
131
+ ])
132
+
133
+
134
+ if __name__ == '__main__':
135
+ print(test())
@@ -0,0 +1,58 @@
1
+ """Derived on-disk locations of the machine-wide elevation store.
2
+
3
+ The store is keyed by :class:`troi.Config` (one store per
4
+ data root, shared by every request on this machine). Rule of thumb
5
+ across the lab's packages: user-settable inputs → Config, derived
6
+ locations → Paths. No inheritance — composition only.
7
+ """
8
+ from attrs import frozen, field
9
+ from troi import Config, config as default_config
10
+
11
+
12
+ @frozen
13
+ class Paths:
14
+ """Where the pycopdem store lives for a given Config.
15
+
16
+ Attributes:
17
+ config: The :class:`troi.Config` supplying the data root.
18
+ root: Store directory (``{config.tmp_dir}/copdem_store``).
19
+ store: The sparse Zarr store holding every downloaded elevation chunk.
20
+ index_db: SQLite ledger of populated chunks.
21
+
22
+ Example:
23
+ ```python
24
+ from pycopdem.paths import Paths
25
+
26
+ Paths().store # '~/Downloads/Troi-Tmp/copdem_store/dem.zarr'
27
+ ```
28
+ """
29
+
30
+ config: Config = default_config
31
+
32
+ root: str = field(init=False)
33
+ store: str = field(init=False)
34
+ index_db: str = field(init=False)
35
+
36
+ root.default(lambda s: f'{s.config.tmp_dir}/copdem_store')
37
+ store.default(lambda s: f'{s.root}/dem.zarr')
38
+ index_db.default(lambda s: f'{s.root}/index.db')
39
+
40
+
41
+ def test_paths_derive_from_config():
42
+ import tempfile
43
+ tmpdir = tempfile.mkdtemp(prefix='pycopdem_paths_test_')
44
+ cfg = Config(out_dir=tmpdir, tmp_dir=tmpdir)
45
+ paths = Paths(cfg)
46
+ return (
47
+ paths.root == f'{tmpdir}/copdem_store'
48
+ and paths.store == f'{tmpdir}/copdem_store/dem.zarr'
49
+ and paths.index_db == f'{tmpdir}/copdem_store/index.db'
50
+ )
51
+
52
+
53
+ def test():
54
+ return test_paths_derive_from_config()
55
+
56
+
57
+ if __name__ == '__main__':
58
+ print(test())
@@ -0,0 +1,313 @@
1
+ """One machine-wide elevation store that fills itself on demand.
2
+
3
+ Every elevation pixel this machine ever downloads lands in a single
4
+ sparse Zarr array on the fixed 1-arc-second EPSG:4326 grid
5
+ (:mod:`pycopdem.grid`):
6
+
7
+ {config.tmp_dir}/copdem_store/
8
+ ├── index.db # SQLite ledger: which chunks are populated
9
+ └── dem.zarr # global sparse array; only written chunks exist on disk
10
+
11
+ Elevation is time-invariant, so there is no time axis and no date
12
+ bookkeeping — the ledger is just the set of populated ``(cy, cx)``
13
+ chunks. ``Store.get_ds(bbox)`` diffs the requested chunks against the
14
+ ledger, fetches only the missing ones (each chunk is a single
15
+ integer-aligned windowed read from one Copernicus COG tile), then
16
+ reads the window. Derivatives — slope, aspect, flow accumulation,
17
+ TWI, HLI — are computed on read (:mod:`pycopdem.derive`), never stored.
18
+ A 1° tile that does not exist on S3 is genuinely all ocean; its chunks
19
+ are stored as nodata and marked complete.
20
+ """
21
+ import sqlite3
22
+ from attrs import frozen, field
23
+ from datetime import datetime, timezone
24
+ from os import makedirs
25
+
26
+ import numpy as np
27
+ import xarray as xr
28
+ import zarr
29
+
30
+ from troi import Config, config as default_config
31
+ from pycopdem import derive, grid
32
+ from pycopdem.copdem import CopernicusDEM, defaultcopdem
33
+ from pycopdem.paths import Paths
34
+
35
+ _SCHEMA = """
36
+ CREATE TABLE IF NOT EXISTS chunks (
37
+ cy INTEGER NOT NULL,
38
+ cx INTEGER NOT NULL,
39
+ written_at TEXT NOT NULL,
40
+ PRIMARY KEY (cy, cx)
41
+ ) WITHOUT ROWID;
42
+ """
43
+
44
+ DERIVATIVES = ('slope', 'aspect', 'accumulation', 'twi', 'hli')
45
+
46
+ # Metres per degree of latitude (per degree of longitude scales by cos(lat)).
47
+ _M_PER_DEG = 111_320.0
48
+
49
+
50
+ @frozen
51
+ class Store:
52
+ """The machine-wide elevation store: one grid, one ledger, zero re-downloads.
53
+
54
+ Composed from :class:`troi.Config` (where the store
55
+ lives) and :class:`pycopdem.copdem.CopernicusDEM` (where tiles come
56
+ from). No inheritance.
57
+
58
+ Example:
59
+ ```python
60
+ from pycopdem.store import Store
61
+
62
+ store = Store()
63
+ ds = store.get_ds(bbox) # elevation only
64
+ ds = store.get_ds(bbox, derivatives=('slope', 'twi')) # + derived layers
65
+ ```
66
+ """
67
+
68
+ config: Config = default_config
69
+ copdem: CopernicusDEM = defaultcopdem
70
+ paths: Paths = field(init=False)
71
+
72
+ paths.default(lambda s: Paths(s.config))
73
+
74
+ def __attrs_post_init__(s):
75
+ makedirs(s.paths.root, exist_ok=True)
76
+
77
+ def _db(s) -> sqlite3.Connection:
78
+ db = sqlite3.connect(s.paths.index_db)
79
+ db.execute('PRAGMA journal_mode=WAL')
80
+ db.executescript(_SCHEMA)
81
+ return db
82
+
83
+ def _array(s, mode: str = 'a') -> zarr.Array:
84
+ root = zarr.open_group(s.paths.store, mode=mode)
85
+ try:
86
+ return root['elevation']
87
+ except KeyError:
88
+ return root.create_array(
89
+ 'elevation',
90
+ shape=(grid.HEIGHT_PX, grid.WIDTH_PX),
91
+ chunks=(grid.CHUNK, grid.CHUNK),
92
+ dtype='float32',
93
+ fill_value=np.nan,
94
+ )
95
+
96
+ # -- fill -------------------------------------------------------------
97
+
98
+ def fill(s, bbox: list[float]) -> int:
99
+ """Ensure every chunk covering ``bbox`` is populated.
100
+
101
+ Troi-agnostic (and, elevation being time-invariant, date-free).
102
+ Returns the number of chunks actually downloaded — 0 means the
103
+ request was already fully covered and no network was touched.
104
+ """
105
+ wanted = grid.chunks_in_window(grid.window_for_bbox(bbox))
106
+ db = s._db()
107
+ try:
108
+ done = set(db.execute('SELECT cy, cx FROM chunks').fetchall())
109
+ missing = [c for c in wanted if c not in done]
110
+ if not missing:
111
+ return 0
112
+ arr = s._array()
113
+ for cy, cx in missing:
114
+ s._fetch_chunk(arr, cy, cx)
115
+ with db:
116
+ db.execute(
117
+ 'INSERT OR REPLACE INTO chunks (cy, cx, written_at) VALUES (?, ?, ?)',
118
+ (cy, cx, datetime.now(timezone.utc).isoformat()),
119
+ )
120
+ return len(missing)
121
+ finally:
122
+ db.close()
123
+
124
+ def _fetch_chunk(s, arr: zarr.Array, cy: int, cx: int) -> None:
125
+ """One integer-aligned windowed read from the chunk's COG tile."""
126
+ import rasterio
127
+ from rasterio.windows import Window
128
+
129
+ tile = grid.tile_name(*grid.tile_of_chunk(cy, cx))
130
+ r0, r1, c0, c1 = grid.chunk_window_in_tile(cy, cx)
131
+ try:
132
+ # The bucket is public, but rasterio recognises *.s3.amazonaws.com
133
+ # URLs and tries to sign requests with whatever AWS credentials
134
+ # are lying around — a stale SSO profile then fails the fetch
135
+ # with TokenRetrievalError before GDAL makes a request. Force
136
+ # anonymous access.
137
+ with rasterio.Env(AWS_NO_SIGN_REQUEST='YES'), \
138
+ rasterio.open(s.copdem.tile_url(tile)) as src:
139
+ data = src.read(
140
+ 1, window=Window(c0, r0, c1 - c0, r1 - r0)
141
+ ).astype('float32')
142
+ if src.nodata is not None:
143
+ data = np.where(data == src.nodata, np.nan, data)
144
+ except rasterio.errors.RasterioIOError as e:
145
+ # Only a missing tile means ocean. Any other IO failure (403,
146
+ # timeout, truncated response) must propagate — recording it as
147
+ # nodata would poison the store permanently.
148
+ msg = str(e)
149
+ if not ('404' in msg or 'No such file' in msg or 'does not exist' in msg):
150
+ raise
151
+ data = np.full((grid.CHUNK, grid.CHUNK), np.nan, dtype='float32')
152
+ arr[cy * grid.CHUNK:(cy + 1) * grid.CHUNK,
153
+ cx * grid.CHUNK:(cx + 1) * grid.CHUNK] = data
154
+
155
+ # -- read -------------------------------------------------------------
156
+
157
+ def get_ds(s, bbox: list[float], derivatives: tuple[str, ...] = ()) -> xr.Dataset:
158
+ """Return the elevation window for ``bbox``, downloading only what's
159
+ missing first, with any requested derivatives computed on read.
160
+
161
+ Args:
162
+ bbox: ``[west, south, east, north]`` in EPSG:4326.
163
+ derivatives: Any of ``'slope'``, ``'aspect'``,
164
+ ``'accumulation'``, ``'twi'``, ``'hli'`` (dependencies are
165
+ resolved automatically — e.g. ``'twi'`` implies computing
166
+ slope and accumulation).
167
+
168
+ Returns:
169
+ xarray.Dataset with dims ``(lat, lon)`` on the fixed grid:
170
+ ``elevation`` plus one variable per requested derivative.
171
+ """
172
+ unknown = set(derivatives) - set(DERIVATIVES)
173
+ if unknown:
174
+ raise ValueError(f'Unknown derivative(s): {sorted(unknown)} — pick from {DERIVATIVES}')
175
+ s.fill(bbox)
176
+
177
+ window = grid.window_for_bbox(bbox)
178
+ row0, row1, col0, col1 = window
179
+ dem = s._array(mode='r')[row0:row1, col0:col1]
180
+ lat, lon = grid.coords_for_window(window)
181
+ ds = xr.Dataset(
182
+ {'elevation': (('lat', 'lon'), dem)},
183
+ coords={'lat': lat, 'lon': lon},
184
+ attrs={'crs': 'EPSG:4326', 'resolution_arcsec': 1},
185
+ )
186
+
187
+ if derivatives:
188
+ centre_lat = float(lat.mean())
189
+ xres = grid.RES * _M_PER_DEG * np.cos(np.radians(centre_lat))
190
+ yres = grid.RES * _M_PER_DEG
191
+ slope_deg = derive.slope(dem, xres, yres)
192
+ need_acc = 'accumulation' in derivatives or 'twi' in derivatives
193
+ acc = None
194
+ if need_acc:
195
+ from affine import Affine
196
+ transform = Affine(grid.RES, 0, grid.LON0 + col0 * grid.RES,
197
+ 0, -grid.RES, grid.LAT_TOP - row0 * grid.RES)
198
+ acc = derive.accumulation(dem, transform)
199
+ if 'slope' in derivatives:
200
+ ds['slope'] = (('lat', 'lon'), slope_deg)
201
+ if 'aspect' in derivatives or 'hli' in derivatives:
202
+ aspect_deg = derive.aspect(dem, xres, yres)
203
+ if 'aspect' in derivatives:
204
+ ds['aspect'] = (('lat', 'lon'), aspect_deg)
205
+ if 'hli' in derivatives:
206
+ ds['hli'] = (('lat', 'lon'), derive.hli(slope_deg, aspect_deg, centre_lat))
207
+ if 'accumulation' in derivatives:
208
+ ds['accumulation'] = (('lat', 'lon'), acc)
209
+ if 'twi' in derivatives:
210
+ ds['twi'] = (('lat', 'lon'), derive.twi(acc, slope_deg))
211
+ return ds
212
+
213
+ # -- Troi adapter (the reproducibility layer speaks Troi) -----------
214
+
215
+ def fill_troi(s, troi) -> int:
216
+ """:meth:`fill` for a :class:`troi.Troi` (dates ignored
217
+ — elevation is time-invariant)."""
218
+ return s.fill(troi.bbox)
219
+
220
+ def get_ds_troi(s, troi, derivatives: tuple[str, ...] = ()) -> xr.Dataset:
221
+ """:meth:`get_ds` for a :class:`troi.Troi`."""
222
+ return s.get_ds(troi.bbox, derivatives=derivatives)
223
+
224
+
225
+ # -- offline tests (synthetic writes, no network) --------------------------
226
+
227
+ _TEST_BBOX = [148.36265, -33.52606, 148.38265, -33.50606]
228
+
229
+
230
+ def _tmp_store() -> Store:
231
+ import tempfile
232
+ tmpdir = tempfile.mkdtemp(prefix='pycopdem_store_test_')
233
+ return Store(config=Config(out_dir=tmpdir, tmp_dir=tmpdir))
234
+
235
+
236
+ def _prime_synthetic(store: Store, bbox, value: float = 500.0, tilt: float = 0.0):
237
+ """Mark every chunk of bbox populated; elevation = value + tilt x column
238
+ (a plane rising eastward when tilt > 0)."""
239
+ window = grid.window_for_bbox(bbox)
240
+ arr = store._array()
241
+ db = store._db()
242
+ with db:
243
+ for cy, cx in grid.chunks_in_window(window):
244
+ cols = np.arange(cx * grid.CHUNK, (cx + 1) * grid.CHUNK, dtype='float32')
245
+ block = np.tile(value + tilt * cols, (grid.CHUNK, 1))
246
+ arr[cy * grid.CHUNK:(cy + 1) * grid.CHUNK,
247
+ cx * grid.CHUNK:(cx + 1) * grid.CHUNK] = block
248
+ db.execute(
249
+ 'INSERT OR REPLACE INTO chunks (cy, cx, written_at) VALUES (?, ?, ?)',
250
+ (cy, cx, 'synthetic'),
251
+ )
252
+ db.close()
253
+
254
+
255
+ def test_synthetic_write_read_roundtrip():
256
+ store = _tmp_store()
257
+ _prime_synthetic(store, _TEST_BBOX, value=512.0)
258
+ ds = store.get_ds(_TEST_BBOX)
259
+ return (
260
+ float(ds['elevation'][0, 0]) == 512.0
261
+ and ds.lat[0] > ds.lat[-1] # lat descending
262
+ )
263
+
264
+
265
+ def test_fill_skips_populated_chunks():
266
+ store = _tmp_store()
267
+ _prime_synthetic(store, _TEST_BBOX)
268
+ return store.fill(_TEST_BBOX) == 0
269
+
270
+
271
+ def test_flat_ground_has_zero_slope():
272
+ store = _tmp_store()
273
+ _prime_synthetic(store, _TEST_BBOX, value=300.0)
274
+ ds = store.get_ds(_TEST_BBOX, derivatives=('slope',))
275
+ return float(ds['slope'].max()) == 0.0
276
+
277
+
278
+ def test_derivatives_on_tilted_plane():
279
+ """A plane rising eastward -> positive slope, west-facing aspect,
280
+ finite TWI, HLI in (0, 1]."""
281
+ store = _tmp_store()
282
+ _prime_synthetic(store, _TEST_BBOX, value=300.0, tilt=0.5)
283
+ ds = store.get_ds(_TEST_BBOX, derivatives=('slope', 'aspect', 'twi', 'hli'))
284
+ return (
285
+ float(ds['slope'].min()) > 0.0
286
+ and abs(float(ds['aspect'][5, 5]) - 270.0) < 1.0
287
+ and np.isfinite(ds['twi'].values).all()
288
+ and 0 < float(ds['hli'][5, 5]) <= 1
289
+ )
290
+
291
+
292
+ def test_unknown_derivative_raises():
293
+ store = _tmp_store()
294
+ _prime_synthetic(store, _TEST_BBOX)
295
+ try:
296
+ store.get_ds(_TEST_BBOX, derivatives=('curvature',))
297
+ except ValueError:
298
+ return True
299
+ return False
300
+
301
+
302
+ def test():
303
+ return all([
304
+ test_synthetic_write_read_roundtrip(),
305
+ test_fill_skips_populated_chunks(),
306
+ test_flat_ground_has_zero_slope(),
307
+ test_derivatives_on_tilted_plane(),
308
+ test_unknown_derivative_raises(),
309
+ ])
310
+
311
+
312
+ if __name__ == '__main__':
313
+ print(test())
@@ -0,0 +1,155 @@
1
+ Metadata-Version: 2.4
2
+ Name: pycopdem
3
+ Version: 0.1.0
4
+ Summary: Cached Copernicus 30 m DEM windows with on-read terrain derivatives — download once per chunk, never twice
5
+ Author: Borevitz Lab, Australian National University
6
+ Author-email: Yasar Adeel Ansari <u6737670@anu.edu.au>
7
+ License: MIT
8
+ Project-URL: Homepage, https://github.com/thestochasticman/pycopdem
9
+ Project-URL: Repository, https://github.com/thestochasticman/pycopdem
10
+ Project-URL: Issues, https://github.com/thestochasticman/pycopdem/issues
11
+ Keywords: dem,copernicus,elevation,terrain,topography,twi,geospatial
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Topic :: Scientific/Engineering :: GIS
20
+ Requires-Python: >=3.11
21
+ Description-Content-Type: text/markdown
22
+ License-File: LICENSE
23
+ Requires-Dist: attrs
24
+ Requires-Dist: typing_extensions
25
+ Requires-Dist: numpy
26
+ Requires-Dist: xarray
27
+ Requires-Dist: zarr
28
+ Requires-Dist: rasterio
29
+ Requires-Dist: affine
30
+ Requires-Dist: pysheds
31
+ Dynamic: license-file
32
+
33
+ # pycopdem
34
+
35
+ **Cached [Copernicus GLO-30](https://registry.opendata.aws/copernicus-dem/)
36
+ elevation with on-read terrain derivatives — download once per chunk,
37
+ never twice.** Every elevation pixel this machine ever downloads lands
38
+ in one sparse, chunk-indexed store on the DEM's native 1-arc-second
39
+ grid; slope, aspect, flow accumulation, TWI and Heat Load Index are
40
+ computed on read and never stored. Part of the
41
+ [Borevitz Lab](https://borevitzlab.anu.edu.au/) ecosystem.
42
+
43
+ ## How it works
44
+
45
+ ```
46
+ {data_root}/copdem_store/
47
+ ├── index.db # SQLite ledger: which chunks are populated
48
+ └── dem.zarr # global sparse array; only written 1200×1200-px chunks exist
49
+ ```
50
+
51
+ - The GLO-30 DEM is served as one COG per 1° × 1° cell on a 1-arc-second
52
+ EPSG:4326 lattice. The store uses the same lattice globally, chunked
53
+ at 1200 px (1/3°) — so **3 × 3 chunks nest exactly inside every
54
+ tile**, and fetching a chunk is a single integer-aligned windowed
55
+ read from one COG. No resampling, ever.
56
+ - Any bbox maps deterministically to a set of chunk ids.
57
+ `Store.get_ds(bbox)` diffs them against the ledger and downloads
58
+ **only the missing chunks**. Elevation is time-invariant, so there's
59
+ no time axis and no date bookkeeping.
60
+ - A 1° tile absent from S3 is genuinely all ocean: its chunks are
61
+ stored as nodata and marked complete.
62
+ - Derivatives are a read-time transform (like everything derived in
63
+ this ecosystem): request them per call, pay compute not disk.
64
+
65
+ ## Usage
66
+
67
+ The core API is **troi-agnostic** — just a bbox:
68
+
69
+ ```python
70
+ from pycopdem.store import Store
71
+
72
+ store = Store()
73
+ bbox = [148.36265, -33.52606, 148.38265, -33.50606] # [W, S, E, N]
74
+
75
+ ds = store.get_ds(bbox) # elevation (lat, lon)
76
+ ds = store.get_ds(bbox, derivatives=('slope', 'aspect', 'twi', 'hli'))
77
+
78
+ store.fill(bbox) # → 0: already local
79
+ ```
80
+
81
+ Derivatives: `slope` and `aspect` (degrees), `accumulation` (pysheds
82
+ fill-pits → fill-depressions → resolve-flats → flowdir → accumulation),
83
+ `twi` (ln(accumulation / tan slope)), `hli` (McCune & Keon 2002).
84
+ Dependencies resolve automatically — asking for `twi` computes slope
85
+ and accumulation internally.
86
+
87
+ Pipelines that speak the shared `troi.troi.Troi` use the
88
+ adapters (dates on the troi are ignored — elevation doesn't change):
89
+
90
+ ```python
91
+ ds = store.get_ds_troi(troi, derivatives=('slope',))
92
+ ```
93
+
94
+ `download_terrain(troi)` remains as a thin wrapper.
95
+
96
+ ## Performance
97
+
98
+ Live measurements against the Copernicus S3 bucket — a ~2 × 2 km AOI
99
+ (one *chunk* = 1200 × 1200 px ≈ 37 × 30 km):
100
+
101
+ | Scenario | Downloaded | Time |
102
+ |---|---|---|
103
+ | Cold fill | 1 chunk | 10.3 s |
104
+ | Same request again | nothing | **0.0 s** |
105
+ | AOI shifted ~28 km east | 1 new chunk | 4.2 s |
106
+ | Read cached window (1200²) | — | 0.4 s |
107
+ | Read + all 5 derivatives | — | 3.2 s (pysheds dominates) |
108
+
109
+ Store footprint: ~10 MB for two chunks (~2 200 km² of elevation).
110
+ Absolute times vary with network; the zero is the point — it's a
111
+ ledger lookup, no network involved.
112
+
113
+ ## Install
114
+
115
+ ### pip
116
+
117
+ ```bash
118
+ pip install git+https://github.com/thestochasticman/pycopdem.git
119
+ ```
120
+
121
+ Dependencies (the `troi` core included, pulled from GitHub) are
122
+ declared in `pyproject.toml` and installed automatically.
123
+
124
+ ### From source
125
+
126
+ ```bash
127
+ git clone https://github.com/thestochasticman/pycopdem.git
128
+ cd pycopdem
129
+ pip install -e .
130
+ ```
131
+
132
+ Package design (shared across the lab's packages — no inheritance,
133
+ composition only):
134
+
135
+ - **`Troi`** (from `troi`) — identity: what region.
136
+ - **`CopernicusDEM`** (`pycopdem.copdem`) — config: tile source.
137
+ - **`Paths`** (`pycopdem.paths`) — derived locations of the store for a
138
+ given `Config`.
139
+ - **`grid`** — the fixed 1-arc-second grid and tile/chunk nesting
140
+ (pure, offline-testable math).
141
+ - **`derive`** — the terrain derivatives (pure array math + pysheds).
142
+ - **`Store`** (`pycopdem.store`) — ties them together.
143
+
144
+ ## Test
145
+
146
+ ```bash
147
+ # offline (pure math + synthetic store):
148
+ python pycopdem/grid.py # True
149
+ python pycopdem/paths.py # True
150
+ python pycopdem/derive.py # True
151
+ python pycopdem/store.py # True
152
+
153
+ # live (small real reads from the Copernicus S3 bucket):
154
+ python pycopdem/download_terrain.py # True
155
+ ```
@@ -0,0 +1,15 @@
1
+ LICENSE
2
+ README.md
3
+ pyproject.toml
4
+ pycopdem/__init__.py
5
+ pycopdem/copdem.py
6
+ pycopdem/derive.py
7
+ pycopdem/download_terrain.py
8
+ pycopdem/grid.py
9
+ pycopdem/paths.py
10
+ pycopdem/store.py
11
+ pycopdem.egg-info/PKG-INFO
12
+ pycopdem.egg-info/SOURCES.txt
13
+ pycopdem.egg-info/dependency_links.txt
14
+ pycopdem.egg-info/requires.txt
15
+ pycopdem.egg-info/top_level.txt
@@ -0,0 +1,8 @@
1
+ attrs
2
+ typing_extensions
3
+ numpy
4
+ xarray
5
+ zarr
6
+ rasterio
7
+ affine
8
+ pysheds
@@ -0,0 +1 @@
1
+ pycopdem
@@ -0,0 +1,55 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61.0", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "pycopdem"
7
+ version = "0.1.0"
8
+ description = "Cached Copernicus 30 m DEM windows with on-read terrain derivatives — download once per chunk, never twice"
9
+ readme = "README.md"
10
+ authors = [
11
+ { name = "Borevitz Lab, Australian National University" },
12
+ { name = "Yasar Adeel Ansari", email = "u6737670@anu.edu.au" },
13
+ ]
14
+ license = { text = "MIT" }
15
+ requires-python = ">=3.11"
16
+ keywords = [
17
+ "dem",
18
+ "copernicus",
19
+ "elevation",
20
+ "terrain",
21
+ "topography",
22
+ "twi",
23
+ "geospatial",
24
+ ]
25
+ classifiers = [
26
+ "Development Status :: 4 - Beta",
27
+ "Intended Audience :: Science/Research",
28
+ "License :: OSI Approved :: MIT License",
29
+ "Operating System :: OS Independent",
30
+ "Programming Language :: Python :: 3",
31
+ "Programming Language :: Python :: 3.11",
32
+ "Programming Language :: Python :: 3.12",
33
+ "Topic :: Scientific/Engineering :: GIS",
34
+ ]
35
+ # The geospatial stack (rasterio, pysheds, xarray, zarr, …) is provided by
36
+ # conda via environment.yml; only the lab core is declared here so
37
+ # `pip install -e .` is fast and doesn't re-resolve it.
38
+ dependencies = [
39
+ "attrs",
40
+ "typing_extensions",
41
+ "numpy",
42
+ "xarray",
43
+ "zarr",
44
+ "rasterio",
45
+ "affine",
46
+ "pysheds",
47
+ ]
48
+
49
+ [project.urls]
50
+ Homepage = "https://github.com/thestochasticman/pycopdem"
51
+ Repository = "https://github.com/thestochasticman/pycopdem"
52
+ Issues = "https://github.com/thestochasticman/pycopdem/issues"
53
+
54
+ [tool.setuptools.packages.find]
55
+ include = ["pycopdem", "pycopdem.*"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+