macrostrat.raster_index 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- macrostrat_raster_index-0.1.0/PKG-INFO +102 -0
- macrostrat_raster_index-0.1.0/README.md +76 -0
- macrostrat_raster_index-0.1.0/macrostrat/raster_index/__init__.py +26 -0
- macrostrat_raster_index-0.1.0/macrostrat/raster_index/cli.py +326 -0
- macrostrat_raster_index-0.1.0/macrostrat/raster_index/defs.py +60 -0
- macrostrat_raster_index-0.1.0/macrostrat/raster_index/footprints.py +94 -0
- macrostrat_raster_index-0.1.0/macrostrat/raster_index/index.py +444 -0
- macrostrat_raster_index-0.1.0/macrostrat/raster_index/scan.py +155 -0
- macrostrat_raster_index-0.1.0/macrostrat/raster_index/schema/01-raster-layers.sql +74 -0
- macrostrat_raster_index-0.1.0/macrostrat/raster_index/schema/02-functions.sql +163 -0
- macrostrat_raster_index-0.1.0/macrostrat/raster_index/testing.py +103 -0
- macrostrat_raster_index-0.1.0/pyproject.toml +71 -0
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: macrostrat.raster_index
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: An index of cloud-optimized raster datasets, for mosaicked tile serving
|
|
5
|
+
Author: Daven Quinn
|
|
6
|
+
Author-email: Daven Quinn <dev@davenquinn.com>
|
|
7
|
+
Classifier: Programming Language :: Python :: 3
|
|
8
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
11
|
+
Requires-Dist: macrostrat-database>=4.5,<5
|
|
12
|
+
Requires-Dist: macrostrat-utils>=1.3.3,<2
|
|
13
|
+
Requires-Dist: sqlalchemy>=2.0.18,<3
|
|
14
|
+
Requires-Dist: rio-tiler>=8,<9
|
|
15
|
+
Requires-Dist: rasterio>=1.4,<2
|
|
16
|
+
Requires-Dist: numpy>=1.24
|
|
17
|
+
Requires-Dist: pydantic>=2.7,<3
|
|
18
|
+
Requires-Dist: typer>=0.12,<0.27
|
|
19
|
+
Requires-Dist: rich>=13,<16
|
|
20
|
+
Requires-Dist: boto3>=1.28.50,<2 ; extra == 's3'
|
|
21
|
+
Requires-Python: >=3.11, <4
|
|
22
|
+
Project-URL: Repository, https://github.com/UW-Macrostrat/python-libraries
|
|
23
|
+
Project-URL: Documentation, https://github.com/UW-Macrostrat/python-libraries/tree/main/raster-index#readme
|
|
24
|
+
Provides-Extra: s3
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# `macrostrat.raster_index`
|
|
28
|
+
|
|
29
|
+
An index of cloud-optimized rasters (COGs), grouped into named **layers** that a
|
|
30
|
+
tile server can serve as single mosaics.
|
|
31
|
+
|
|
32
|
+
Nothing here stores pixels. A row in `raster_layers.raster` is a reference to a
|
|
33
|
+
COG in object storage plus the metadata needed to decide whether reading it is
|
|
34
|
+
worthwhile for a given tile: its footprint, its native zoom range, and its data
|
|
35
|
+
type. The schema name is `raster_layers` rather than `raster`/`rasters` to stay
|
|
36
|
+
clear of PostGIS Raster's vocabulary.
|
|
37
|
+
|
|
38
|
+
Serving these layers is [`macrostrat.raster_layers`](https://github.com/UW-Macrostrat/python-libraries/tree/main/raster-layers).
|
|
39
|
+
|
|
40
|
+
## Usage
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
from macrostrat.raster_index import RasterIndex, LayerDefinition
|
|
44
|
+
|
|
45
|
+
index = RasterIndex("postgresql://localhost:5432/macrostrat")
|
|
46
|
+
index.create_schema() # or apply `schema_files()` through your own system
|
|
47
|
+
|
|
48
|
+
index.register_layer(
|
|
49
|
+
LayerDefinition(slug="emit-minerals", name="EMIT mineral maps", maxzoom=14)
|
|
50
|
+
)
|
|
51
|
+
index.add_raster(
|
|
52
|
+
"https://storage.example.org/rasters/nevada.tif", layer="emit-minerals"
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
index.assets_for_tile(x=180, y=411, z=10, layers=["emit-minerals"])
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## Schema
|
|
59
|
+
|
|
60
|
+
`schema_files()` returns the SQL defining the schema, in application order, so a
|
|
61
|
+
host application can fold it into its own schema management rather than calling
|
|
62
|
+
`create_schema()`. Two tables and a few functions:
|
|
63
|
+
|
|
64
|
+
- `raster_layers.layer` — a named mosaic, and the defaults its rasters inherit
|
|
65
|
+
(zoom range, rescale range, colormap).
|
|
66
|
+
- `raster_layers.raster` — one COG: `href`, EPSG:4326 `footprint`, zoom range,
|
|
67
|
+
`dtype`/`nbands`/`nodata`, and the full reader metadata as `info`.
|
|
68
|
+
- `raster_layers.get_rasters(x, y, z, layers[])` — asset selection, ordered by
|
|
69
|
+
layer priority then resolution. The core of the whole package.
|
|
70
|
+
- `raster_layers.should_generate_tile(...)` — whether any asset actually resolves
|
|
71
|
+
at this zoom, for cache warmers and render short-circuits.
|
|
72
|
+
- `raster_layers.layer_footprints(layers[])` — footprints as GeoJSON features.
|
|
73
|
+
|
|
74
|
+
## CLI
|
|
75
|
+
|
|
76
|
+
`raster-index` reads `RASTER_INDEX_DATABASE` (or `DATABASE_URL`):
|
|
77
|
+
|
|
78
|
+
```sh
|
|
79
|
+
raster-index define-layer emit-minerals --name "EMIT mineral maps"
|
|
80
|
+
raster-index scan https://storage.example.org/remote-sensing-data/emit-mineral-maps/ \
|
|
81
|
+
--layer emit-minerals
|
|
82
|
+
raster-index set-colormap emit-minerals --from https://storage.example.org/.../nevada.tif
|
|
83
|
+
raster-index assets 10 180 411 --layer emit-minerals
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
The connection is a parameter of the app itself: `--database`, or those
|
|
87
|
+
environment variables. A host application that already knows its own connection
|
|
88
|
+
(Macrostrat mounts these as `macrostrat raster`) calls
|
|
89
|
+
`set_default_connection(url_or_callable)` once, and its users never pass
|
|
90
|
+
`--database` — though it still works, and still wins. Scanning object stores
|
|
91
|
+
needs the `s3` extra (boto3), whichever URL form you use — an `https://` bucket
|
|
92
|
+
URL is *rewritten* into an endpoint/bucket/prefix and listed through the same
|
|
93
|
+
S3 API, rather than being a second code path.
|
|
94
|
+
|
|
95
|
+
## Known limitations
|
|
96
|
+
|
|
97
|
+
- Footprints are bounding boxes, so rasters crossing the antimeridian are
|
|
98
|
+
indexed incorrectly. The column is typed `geometry`, not `polygon`, so a
|
|
99
|
+
mask-derived footprint can replace them without a migration.
|
|
100
|
+
- WebMercatorQuad only. Alternate tile grids (and non-Earth bodies, as in
|
|
101
|
+
[mars-tiler](https://github.com/davenquinn/mars-tiler)) would need a per-grid
|
|
102
|
+
bounds table.
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# `macrostrat.raster_index`
|
|
2
|
+
|
|
3
|
+
An index of cloud-optimized rasters (COGs), grouped into named **layers** that a
|
|
4
|
+
tile server can serve as single mosaics.
|
|
5
|
+
|
|
6
|
+
Nothing here stores pixels. A row in `raster_layers.raster` is a reference to a
|
|
7
|
+
COG in object storage plus the metadata needed to decide whether reading it is
|
|
8
|
+
worthwhile for a given tile: its footprint, its native zoom range, and its data
|
|
9
|
+
type. The schema name is `raster_layers` rather than `raster`/`rasters` to stay
|
|
10
|
+
clear of PostGIS Raster's vocabulary.
|
|
11
|
+
|
|
12
|
+
Serving these layers is [`macrostrat.raster_layers`](https://github.com/UW-Macrostrat/python-libraries/tree/main/raster-layers).
|
|
13
|
+
|
|
14
|
+
## Usage
|
|
15
|
+
|
|
16
|
+
```python
|
|
17
|
+
from macrostrat.raster_index import RasterIndex, LayerDefinition
|
|
18
|
+
|
|
19
|
+
index = RasterIndex("postgresql://localhost:5432/macrostrat")
|
|
20
|
+
index.create_schema() # or apply `schema_files()` through your own system
|
|
21
|
+
|
|
22
|
+
index.register_layer(
|
|
23
|
+
LayerDefinition(slug="emit-minerals", name="EMIT mineral maps", maxzoom=14)
|
|
24
|
+
)
|
|
25
|
+
index.add_raster(
|
|
26
|
+
"https://storage.example.org/rasters/nevada.tif", layer="emit-minerals"
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
index.assets_for_tile(x=180, y=411, z=10, layers=["emit-minerals"])
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## Schema
|
|
33
|
+
|
|
34
|
+
`schema_files()` returns the SQL defining the schema, in application order, so a
|
|
35
|
+
host application can fold it into its own schema management rather than calling
|
|
36
|
+
`create_schema()`. Two tables and a few functions:
|
|
37
|
+
|
|
38
|
+
- `raster_layers.layer` — a named mosaic, and the defaults its rasters inherit
|
|
39
|
+
(zoom range, rescale range, colormap).
|
|
40
|
+
- `raster_layers.raster` — one COG: `href`, EPSG:4326 `footprint`, zoom range,
|
|
41
|
+
`dtype`/`nbands`/`nodata`, and the full reader metadata as `info`.
|
|
42
|
+
- `raster_layers.get_rasters(x, y, z, layers[])` — asset selection, ordered by
|
|
43
|
+
layer priority then resolution. The core of the whole package.
|
|
44
|
+
- `raster_layers.should_generate_tile(...)` — whether any asset actually resolves
|
|
45
|
+
at this zoom, for cache warmers and render short-circuits.
|
|
46
|
+
- `raster_layers.layer_footprints(layers[])` — footprints as GeoJSON features.
|
|
47
|
+
|
|
48
|
+
## CLI
|
|
49
|
+
|
|
50
|
+
`raster-index` reads `RASTER_INDEX_DATABASE` (or `DATABASE_URL`):
|
|
51
|
+
|
|
52
|
+
```sh
|
|
53
|
+
raster-index define-layer emit-minerals --name "EMIT mineral maps"
|
|
54
|
+
raster-index scan https://storage.example.org/remote-sensing-data/emit-mineral-maps/ \
|
|
55
|
+
--layer emit-minerals
|
|
56
|
+
raster-index set-colormap emit-minerals --from https://storage.example.org/.../nevada.tif
|
|
57
|
+
raster-index assets 10 180 411 --layer emit-minerals
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
The connection is a parameter of the app itself: `--database`, or those
|
|
61
|
+
environment variables. A host application that already knows its own connection
|
|
62
|
+
(Macrostrat mounts these as `macrostrat raster`) calls
|
|
63
|
+
`set_default_connection(url_or_callable)` once, and its users never pass
|
|
64
|
+
`--database` — though it still works, and still wins. Scanning object stores
|
|
65
|
+
needs the `s3` extra (boto3), whichever URL form you use — an `https://` bucket
|
|
66
|
+
URL is *rewritten* into an endpoint/bucket/prefix and listed through the same
|
|
67
|
+
S3 API, rather than being a second code path.
|
|
68
|
+
|
|
69
|
+
## Known limitations
|
|
70
|
+
|
|
71
|
+
- Footprints are bounding boxes, so rasters crossing the antimeridian are
|
|
72
|
+
indexed incorrectly. The column is typed `geometry`, not `polygon`, so a
|
|
73
|
+
mask-derived footprint can replace them without a migration.
|
|
74
|
+
- WebMercatorQuad only. Alternate tile grids (and non-Earth bodies, as in
|
|
75
|
+
[mars-tiler](https://github.com/davenquinn/mars-tiler)) would need a per-grid
|
|
76
|
+
bounds table.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""An index of cloud-optimized rasters, for mosaicked tile serving.
|
|
2
|
+
|
|
3
|
+
Rasters live in object storage; this package records *where* they are, *what*
|
|
4
|
+
they cover, and *which named layer* they belong to, so a tile server can answer
|
|
5
|
+
"which files do I read for this tile?" with a single spatial query.
|
|
6
|
+
|
|
7
|
+
Serving is a separate concern, handled by `macrostrat.raster_layers`.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from .defs import LayerDefinition, RasterAsset, RasterInfo
|
|
11
|
+
from .footprints import get_raster_info
|
|
12
|
+
from .index import RasterIndex, schema_files
|
|
13
|
+
from .scan import BucketPrefix, RasterObject, parse_bucket_url, scan_prefix
|
|
14
|
+
|
|
15
|
+
__all__ = [
|
|
16
|
+
"RasterIndex",
|
|
17
|
+
"schema_files",
|
|
18
|
+
"get_raster_info",
|
|
19
|
+
"scan_prefix",
|
|
20
|
+
"RasterObject",
|
|
21
|
+
"BucketPrefix",
|
|
22
|
+
"parse_bucket_url",
|
|
23
|
+
"RasterAsset",
|
|
24
|
+
"RasterInfo",
|
|
25
|
+
"LayerDefinition",
|
|
26
|
+
]
|
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
"""Command-line surface for the raster index.
|
|
2
|
+
|
|
3
|
+
A plain module-level Typer app. The database is a parameter of the app itself —
|
|
4
|
+
`--database`, or the `RASTER_INDEX_DATABASE`/`DATABASE_URL` environment
|
|
5
|
+
variables — resolved once in the callback and handed to commands through the
|
|
6
|
+
Click context.
|
|
7
|
+
|
|
8
|
+
A host application that already knows its own connection (Macrostrat mounts
|
|
9
|
+
these commands as `macrostrat raster`) calls `set_default_connection()` to supply
|
|
10
|
+
it, and its users never have to pass `--database`.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from typing import Callable, Optional, Union
|
|
15
|
+
|
|
16
|
+
from rich import print
|
|
17
|
+
from rich.table import Table
|
|
18
|
+
from typer import Argument, BadParameter, Context, Exit, Option, Typer
|
|
19
|
+
|
|
20
|
+
from .defs import LayerDefinition
|
|
21
|
+
from .index import Connectable, RasterIndex
|
|
22
|
+
from .scan import scan_prefix
|
|
23
|
+
|
|
24
|
+
__all__ = ["cli", "set_default_connection", "index_for"]
|
|
25
|
+
|
|
26
|
+
# A connection, or something that produces one when a command actually needs it.
|
|
27
|
+
ConnectionSource = Union[Connectable, Callable[[], Connectable]]
|
|
28
|
+
|
|
29
|
+
# Set by a host application to supply its own database. Lower priority than an
|
|
30
|
+
# explicit `--database` so a user can always point the CLI somewhere else.
|
|
31
|
+
_default_connection: Optional[ConnectionSource] = None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def set_default_connection(source: Optional[ConnectionSource]) -> None:
|
|
35
|
+
"""Supply the connection to use when `--database` isn't given.
|
|
36
|
+
|
|
37
|
+
Accepts a URL, an engine or `Database`, or a zero-argument callable
|
|
38
|
+
returning one — a callable defers reading the host's configuration until a
|
|
39
|
+
command actually runs.
|
|
40
|
+
"""
|
|
41
|
+
global _default_connection
|
|
42
|
+
_default_connection = source
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass
|
|
46
|
+
class RasterIndexContext:
|
|
47
|
+
"""CLI state, attached to the Click context by the callback.
|
|
48
|
+
|
|
49
|
+
The index is built on first use rather than in the callback, so `--help` and
|
|
50
|
+
argument errors don't require a working database.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
connection: Optional[ConnectionSource] = None
|
|
54
|
+
_index: Optional[RasterIndex] = None
|
|
55
|
+
|
|
56
|
+
@property
|
|
57
|
+
def index(self) -> RasterIndex:
|
|
58
|
+
if self._index is not None:
|
|
59
|
+
return self._index
|
|
60
|
+
|
|
61
|
+
source = self.connection
|
|
62
|
+
if callable(source) and not isinstance(source, str):
|
|
63
|
+
source = source()
|
|
64
|
+
if source is None:
|
|
65
|
+
raise BadParameter(
|
|
66
|
+
"No database configured. Pass --database, or set "
|
|
67
|
+
"RASTER_INDEX_DATABASE (or DATABASE_URL).",
|
|
68
|
+
param_hint="--database",
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
self._index = RasterIndex(source)
|
|
72
|
+
return self._index
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def index_for(ctx: Context, *, require_schema: bool = True) -> RasterIndex:
|
|
76
|
+
"""The index for the running command.
|
|
77
|
+
|
|
78
|
+
The schema check turns the common first-run mistake — a database that's
|
|
79
|
+
never been provisioned — into a pointed message rather than an
|
|
80
|
+
`UndefinedTable` traceback from whichever query happened to run first.
|
|
81
|
+
"""
|
|
82
|
+
index = ctx.find_object(RasterIndexContext).index
|
|
83
|
+
if require_schema and not index.schema_exists():
|
|
84
|
+
print(
|
|
85
|
+
"[yellow]No [bold]raster_layers[/bold] schema in this database.\n"
|
|
86
|
+
"[dim]Apply it through your schema-management system, or run "
|
|
87
|
+
"`create-schema` for a scratch database."
|
|
88
|
+
)
|
|
89
|
+
raise Exit(1)
|
|
90
|
+
return index
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
cli = Typer(no_args_is_help=True, short_help="Manage indexed raster datasets")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@cli.callback()
|
|
97
|
+
def main(
|
|
98
|
+
ctx: Context,
|
|
99
|
+
database: Optional[str] = Option(
|
|
100
|
+
None,
|
|
101
|
+
"--database",
|
|
102
|
+
envvar=["RASTER_INDEX_DATABASE", "DATABASE_URL"],
|
|
103
|
+
help="PostgreSQL connection string for the raster index",
|
|
104
|
+
show_default=False,
|
|
105
|
+
),
|
|
106
|
+
):
|
|
107
|
+
"""Manage indexed raster datasets."""
|
|
108
|
+
ctx.obj = RasterIndexContext(connection=database or _default_connection)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@cli.command(name="create-schema")
|
|
112
|
+
def create_schema(ctx: Context):
|
|
113
|
+
"""Apply the `raster_layers` schema directly.
|
|
114
|
+
|
|
115
|
+
For scratch databases. Managed deployments should build the schema through
|
|
116
|
+
their own schema-management system.
|
|
117
|
+
"""
|
|
118
|
+
index_for(ctx, require_schema=False).create_schema()
|
|
119
|
+
print("[green]Created [bold]raster_layers[/bold] schema")
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@cli.command(name="layers")
|
|
123
|
+
def list_layers(ctx: Context):
|
|
124
|
+
"""List indexed raster layers."""
|
|
125
|
+
index = index_for(ctx)
|
|
126
|
+
|
|
127
|
+
counts = {}
|
|
128
|
+
for raster in index.rasters():
|
|
129
|
+
counts[raster["layer"]] = counts.get(raster["layer"], 0) + 1
|
|
130
|
+
|
|
131
|
+
table = Table("Layer", "Name", "Rasters", "Zooms", box=None)
|
|
132
|
+
for layer in index.layers():
|
|
133
|
+
table.add_row(
|
|
134
|
+
layer.slug,
|
|
135
|
+
layer.name or "",
|
|
136
|
+
str(counts.get(layer.slug, 0)),
|
|
137
|
+
_zoom_range(layer.minzoom, layer.maxzoom),
|
|
138
|
+
)
|
|
139
|
+
print(table)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@cli.command(name="rasters")
|
|
143
|
+
def list_rasters(ctx: Context, layer: Optional[str] = Argument(None)):
|
|
144
|
+
"""List indexed rasters, optionally within a single layer."""
|
|
145
|
+
table = Table("Layer", "Raster", "Zooms", "Type", "Href", box=None)
|
|
146
|
+
for raster in index_for(ctx).rasters(layer):
|
|
147
|
+
table.add_row(
|
|
148
|
+
raster["layer"],
|
|
149
|
+
raster["slug"],
|
|
150
|
+
_zoom_range(raster["minzoom"], raster["maxzoom"]),
|
|
151
|
+
f"{raster['dtype']}×{raster['nbands']}",
|
|
152
|
+
raster["href"],
|
|
153
|
+
)
|
|
154
|
+
print(table)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
@cli.command(name="define-layer")
|
|
158
|
+
def define_layer(
|
|
159
|
+
ctx: Context,
|
|
160
|
+
slug: str,
|
|
161
|
+
name: Optional[str] = Option(None),
|
|
162
|
+
description: Optional[str] = Option(None),
|
|
163
|
+
minzoom: Optional[int] = Option(None),
|
|
164
|
+
maxzoom: Optional[int] = Option(None),
|
|
165
|
+
):
|
|
166
|
+
"""Create or update a layer definition."""
|
|
167
|
+
index_for(ctx).register_layer(
|
|
168
|
+
LayerDefinition(
|
|
169
|
+
slug=slug,
|
|
170
|
+
name=name,
|
|
171
|
+
description=description,
|
|
172
|
+
minzoom=minzoom,
|
|
173
|
+
maxzoom=maxzoom,
|
|
174
|
+
)
|
|
175
|
+
)
|
|
176
|
+
print(f"[green]Defined layer [bold]{slug}[/bold]")
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
@cli.command(name="set-colormap")
|
|
180
|
+
def set_colormap(
|
|
181
|
+
ctx: Context,
|
|
182
|
+
layer: str,
|
|
183
|
+
source: str = Option(
|
|
184
|
+
...,
|
|
185
|
+
"--from",
|
|
186
|
+
help="Raster whose embedded palette should become the layer's colormap",
|
|
187
|
+
),
|
|
188
|
+
):
|
|
189
|
+
"""Copy a raster's embedded color table onto its layer.
|
|
190
|
+
|
|
191
|
+
Categorical rasters (classification maps) carry their palette in the file.
|
|
192
|
+
Serving them means having that palette at render time, which is what the
|
|
193
|
+
layer's colormap is for.
|
|
194
|
+
"""
|
|
195
|
+
from .footprints import get_raster_info
|
|
196
|
+
|
|
197
|
+
colormap = get_raster_info(source).colormap
|
|
198
|
+
if not colormap:
|
|
199
|
+
print(f"[yellow]{source} has no embedded color table")
|
|
200
|
+
raise Exit(1)
|
|
201
|
+
|
|
202
|
+
index = index_for(ctx)
|
|
203
|
+
existing = {l.slug: l for l in index.layers()}
|
|
204
|
+
if layer not in existing:
|
|
205
|
+
print(f"[yellow]Layer [bold]{layer}[/bold] is not defined")
|
|
206
|
+
raise Exit(1)
|
|
207
|
+
|
|
208
|
+
# Stored as JSON, so keys become strings and tuples become lists; the
|
|
209
|
+
# serving side normalizes them back.
|
|
210
|
+
index.register_layer(
|
|
211
|
+
existing[layer],
|
|
212
|
+
colormap={str(k): list(v) for k, v in colormap.items()},
|
|
213
|
+
)
|
|
214
|
+
print(f"[green]Set a {len(colormap)}-entry colormap on [bold]{layer}[/bold]")
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
@cli.command(name="add")
|
|
218
|
+
def add(
|
|
219
|
+
ctx: Context,
|
|
220
|
+
hrefs: list[str] = Argument(..., help="Raster URLs or paths"),
|
|
221
|
+
layer: str = Option(..., "--layer", "-l", help="Layer to add rasters to"),
|
|
222
|
+
):
|
|
223
|
+
"""Register rasters in a layer, reading footprints and zoom ranges."""
|
|
224
|
+
results = index_for(ctx).add_rasters(hrefs, layer)
|
|
225
|
+
print(f"[green]Registered {len(results)}/{len(hrefs)} rasters in {layer}")
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
@cli.command(name="scan")
|
|
229
|
+
def scan(
|
|
230
|
+
ctx: Context,
|
|
231
|
+
url: str = Argument(
|
|
232
|
+
...,
|
|
233
|
+
help="Bucket prefix, e.g. https://storage.example.org/bucket/prefix/",
|
|
234
|
+
),
|
|
235
|
+
layer: str = Option(..., "--layer", "-l", help="Layer to add rasters to"),
|
|
236
|
+
dry_run: bool = Option(False, help="Show what would be registered"),
|
|
237
|
+
credentials: bool = Option(False, help="Sign requests with AWS credentials"),
|
|
238
|
+
endpoint_url: Optional[str] = Option(
|
|
239
|
+
None, help="S3-compatible endpoint to list against (inferred from https URLs)"
|
|
240
|
+
),
|
|
241
|
+
public_url: Optional[str] = Option(
|
|
242
|
+
None, help="Rewrite hrefs onto this origin (inferred from https URLs)"
|
|
243
|
+
),
|
|
244
|
+
):
|
|
245
|
+
"""Register every raster under a bucket prefix.
|
|
246
|
+
|
|
247
|
+
An `https://` URL — the one you'd paste into a browser — needs nothing else:
|
|
248
|
+
its origin is the endpoint, its first path segment the bucket, and the
|
|
249
|
+
rasters are indexed at that same origin. Use `s3://` (with `--endpoint-url`)
|
|
250
|
+
for buckets whose location isn't implied by a public URL. Listing is
|
|
251
|
+
unsigned unless `--credentials` is given.
|
|
252
|
+
"""
|
|
253
|
+
objects = list(
|
|
254
|
+
scan_prefix(
|
|
255
|
+
url,
|
|
256
|
+
endpoint_url=endpoint_url,
|
|
257
|
+
public_url=public_url,
|
|
258
|
+
credentials=credentials,
|
|
259
|
+
)
|
|
260
|
+
)
|
|
261
|
+
if not objects:
|
|
262
|
+
print(f"[yellow]No rasters found under {url}")
|
|
263
|
+
return
|
|
264
|
+
|
|
265
|
+
if dry_run:
|
|
266
|
+
for obj in objects:
|
|
267
|
+
print(f"{obj.href} [dim]({obj.size / 1e6:.1f} MB)")
|
|
268
|
+
print(f"[dim]{len(objects)} rasters (dry run — nothing registered)")
|
|
269
|
+
return
|
|
270
|
+
|
|
271
|
+
results = index_for(ctx).add_rasters([o.href for o in objects], layer)
|
|
272
|
+
print(f"[green]Registered {len(results)}/{len(objects)} rasters in {layer}")
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
@cli.command(name="remove")
|
|
276
|
+
def remove(ctx: Context, href: str):
|
|
277
|
+
"""Remove a raster from the index."""
|
|
278
|
+
count = index_for(ctx).remove_raster(href)
|
|
279
|
+
if count:
|
|
280
|
+
print(f"[green]Removed {href}")
|
|
281
|
+
else:
|
|
282
|
+
print(f"[yellow]{href} is not in the index")
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
@cli.command(name="info")
|
|
286
|
+
def info(href: str):
|
|
287
|
+
"""Show the metadata that would be indexed for a raster.
|
|
288
|
+
|
|
289
|
+
Reads the raster directly, so it needs no database.
|
|
290
|
+
"""
|
|
291
|
+
from .footprints import get_raster_info
|
|
292
|
+
|
|
293
|
+
print(get_raster_info(href).model_dump(exclude={"metadata"}))
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
@cli.command(name="assets")
|
|
297
|
+
def assets(
|
|
298
|
+
ctx: Context,
|
|
299
|
+
z: int,
|
|
300
|
+
x: int,
|
|
301
|
+
y: int,
|
|
302
|
+
layers: list[str] = Option(..., "--layer", "-l"),
|
|
303
|
+
):
|
|
304
|
+
"""Show which rasters would be read for a tile."""
|
|
305
|
+
index = index_for(ctx)
|
|
306
|
+
|
|
307
|
+
table = Table("Layer", "Raster", "Zooms", "Overscaled", box=None)
|
|
308
|
+
for asset in index.assets_for_tile(x, y, z, layers):
|
|
309
|
+
table.add_row(
|
|
310
|
+
asset.layer,
|
|
311
|
+
asset.slug or "",
|
|
312
|
+
_zoom_range(asset.minzoom, asset.maxzoom),
|
|
313
|
+
"yes" if asset.overscaled else "",
|
|
314
|
+
)
|
|
315
|
+
print(table)
|
|
316
|
+
|
|
317
|
+
if not index.should_generate_tile(x, y, z, layers):
|
|
318
|
+
print("[yellow]No usable assets — this tile should not be generated")
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _zoom_range(minzoom, maxzoom) -> str:
|
|
322
|
+
if minzoom is None and maxzoom is None:
|
|
323
|
+
return ""
|
|
324
|
+
low = minzoom if minzoom is not None else "?"
|
|
325
|
+
high = maxzoom if maxzoom is not None else "?"
|
|
326
|
+
return f"{low}–{high}"
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""Types shared between indexing and serving.
|
|
2
|
+
|
|
3
|
+
These are the contract between `macrostrat.raster_index` and
|
|
4
|
+
`macrostrat.raster_layers`: the serving side never touches the tables directly,
|
|
5
|
+
it consumes `RasterAsset`s.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from typing import Any, Optional
|
|
9
|
+
|
|
10
|
+
from pydantic import BaseModel, Field
|
|
11
|
+
|
|
12
|
+
__all__ = ["RasterAsset", "RasterInfo", "LayerDefinition"]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class RasterAsset(BaseModel):
|
|
16
|
+
"""A raster selected for a specific tile.
|
|
17
|
+
|
|
18
|
+
Produced by `raster_layers.get_rasters`; consumed by the mosaic reader.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
href: str
|
|
22
|
+
layer: str
|
|
23
|
+
slug: Optional[str] = None
|
|
24
|
+
minzoom: Optional[int] = None
|
|
25
|
+
maxzoom: Optional[int] = None
|
|
26
|
+
rescale_range: Optional[list[float]] = None
|
|
27
|
+
# True when the requested tile is zoomed in past what this raster resolves.
|
|
28
|
+
overscaled: bool = False
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class RasterInfo(BaseModel):
|
|
32
|
+
"""Metadata derived by opening a raster, before it is written to the index."""
|
|
33
|
+
|
|
34
|
+
href: str
|
|
35
|
+
# Bounding box in EPSG:4326, as (west, south, east, north).
|
|
36
|
+
bounds: tuple[float, float, float, float]
|
|
37
|
+
# Footprint as a GeoJSON geometry dict, in EPSG:4326.
|
|
38
|
+
geometry: dict[str, Any]
|
|
39
|
+
minzoom: int
|
|
40
|
+
maxzoom: int
|
|
41
|
+
dtype: str
|
|
42
|
+
nbands: int
|
|
43
|
+
nodata: Optional[float] = None
|
|
44
|
+
crs: Optional[str] = None
|
|
45
|
+
# Colormap embedded in the raster itself (a GDAL color table), if any.
|
|
46
|
+
colormap: Optional[dict[int, tuple[int, int, int, int]]] = None
|
|
47
|
+
metadata: dict[str, Any] = Field(default_factory=dict)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class LayerDefinition(BaseModel):
|
|
51
|
+
"""A named mosaic, and the defaults its rasters inherit."""
|
|
52
|
+
|
|
53
|
+
slug: str
|
|
54
|
+
name: Optional[str] = None
|
|
55
|
+
description: Optional[str] = None
|
|
56
|
+
minzoom: Optional[int] = None
|
|
57
|
+
maxzoom: Optional[int] = None
|
|
58
|
+
rescale_range: Optional[list[float]] = None
|
|
59
|
+
colormap: Optional[dict[str, Any]] = None
|
|
60
|
+
metadata: Optional[dict[str, Any]] = None
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""Derive index metadata by opening a raster.
|
|
2
|
+
|
|
3
|
+
This is the only place in the package that touches pixels (or, more precisely,
|
|
4
|
+
headers): everything the index stores about a raster comes from here, so
|
|
5
|
+
registering a raster and re-registering it later are guaranteed to agree.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from typing import Any, Optional
|
|
9
|
+
|
|
10
|
+
import rasterio
|
|
11
|
+
from rio_tiler.constants import WGS84_CRS
|
|
12
|
+
from rio_tiler.io import Reader
|
|
13
|
+
|
|
14
|
+
from macrostrat.utils import get_logger
|
|
15
|
+
|
|
16
|
+
from .defs import RasterInfo
|
|
17
|
+
|
|
18
|
+
log = get_logger(__name__)
|
|
19
|
+
|
|
20
|
+
__all__ = ["get_raster_info", "bounds_to_geometry"]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def bounds_to_geometry(bounds: tuple[float, float, float, float]) -> dict[str, Any]:
|
|
24
|
+
"""A GeoJSON polygon for a (west, south, east, north) bounding box."""
|
|
25
|
+
west, south, east, north = bounds
|
|
26
|
+
return {
|
|
27
|
+
"type": "Polygon",
|
|
28
|
+
"coordinates": [
|
|
29
|
+
[
|
|
30
|
+
[west, north],
|
|
31
|
+
[west, south],
|
|
32
|
+
[east, south],
|
|
33
|
+
[east, north],
|
|
34
|
+
[west, north],
|
|
35
|
+
]
|
|
36
|
+
],
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def get_raster_info(href: str, **reader_options) -> RasterInfo:
|
|
41
|
+
"""Open a raster and collect everything the index needs to know about it.
|
|
42
|
+
|
|
43
|
+
The footprint is the geographic bounding box. That is enough for asset
|
|
44
|
+
selection — a false positive costs one wasted read, and rio-tiler masks the
|
|
45
|
+
result anyway — and it avoids paying for a mask read at registration time.
|
|
46
|
+
Rasters spanning the antimeridian are the known failure case: their
|
|
47
|
+
bounding box wraps the wrong way around the globe.
|
|
48
|
+
"""
|
|
49
|
+
with Reader(href, **reader_options) as src:
|
|
50
|
+
bounds = tuple(src.get_geographic_bounds(WGS84_CRS))
|
|
51
|
+
info = src.info()
|
|
52
|
+
dataset = src.dataset
|
|
53
|
+
|
|
54
|
+
crs = _crs_string(src.crs)
|
|
55
|
+
# rio-tiler normalizes an absent color table to `{}`; keep it as None so
|
|
56
|
+
# "no colormap" and "empty colormap" don't have to be distinguished
|
|
57
|
+
# downstream.
|
|
58
|
+
colormap = src.colormap or None
|
|
59
|
+
|
|
60
|
+
return RasterInfo(
|
|
61
|
+
href=href,
|
|
62
|
+
bounds=bounds,
|
|
63
|
+
geometry=bounds_to_geometry(bounds),
|
|
64
|
+
minzoom=src.minzoom,
|
|
65
|
+
maxzoom=src.maxzoom,
|
|
66
|
+
dtype=dataset.meta["dtype"],
|
|
67
|
+
nbands=dataset.count,
|
|
68
|
+
nodata=_first_nodata(dataset),
|
|
69
|
+
crs=crs,
|
|
70
|
+
colormap=colormap,
|
|
71
|
+
metadata=info.model_dump(mode="json", exclude={"colormap"}),
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _crs_string(crs: Optional[rasterio.crs.CRS]) -> Optional[str]:
|
|
76
|
+
if crs is None:
|
|
77
|
+
return None
|
|
78
|
+
# Prefer an authority code; fall back to WKT for CRSs PROJ can't name
|
|
79
|
+
# (planetary bodies, custom local grids).
|
|
80
|
+
try:
|
|
81
|
+
epsg = crs.to_epsg()
|
|
82
|
+
except Exception: # pragma: no cover - PROJ can raise on exotic CRSs
|
|
83
|
+
epsg = None
|
|
84
|
+
if epsg is not None:
|
|
85
|
+
return f"EPSG:{epsg}"
|
|
86
|
+
return crs.to_string()
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _first_nodata(dataset) -> Optional[float]:
|
|
90
|
+
"""The dataset's nodata value, if all bands agree on one."""
|
|
91
|
+
values = {v for v in dataset.nodatavals if v is not None}
|
|
92
|
+
if len(values) != 1:
|
|
93
|
+
return None
|
|
94
|
+
return float(values.pop())
|