geoserver-dataops 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- geoserver_dataops-0.2.0/LICENSE +21 -0
- geoserver_dataops-0.2.0/PKG-INFO +151 -0
- geoserver_dataops-0.2.0/README.md +130 -0
- geoserver_dataops-0.2.0/geoserver_dataops/__init__.py +4 -0
- geoserver_dataops-0.2.0/geoserver_dataops/__main__.py +3 -0
- geoserver_dataops-0.2.0/geoserver_dataops/cli.py +141 -0
- geoserver_dataops-0.2.0/geoserver_dataops/crs.py +21 -0
- geoserver_dataops-0.2.0/geoserver_dataops/discover.py +324 -0
- geoserver_dataops-0.2.0/geoserver_dataops/ingest.py +509 -0
- geoserver_dataops-0.2.0/geoserver_dataops/loader.py +132 -0
- geoserver_dataops-0.2.0/geoserver_dataops/raster.py +165 -0
- geoserver_dataops-0.2.0/geoserver_dataops/remote.py +260 -0
- geoserver_dataops-0.2.0/geoserver_dataops/schedule.py +96 -0
- geoserver_dataops-0.2.0/geoserver_dataops/schema/v1alpha1.json +346 -0
- geoserver_dataops-0.2.0/geoserver_dataops/secrets.py +65 -0
- geoserver_dataops-0.2.0/geoserver_dataops/sources/__init__.py +121 -0
- geoserver_dataops-0.2.0/geoserver_dataops/sources/arcgis.py +219 -0
- geoserver_dataops-0.2.0/geoserver_dataops/sources/catalogue.py +259 -0
- geoserver_dataops-0.2.0/geoserver_dataops/sources/ckan.py +135 -0
- geoserver_dataops-0.2.0/geoserver_dataops/sources/formats.py +104 -0
- geoserver_dataops-0.2.0/geoserver_dataops/sources/geofabrik.py +102 -0
- geoserver_dataops-0.2.0/geoserver_dataops/sources/koordinates.py +127 -0
- geoserver_dataops-0.2.0/geoserver_dataops/sources/ogcapi.py +115 -0
- geoserver_dataops-0.2.0/geoserver_dataops/sources/opendata.py +152 -0
- geoserver_dataops-0.2.0/geoserver_dataops/sources/os_downloads.py +134 -0
- geoserver_dataops-0.2.0/geoserver_dataops/sources/overture.py +205 -0
- geoserver_dataops-0.2.0/geoserver_dataops/sources/stac.py +184 -0
- geoserver_dataops-0.2.0/geoserver_dataops/sources/url.py +99 -0
- geoserver_dataops-0.2.0/geoserver_dataops/sources/wfs.py +123 -0
- geoserver_dataops-0.2.0/geoserver_dataops/status.py +90 -0
- geoserver_dataops-0.2.0/geoserver_dataops/sync.py +315 -0
- geoserver_dataops-0.2.0/geoserver_dataops/validate.py +212 -0
- geoserver_dataops-0.2.0/geoserver_dataops.egg-info/PKG-INFO +151 -0
- geoserver_dataops-0.2.0/geoserver_dataops.egg-info/SOURCES.txt +67 -0
- geoserver_dataops-0.2.0/geoserver_dataops.egg-info/dependency_links.txt +1 -0
- geoserver_dataops-0.2.0/geoserver_dataops.egg-info/entry_points.txt +2 -0
- geoserver_dataops-0.2.0/geoserver_dataops.egg-info/requires.txt +13 -0
- geoserver_dataops-0.2.0/geoserver_dataops.egg-info/top_level.txt +1 -0
- geoserver_dataops-0.2.0/pyproject.toml +35 -0
- geoserver_dataops-0.2.0/setup.cfg +4 -0
- geoserver_dataops-0.2.0/tests/test_arcgis.py +96 -0
- geoserver_dataops-0.2.0/tests/test_arcgis_integration.py +66 -0
- geoserver_dataops-0.2.0/tests/test_catalogue.py +112 -0
- geoserver_dataops-0.2.0/tests/test_catalogue_integration.py +48 -0
- geoserver_dataops-0.2.0/tests/test_discover.py +677 -0
- geoserver_dataops-0.2.0/tests/test_discover_services.py +331 -0
- geoserver_dataops-0.2.0/tests/test_download.py +79 -0
- geoserver_dataops-0.2.0/tests/test_e2e_geoserver.py +212 -0
- geoserver_dataops-0.2.0/tests/test_geofabrik.py +100 -0
- geoserver_dataops-0.2.0/tests/test_idle_integration.py +43 -0
- geoserver_dataops-0.2.0/tests/test_koordinates.py +92 -0
- geoserver_dataops-0.2.0/tests/test_lock_integration.py +57 -0
- geoserver_dataops-0.2.0/tests/test_log_integration.py +20 -0
- geoserver_dataops-0.2.0/tests/test_opendata.py +54 -0
- geoserver_dataops-0.2.0/tests/test_opendata_integration.py +50 -0
- geoserver_dataops-0.2.0/tests/test_overture.py +76 -0
- geoserver_dataops-0.2.0/tests/test_overture_integration.py +46 -0
- geoserver_dataops-0.2.0/tests/test_raster.py +170 -0
- geoserver_dataops-0.2.0/tests/test_raster_backups.py +31 -0
- geoserver_dataops-0.2.0/tests/test_raster_sync_integration.py +44 -0
- geoserver_dataops-0.2.0/tests/test_run_integration.py +133 -0
- geoserver_dataops-0.2.0/tests/test_schedule.py +20 -0
- geoserver_dataops-0.2.0/tests/test_stac.py +137 -0
- geoserver_dataops-0.2.0/tests/test_stac_integration.py +84 -0
- geoserver_dataops-0.2.0/tests/test_swap_integration.py +87 -0
- geoserver_dataops-0.2.0/tests/test_sync.py +175 -0
- geoserver_dataops-0.2.0/tests/test_sync_integration.py +251 -0
- geoserver_dataops-0.2.0/tests/test_validate.py +50 -0
- geoserver_dataops-0.2.0/tests/test_versions.py +49 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Nathan Leeks
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: geoserver-dataops
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: The data GeoServer publishes, loaded into PostGIS and kept up to date from YAML files: ArcGIS, CKAN, STAC, Overture, OpenStreetMap, WFS, OGC API and more
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Requires-Python: >=3.10
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: pyyaml>=6
|
|
10
|
+
Requires-Dist: jsonschema>=4.18
|
|
11
|
+
Requires-Dist: requests>=2.28
|
|
12
|
+
Requires-Dist: psycopg2-binary>=2.9
|
|
13
|
+
Provides-Extra: dev
|
|
14
|
+
Requires-Dist: pytest; extra == "dev"
|
|
15
|
+
Requires-Dist: pyarrow>=14; extra == "dev"
|
|
16
|
+
Requires-Dist: shapely>=2; extra == "dev"
|
|
17
|
+
Provides-Extra: overture
|
|
18
|
+
Requires-Dist: pyarrow>=14; extra == "overture"
|
|
19
|
+
Requires-Dist: shapely>=2; extra == "overture"
|
|
20
|
+
Dynamic: license-file
|
|
21
|
+
|
|
22
|
+
<p align="center"><img src="docs/logo.svg" alt="GeoServer DataOps" width="120"></p>
|
|
23
|
+
|
|
24
|
+
<h1 align="center">GeoServer DataOps</h1>
|
|
25
|
+
|
|
26
|
+
<p align="center">The data GeoServer publishes, loaded into PostGIS and kept up to date, from YAML files.<br>For <a href="https://github.com/geoserver/geoserver-cloud">GeoServer Cloud</a> and standard GeoServer, beside <a href="https://github.com/Yibbzz/geoserver-configops">GeoServer ConfigOps</a>.</p>
|
|
27
|
+
|
|
28
|
+
<!-- badges -->
|
|
29
|
+
   
|
|
30
|
+
<!-- /badges -->
|
|
31
|
+
|
|
32
|
+
<!-- tested-on -->
|
|
33
|
+
Tested on **GeoServer Cloud 3.0.2.1**, **GeoServer 3.0.2** and **GeoServer 2.28.6**, with **PostGIS 18-3.7**: on every pull request each one publishes what `dataops sync` loaded and serves it back, before and after a reload, and a daily job (`.github/workflows/versions.yaml`) tries each new release.
|
|
34
|
+
<!-- /tested-on -->
|
|
35
|
+
|
|
36
|
+
`dataops` keeps the open datasets you name in PostGIS, each at its newest version:
|
|
37
|
+
|
|
38
|
+
- **Find** data in the places it is published, worldwide: ArcGIS services, CKAN, Socrata and OpenDataSoft portals, INSPIRE and GeoNetwork catalogues (CSW, OGC API Records), WFS and OGC API Features, STAC imagery catalogues, Overture Maps, OpenStreetMap (Geofabrik), New Zealand's Koordinates services, the Ordnance Survey, or plain links. `discover` lists what a source offers and the layers inside a download.
|
|
39
|
+
- **Load** it safely: staged, checked (invalid geometries, row counts), then swapped in for the old tables in one transaction, which are kept as backups. A raster download becomes one Cloud-Optimised GeoTIFF.
|
|
40
|
+
- **Keep it up to date.** Each dataset has a schedule; `sync`, run nightly, checks those that are due and loads only a new version.
|
|
41
|
+
|
|
42
|
+
It loads the data; [GeoServer ConfigOps](https://github.com/Yibbzz/geoserver-configops) configures GeoServer to publish it, and `configops refresh` brings GeoServer up to date after each load.
|
|
43
|
+
|
|
44
|
+
> **Status: 0.x.** The file format may still change between releases. Formerly **geodata-sync**: the command was `geodata`, the files' `apiVersion` `geodata/v1alpha1` and the variables `GEODATA_*`; [docs/development.md](docs/development.md#renamed-from-geodata-sync) says what to change.
|
|
45
|
+
|
|
46
|
+
## Install
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
pip install geoserver-dataops # Python 3.10+, and GDAL's tools (apt install gdal-bin)
|
|
50
|
+
docker run --rm ghcr.io/yibbzz/geoserver-dataops --help # or the image, GDAL included
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Example
|
|
54
|
+
|
|
55
|
+
[`examples/quickstart/project.yaml`](examples/quickstart/project.yaml): Surrey Heath's conservation areas, from data.gov.uk, checked monthly.
|
|
56
|
+
|
|
57
|
+
```yaml
|
|
58
|
+
apiVersion: dataops/v1alpha1
|
|
59
|
+
kind: Project
|
|
60
|
+
metadata: { name: quickstart }
|
|
61
|
+
spec:
|
|
62
|
+
database:
|
|
63
|
+
connection: { secret: database } # keys: host, port, database, username, password
|
|
64
|
+
crs: EPSG:27700
|
|
65
|
+
---
|
|
66
|
+
apiVersion: dataops/v1alpha1
|
|
67
|
+
kind: Source
|
|
68
|
+
metadata: { name: data-gov-uk }
|
|
69
|
+
spec: { type: ckan, url: https://data.gov.uk }
|
|
70
|
+
---
|
|
71
|
+
apiVersion: dataops/v1alpha1
|
|
72
|
+
kind: Dataset
|
|
73
|
+
metadata: { name: conservation-areas }
|
|
74
|
+
spec:
|
|
75
|
+
source: data-gov-uk
|
|
76
|
+
product: conservation-areas125 # `dataops discover data-gov-uk --search "conservation areas"` finds it
|
|
77
|
+
update: monthly
|
|
78
|
+
tables:
|
|
79
|
+
conservation_areas: {} # the layer in the download; table conservation_areas_conservation_areas
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
```console
|
|
83
|
+
$ docker compose -f examples/compose.yaml up -d --wait
|
|
84
|
+
|
|
85
|
+
$ cat > examples/quickstart/secrets.yaml <<'EOF'
|
|
86
|
+
database: { host: localhost, port: "5433", database: postgres, username: postgres, password: postgres }
|
|
87
|
+
EOF
|
|
88
|
+
|
|
89
|
+
$ dataops sync -C examples/quickstart
|
|
90
|
+
conservation-areas: loaded 2026-11-11 in 0s: data_layers.conservation_areas_conservation_areas (10 rows)
|
|
91
|
+
note: 1 invalid geometry left out
|
|
92
|
+
$ dataops status -C examples/quickstart
|
|
93
|
+
DATASET UPDATE LOADED AT ROWS NEXT CHECK LAST CHECK
|
|
94
|
+
conservation-areas monthly 2026-11-11 2026-11-13 08:51 10 2026-12-13 07:51 success
|
|
95
|
+
$ dataops sync -C examples/quickstart
|
|
96
|
+
conservation-areas: not due (monthly)
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
`docker compose -f examples/compose.yaml down -v` removes the database. Two larger examples: [`examples/uk-basemap`](examples/uk-basemap/) (OS Open Roads, Boundary-Line and the Environment Agency's flood zones: 3 GB, 20 minutes) and [`examples/terrain`](examples/terrain/) (OS Terrain 50 as one Cloud-Optimised GeoTIFF).
|
|
100
|
+
|
|
101
|
+
## Common tasks
|
|
102
|
+
|
|
103
|
+
**Find a dataset** and the layers inside it:
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
dataops discover os-downloads # what a source type offers
|
|
107
|
+
dataops discover data-gov-uk --search "flood zones" -C my-data # search a Source of the project
|
|
108
|
+
dataops discover data-gov-uk <product> -C my-data # its files, and their layers
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
**Load now** rather than when due: `dataops sync -C my-data <dataset>`; `--force` loads even when the version hasn't changed.
|
|
112
|
+
|
|
113
|
+
**Refresh GeoServer after a load.** `sync --loaded` lists what it loaded, and `configops refresh` works out those layers' bounds again and clears their cached tiles:
|
|
114
|
+
|
|
115
|
+
```bash
|
|
116
|
+
dataops sync -C my-data --loaded loaded.txt
|
|
117
|
+
configops refresh -C my-geoserver --from loaded.txt
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
**Keep logins out of the files.** A file names a Secret (`connection: { secret: database }`); its value comes from a Kubernetes Secret, a `DATAOPS_<SECRET>_<KEY>` environment variable or a git-ignored `secrets.yaml`. Credentials in addresses are masked in errors and the ingest log.
|
|
121
|
+
|
|
122
|
+
**Run it in Kubernetes** as a nightly CronJob that loads, then refreshes GeoServer: [docs/kubernetes.md](docs/kubernetes.md).
|
|
123
|
+
|
|
124
|
+
## Built to be safe to run
|
|
125
|
+
|
|
126
|
+
- **A load that fails changes nothing.** Live tables are only swapped once every table of the dataset has loaded and passed its checks.
|
|
127
|
+
- **The old data is kept.** Each swap moves the old table to a backup schema; the newest `backups` are kept.
|
|
128
|
+
- **What reads the tables keeps working.** The new table gets the old one's grants, and views on it are moved across.
|
|
129
|
+
- **One sync at a time.** A run takes a lock in PostgreSQL, so a nightly job and a person can't load through each other.
|
|
130
|
+
- **Downloads aren't trusted.** GDAL may only open them as data formats, and what an archive unpacks to is limited.
|
|
131
|
+
- **Checked before it runs.** `dataops validate` reads the files without a network; problems come out as `file:line`.
|
|
132
|
+
|
|
133
|
+
## What to know
|
|
134
|
+
|
|
135
|
+
- Big layers over WFS or OGC API Features don't load reliably: use a file download when there is one ([docs/sources-notes.md](docs/sources-notes.md)).
|
|
136
|
+
- A WFS or OGC API Features source has no version to compare, so it is loaded in full each time it is due; so is an ArcGIS layer that doesn't report its last edit.
|
|
137
|
+
- Overture reads GeoParquet with pyarrow: `pip install 'geoserver-dataops[overture]'` (the image has it). Koordinates needs an API key.
|
|
138
|
+
- A materialized view on a live table stops its load: drop it before and create it again after.
|
|
139
|
+
|
|
140
|
+
## Documentation
|
|
141
|
+
|
|
142
|
+
| | |
|
|
143
|
+
| --- | --- |
|
|
144
|
+
| [docs/spec.md](docs/spec.md) | The file format and every command |
|
|
145
|
+
| [docs/kubernetes.md](docs/kubernetes.md) | Running it nightly in a cluster |
|
|
146
|
+
| [docs/sources-notes.md](docs/sources-notes.md) | What we learned about the sources |
|
|
147
|
+
| [docs/development.md](docs/development.md) | Tests, CI, new GeoServer versions, releasing |
|
|
148
|
+
|
|
149
|
+
## Licence
|
|
150
|
+
|
|
151
|
+
MIT (see `LICENSE`). Data you load keeps its own licence: OS OpenData is under the [Open Government Licence v3.0](https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/) and needs "Contains OS data © Crown copyright and database right <year>" wherever it's shown.
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
<p align="center"><img src="docs/logo.svg" alt="GeoServer DataOps" width="120"></p>
|
|
2
|
+
|
|
3
|
+
<h1 align="center">GeoServer DataOps</h1>
|
|
4
|
+
|
|
5
|
+
<p align="center">The data GeoServer publishes, loaded into PostGIS and kept up to date, from YAML files.<br>For <a href="https://github.com/geoserver/geoserver-cloud">GeoServer Cloud</a> and standard GeoServer, beside <a href="https://github.com/Yibbzz/geoserver-configops">GeoServer ConfigOps</a>.</p>
|
|
6
|
+
|
|
7
|
+
<!-- badges -->
|
|
8
|
+
   
|
|
9
|
+
<!-- /badges -->
|
|
10
|
+
|
|
11
|
+
<!-- tested-on -->
|
|
12
|
+
Tested on **GeoServer Cloud 3.0.2.1**, **GeoServer 3.0.2** and **GeoServer 2.28.6**, with **PostGIS 18-3.7**: on every pull request each one publishes what `dataops sync` loaded and serves it back, before and after a reload, and a daily job (`.github/workflows/versions.yaml`) tries each new release.
|
|
13
|
+
<!-- /tested-on -->
|
|
14
|
+
|
|
15
|
+
`dataops` keeps the open datasets you name in PostGIS, each at its newest version:
|
|
16
|
+
|
|
17
|
+
- **Find** data in the places it is published, worldwide: ArcGIS services, CKAN, Socrata and OpenDataSoft portals, INSPIRE and GeoNetwork catalogues (CSW, OGC API Records), WFS and OGC API Features, STAC imagery catalogues, Overture Maps, OpenStreetMap (Geofabrik), New Zealand's Koordinates services, the Ordnance Survey, or plain links. `discover` lists what a source offers and the layers inside a download.
|
|
18
|
+
- **Load** it safely: staged, checked (invalid geometries, row counts), then swapped in for the old tables in one transaction, which are kept as backups. A raster download becomes one Cloud-Optimised GeoTIFF.
|
|
19
|
+
- **Keep it up to date.** Each dataset has a schedule; `sync`, run nightly, checks those that are due and loads only a new version.
|
|
20
|
+
|
|
21
|
+
It loads the data; [GeoServer ConfigOps](https://github.com/Yibbzz/geoserver-configops) configures GeoServer to publish it, and `configops refresh` brings GeoServer up to date after each load.
|
|
22
|
+
|
|
23
|
+
> **Status: 0.x.** The file format may still change between releases. Formerly **geodata-sync**: the command was `geodata`, the files' `apiVersion` `geodata/v1alpha1` and the variables `GEODATA_*`; [docs/development.md](docs/development.md#renamed-from-geodata-sync) says what to change.
|
|
24
|
+
|
|
25
|
+
## Install
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
pip install geoserver-dataops # Python 3.10+, and GDAL's tools (apt install gdal-bin)
|
|
29
|
+
docker run --rm ghcr.io/yibbzz/geoserver-dataops --help # or the image, GDAL included
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
## Example
|
|
33
|
+
|
|
34
|
+
[`examples/quickstart/project.yaml`](examples/quickstart/project.yaml): Surrey Heath's conservation areas, from data.gov.uk, checked monthly.
|
|
35
|
+
|
|
36
|
+
```yaml
|
|
37
|
+
apiVersion: dataops/v1alpha1
|
|
38
|
+
kind: Project
|
|
39
|
+
metadata: { name: quickstart }
|
|
40
|
+
spec:
|
|
41
|
+
database:
|
|
42
|
+
connection: { secret: database } # keys: host, port, database, username, password
|
|
43
|
+
crs: EPSG:27700
|
|
44
|
+
---
|
|
45
|
+
apiVersion: dataops/v1alpha1
|
|
46
|
+
kind: Source
|
|
47
|
+
metadata: { name: data-gov-uk }
|
|
48
|
+
spec: { type: ckan, url: https://data.gov.uk }
|
|
49
|
+
---
|
|
50
|
+
apiVersion: dataops/v1alpha1
|
|
51
|
+
kind: Dataset
|
|
52
|
+
metadata: { name: conservation-areas }
|
|
53
|
+
spec:
|
|
54
|
+
source: data-gov-uk
|
|
55
|
+
product: conservation-areas125 # `dataops discover data-gov-uk --search "conservation areas"` finds it
|
|
56
|
+
update: monthly
|
|
57
|
+
tables:
|
|
58
|
+
conservation_areas: {} # the layer in the download; table conservation_areas_conservation_areas
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
```console
|
|
62
|
+
$ docker compose -f examples/compose.yaml up -d --wait
|
|
63
|
+
|
|
64
|
+
$ cat > examples/quickstart/secrets.yaml <<'EOF'
|
|
65
|
+
database: { host: localhost, port: "5433", database: postgres, username: postgres, password: postgres }
|
|
66
|
+
EOF
|
|
67
|
+
|
|
68
|
+
$ dataops sync -C examples/quickstart
|
|
69
|
+
conservation-areas: loaded 2026-11-11 in 0s: data_layers.conservation_areas_conservation_areas (10 rows)
|
|
70
|
+
note: 1 invalid geometry left out
|
|
71
|
+
$ dataops status -C examples/quickstart
|
|
72
|
+
DATASET UPDATE LOADED AT ROWS NEXT CHECK LAST CHECK
|
|
73
|
+
conservation-areas monthly 2026-11-11 2026-11-13 08:51 10 2026-12-13 07:51 success
|
|
74
|
+
$ dataops sync -C examples/quickstart
|
|
75
|
+
conservation-areas: not due (monthly)
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
`docker compose -f examples/compose.yaml down -v` removes the database. Two larger examples: [`examples/uk-basemap`](examples/uk-basemap/) (OS Open Roads, Boundary-Line and the Environment Agency's flood zones: 3 GB, 20 minutes) and [`examples/terrain`](examples/terrain/) (OS Terrain 50 as one Cloud-Optimised GeoTIFF).
|
|
79
|
+
|
|
80
|
+
## Common tasks
|
|
81
|
+
|
|
82
|
+
**Find a dataset** and the layers inside it:
|
|
83
|
+
|
|
84
|
+
```bash
|
|
85
|
+
dataops discover os-downloads # what a source type offers
|
|
86
|
+
dataops discover data-gov-uk --search "flood zones" -C my-data # search a Source of the project
|
|
87
|
+
dataops discover data-gov-uk <product> -C my-data # its files, and their layers
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
**Load now** rather than when due: `dataops sync -C my-data <dataset>`; `--force` loads even when the version hasn't changed.
|
|
91
|
+
|
|
92
|
+
**Refresh GeoServer after a load.** `sync --loaded` lists what it loaded, and `configops refresh` works out those layers' bounds again and clears their cached tiles:
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
dataops sync -C my-data --loaded loaded.txt
|
|
96
|
+
configops refresh -C my-geoserver --from loaded.txt
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
**Keep logins out of the files.** A file names a Secret (`connection: { secret: database }`); its value comes from a Kubernetes Secret, a `DATAOPS_<SECRET>_<KEY>` environment variable or a git-ignored `secrets.yaml`. Credentials in addresses are masked in errors and the ingest log.
|
|
100
|
+
|
|
101
|
+
**Run it in Kubernetes** as a nightly CronJob that loads, then refreshes GeoServer: [docs/kubernetes.md](docs/kubernetes.md).
|
|
102
|
+
|
|
103
|
+
## Built to be safe to run
|
|
104
|
+
|
|
105
|
+
- **A load that fails changes nothing.** Live tables are only swapped once every table of the dataset has loaded and passed its checks.
|
|
106
|
+
- **The old data is kept.** Each swap moves the old table to a backup schema; the newest `backups` are kept.
|
|
107
|
+
- **What reads the tables keeps working.** The new table gets the old one's grants, and views on it are moved across.
|
|
108
|
+
- **One sync at a time.** A run takes a lock in PostgreSQL, so a nightly job and a person can't load through each other.
|
|
109
|
+
- **Downloads aren't trusted.** GDAL may only open them as data formats, and what an archive unpacks to is limited.
|
|
110
|
+
- **Checked before it runs.** `dataops validate` reads the files without a network; problems come out as `file:line`.
|
|
111
|
+
|
|
112
|
+
## What to know
|
|
113
|
+
|
|
114
|
+
- Big layers over WFS or OGC API Features don't load reliably: use a file download when there is one ([docs/sources-notes.md](docs/sources-notes.md)).
|
|
115
|
+
- A WFS or OGC API Features source has no version to compare, so it is loaded in full each time it is due; so is an ArcGIS layer that doesn't report its last edit.
|
|
116
|
+
- Overture reads GeoParquet with pyarrow: `pip install 'geoserver-dataops[overture]'` (the image has it). Koordinates needs an API key.
|
|
117
|
+
- A materialized view on a live table stops its load: drop it before and create it again after.
|
|
118
|
+
|
|
119
|
+
## Documentation
|
|
120
|
+
|
|
121
|
+
| | |
|
|
122
|
+
| --- | --- |
|
|
123
|
+
| [docs/spec.md](docs/spec.md) | The file format and every command |
|
|
124
|
+
| [docs/kubernetes.md](docs/kubernetes.md) | Running it nightly in a cluster |
|
|
125
|
+
| [docs/sources-notes.md](docs/sources-notes.md) | What we learned about the sources |
|
|
126
|
+
| [docs/development.md](docs/development.md) | Tests, CI, new GeoServer versions, releasing |
|
|
127
|
+
|
|
128
|
+
## Licence
|
|
129
|
+
|
|
130
|
+
MIT (see `LICENSE`). Data you load keeps its own licence: OS OpenData is under the [Open Government Licence v3.0](https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/) and needs "Contains OS data © Crown copyright and database right <year>" wherever it's shown.
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""The dataops command."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
from . import __version__
|
|
12
|
+
from .discover import DiscoverError
|
|
13
|
+
from .discover import run as discover
|
|
14
|
+
from .status import run as status
|
|
15
|
+
from .sync import SyncError
|
|
16
|
+
from .sync import run as sync
|
|
17
|
+
from .loader import load
|
|
18
|
+
from .validate import validate
|
|
19
|
+
|
|
20
|
+
def shown(path: Path) -> str:
|
|
21
|
+
"""A path relative to the current directory when that's shorter."""
|
|
22
|
+
try:
|
|
23
|
+
rel = os.path.relpath(path)
|
|
24
|
+
except ValueError:
|
|
25
|
+
return str(path)
|
|
26
|
+
return rel if len(rel) < len(str(path)) else str(path)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def cmd_validate(args) -> int:
|
|
30
|
+
root = Path(args.dir)
|
|
31
|
+
if not root.is_dir():
|
|
32
|
+
print(f"dataops: {root} is not a directory", file=sys.stderr)
|
|
33
|
+
return 2
|
|
34
|
+
project = load(root)
|
|
35
|
+
problems = validate(project)
|
|
36
|
+
errors = [p for p in problems if p.severity == "error"]
|
|
37
|
+
|
|
38
|
+
if args.format == "json":
|
|
39
|
+
print(json.dumps([
|
|
40
|
+
{"file": shown(p.file), "line": p.line, "severity": p.severity, "resource": p.resource, "message": p.message}
|
|
41
|
+
for p in problems
|
|
42
|
+
], indent=2))
|
|
43
|
+
else:
|
|
44
|
+
for p in problems:
|
|
45
|
+
where = f"{shown(p.file)}:{p.line}" if p.line else shown(p.file)
|
|
46
|
+
resource = f"{p.resource}: " if p.resource else ""
|
|
47
|
+
print(f"{where}: {p.severity}: {resource}{p.message}")
|
|
48
|
+
warnings = len(problems) - len(errors)
|
|
49
|
+
summary = f"{len(project.resources)} resources in {len(project.files)} files"
|
|
50
|
+
if problems:
|
|
51
|
+
print(f"\n{len(errors)} error(s), {warnings} warning(s); {summary}")
|
|
52
|
+
else:
|
|
53
|
+
print(f"Valid: {summary}")
|
|
54
|
+
return 1 if errors else 0
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def cmd_discover(args) -> int:
|
|
58
|
+
try:
|
|
59
|
+
return discover(
|
|
60
|
+
args.source, args.product, Path(args.dir), fmt=args.format,
|
|
61
|
+
layers=not args.no_layers, use_cache=not args.refresh,
|
|
62
|
+
search=args.search, limit=args.limit, resource=args.resource, url=args.url,
|
|
63
|
+
)
|
|
64
|
+
except DiscoverError as e:
|
|
65
|
+
print(f"dataops discover: {e}", file=sys.stderr)
|
|
66
|
+
return e.code
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def cmd_status(args) -> int:
|
|
70
|
+
try:
|
|
71
|
+
return status(Path(args.dir), fmt=args.format)
|
|
72
|
+
except SyncError as e:
|
|
73
|
+
print(f"dataops status: {e}", file=sys.stderr)
|
|
74
|
+
return e.code
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def cmd_sync(args) -> int:
|
|
78
|
+
try:
|
|
79
|
+
return sync(Path(args.dir), args.datasets, force=args.force, loaded_file=args.loaded)
|
|
80
|
+
except SyncError as e:
|
|
81
|
+
print(f"dataops sync: {e}", file=sys.stderr)
|
|
82
|
+
return e.code
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def main(argv: list[str] | None = None) -> int:
|
|
86
|
+
parser = argparse.ArgumentParser(prog="dataops", description="GeoServer DataOps: the data GeoServer publishes, loaded into PostGIS and kept up to date, from YAML files.")
|
|
87
|
+
parser.add_argument("--version", action="version", version=f"dataops {__version__}")
|
|
88
|
+
commands = parser.add_subparsers(dest="command", metavar="command")
|
|
89
|
+
|
|
90
|
+
v = commands.add_parser("validate", help="check the files against the spec (no network)")
|
|
91
|
+
v.add_argument("dir", nargs="?", default=".", help="project directory (default: current)")
|
|
92
|
+
v.add_argument("--format", choices=["text", "json"], default="text")
|
|
93
|
+
v.set_defaults(func=cmd_validate)
|
|
94
|
+
|
|
95
|
+
d = commands.add_parser(
|
|
96
|
+
"discover", help="list what a source offers, or the layers inside one product",
|
|
97
|
+
description="List the products a source offers, or the downloads and layers of one product. "
|
|
98
|
+
"SOURCE is a Source in the project (e.g. os) or a source type: os-downloads, ckan, wfs, ogc-api-features, arcgis, stac, ogc-api-records, csw, socrata, opendatasoft or koordinates (with --url), overture, geofabrik, "
|
|
99
|
+
"or url (the product is then a file's address).",
|
|
100
|
+
)
|
|
101
|
+
d.add_argument("source", help="a Source name from the project, or a source type")
|
|
102
|
+
d.add_argument("product", nargs="?", help="a product ID (a CKAN package, a file's URL), to list its downloads and layers")
|
|
103
|
+
d.add_argument("--search", metavar="WORDS", help="list only products matching these words")
|
|
104
|
+
d.add_argument("--limit", type=int, metavar="N", help="list at most N products (CKAN default: 20)")
|
|
105
|
+
d.add_argument("--resource", metavar="ID", help="read this download of the product (a CKAN resource, or a catalogue record's link: its ID, name or address)")
|
|
106
|
+
d.add_argument("--url", help="the source's address, e.g. a CKAN portal (overrides the Source's spec.url)")
|
|
107
|
+
d.add_argument("-C", "--dir", default=".", help="project directory, for Sources and secrets (default: current)")
|
|
108
|
+
d.add_argument("--format", choices=["text", "json"], default="text")
|
|
109
|
+
d.add_argument("--no-layers", action="store_true", help="list downloads only; don't read layers from the file")
|
|
110
|
+
d.add_argument("--refresh", action="store_true", help="read layers again instead of using the cache")
|
|
111
|
+
d.set_defaults(func=cmd_discover)
|
|
112
|
+
|
|
113
|
+
st = commands.add_parser("status", help="datasets: loaded, when, next check",
|
|
114
|
+
description="Show the project's datasets from the ingest log. Changes nothing.")
|
|
115
|
+
st.add_argument("-C", "--dir", default=".", help="project directory (default: current)")
|
|
116
|
+
st.add_argument("--format", choices=["text", "json"], default="text")
|
|
117
|
+
st.set_defaults(func=cmd_status)
|
|
118
|
+
|
|
119
|
+
s = commands.add_parser(
|
|
120
|
+
"sync", help="load new versions of datasets into PostGIS",
|
|
121
|
+
description="Check each due Dataset for a new version and load it: stage, check, back up, swap in. "
|
|
122
|
+
"Named datasets are checked whatever their schedule. The database comes from the Project's connection secret.",
|
|
123
|
+
)
|
|
124
|
+
s.add_argument("datasets", nargs="*", help="only these Datasets, due or not")
|
|
125
|
+
s.add_argument("-C", "--dir", default=".", help="project directory (default: current)")
|
|
126
|
+
s.add_argument("--force", action="store_true", help="load even when the version hasn't changed")
|
|
127
|
+
s.add_argument("--loaded", type=Path, metavar="FILE",
|
|
128
|
+
help="write what was loaded to FILE, one per line (schema.table, or a raster's path), for `configops refresh --from FILE`")
|
|
129
|
+
s.set_defaults(func=cmd_sync)
|
|
130
|
+
|
|
131
|
+
argv = sys.argv[1:] if argv is None else argv
|
|
132
|
+
if argv[:1] == ["discover"]:
|
|
133
|
+
# options may come between the source and the product
|
|
134
|
+
args = d.parse_intermixed_args(argv[1:])
|
|
135
|
+
args.command = "discover"
|
|
136
|
+
else:
|
|
137
|
+
args = parser.parse_args(argv)
|
|
138
|
+
if not args.command:
|
|
139
|
+
parser.print_help()
|
|
140
|
+
return 2
|
|
141
|
+
return args.func(args)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Coordinate reference system names, in the short form the spec uses (EPSG:27700)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
|
|
7
|
+
# urn:ogc:def:crs:EPSG::27700, urn:x-ogc:def:crs:EPSG:27700, EPSG:27700,
|
|
8
|
+
# http://www.opengis.net/def/crs/EPSG/0/27700, http://www.opengis.net/gml/srs/epsg.xml#27700
|
|
9
|
+
EPSG = re.compile(r"(?:EPSG(?::[\d.]*)?:|/EPSG/[\d.]+/|epsg\.xml#)(\d+)$", re.IGNORECASE)
|
|
10
|
+
CRS84 = re.compile(r"(?:OGC/1\.3/|OGC:(?:1\.3:)?)CRS84$", re.IGNORECASE)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def short(name: str | None) -> str | None:
|
|
14
|
+
"""EPSG:<code> or CRS84 when recognised, else the name as given."""
|
|
15
|
+
if not name:
|
|
16
|
+
return None
|
|
17
|
+
name = name.strip()
|
|
18
|
+
if CRS84.search(name):
|
|
19
|
+
return "CRS84"
|
|
20
|
+
m = EPSG.search(name)
|
|
21
|
+
return f"EPSG:{m.group(1)}" if m else name
|