geoserver-dataops 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. geoserver_dataops-0.2.0/LICENSE +21 -0
  2. geoserver_dataops-0.2.0/PKG-INFO +151 -0
  3. geoserver_dataops-0.2.0/README.md +130 -0
  4. geoserver_dataops-0.2.0/geoserver_dataops/__init__.py +4 -0
  5. geoserver_dataops-0.2.0/geoserver_dataops/__main__.py +3 -0
  6. geoserver_dataops-0.2.0/geoserver_dataops/cli.py +141 -0
  7. geoserver_dataops-0.2.0/geoserver_dataops/crs.py +21 -0
  8. geoserver_dataops-0.2.0/geoserver_dataops/discover.py +324 -0
  9. geoserver_dataops-0.2.0/geoserver_dataops/ingest.py +509 -0
  10. geoserver_dataops-0.2.0/geoserver_dataops/loader.py +132 -0
  11. geoserver_dataops-0.2.0/geoserver_dataops/raster.py +165 -0
  12. geoserver_dataops-0.2.0/geoserver_dataops/remote.py +260 -0
  13. geoserver_dataops-0.2.0/geoserver_dataops/schedule.py +96 -0
  14. geoserver_dataops-0.2.0/geoserver_dataops/schema/v1alpha1.json +346 -0
  15. geoserver_dataops-0.2.0/geoserver_dataops/secrets.py +65 -0
  16. geoserver_dataops-0.2.0/geoserver_dataops/sources/__init__.py +121 -0
  17. geoserver_dataops-0.2.0/geoserver_dataops/sources/arcgis.py +219 -0
  18. geoserver_dataops-0.2.0/geoserver_dataops/sources/catalogue.py +259 -0
  19. geoserver_dataops-0.2.0/geoserver_dataops/sources/ckan.py +135 -0
  20. geoserver_dataops-0.2.0/geoserver_dataops/sources/formats.py +104 -0
  21. geoserver_dataops-0.2.0/geoserver_dataops/sources/geofabrik.py +102 -0
  22. geoserver_dataops-0.2.0/geoserver_dataops/sources/koordinates.py +127 -0
  23. geoserver_dataops-0.2.0/geoserver_dataops/sources/ogcapi.py +115 -0
  24. geoserver_dataops-0.2.0/geoserver_dataops/sources/opendata.py +152 -0
  25. geoserver_dataops-0.2.0/geoserver_dataops/sources/os_downloads.py +134 -0
  26. geoserver_dataops-0.2.0/geoserver_dataops/sources/overture.py +205 -0
  27. geoserver_dataops-0.2.0/geoserver_dataops/sources/stac.py +184 -0
  28. geoserver_dataops-0.2.0/geoserver_dataops/sources/url.py +99 -0
  29. geoserver_dataops-0.2.0/geoserver_dataops/sources/wfs.py +123 -0
  30. geoserver_dataops-0.2.0/geoserver_dataops/status.py +90 -0
  31. geoserver_dataops-0.2.0/geoserver_dataops/sync.py +315 -0
  32. geoserver_dataops-0.2.0/geoserver_dataops/validate.py +212 -0
  33. geoserver_dataops-0.2.0/geoserver_dataops.egg-info/PKG-INFO +151 -0
  34. geoserver_dataops-0.2.0/geoserver_dataops.egg-info/SOURCES.txt +67 -0
  35. geoserver_dataops-0.2.0/geoserver_dataops.egg-info/dependency_links.txt +1 -0
  36. geoserver_dataops-0.2.0/geoserver_dataops.egg-info/entry_points.txt +2 -0
  37. geoserver_dataops-0.2.0/geoserver_dataops.egg-info/requires.txt +13 -0
  38. geoserver_dataops-0.2.0/geoserver_dataops.egg-info/top_level.txt +1 -0
  39. geoserver_dataops-0.2.0/pyproject.toml +35 -0
  40. geoserver_dataops-0.2.0/setup.cfg +4 -0
  41. geoserver_dataops-0.2.0/tests/test_arcgis.py +96 -0
  42. geoserver_dataops-0.2.0/tests/test_arcgis_integration.py +66 -0
  43. geoserver_dataops-0.2.0/tests/test_catalogue.py +112 -0
  44. geoserver_dataops-0.2.0/tests/test_catalogue_integration.py +48 -0
  45. geoserver_dataops-0.2.0/tests/test_discover.py +677 -0
  46. geoserver_dataops-0.2.0/tests/test_discover_services.py +331 -0
  47. geoserver_dataops-0.2.0/tests/test_download.py +79 -0
  48. geoserver_dataops-0.2.0/tests/test_e2e_geoserver.py +212 -0
  49. geoserver_dataops-0.2.0/tests/test_geofabrik.py +100 -0
  50. geoserver_dataops-0.2.0/tests/test_idle_integration.py +43 -0
  51. geoserver_dataops-0.2.0/tests/test_koordinates.py +92 -0
  52. geoserver_dataops-0.2.0/tests/test_lock_integration.py +57 -0
  53. geoserver_dataops-0.2.0/tests/test_log_integration.py +20 -0
  54. geoserver_dataops-0.2.0/tests/test_opendata.py +54 -0
  55. geoserver_dataops-0.2.0/tests/test_opendata_integration.py +50 -0
  56. geoserver_dataops-0.2.0/tests/test_overture.py +76 -0
  57. geoserver_dataops-0.2.0/tests/test_overture_integration.py +46 -0
  58. geoserver_dataops-0.2.0/tests/test_raster.py +170 -0
  59. geoserver_dataops-0.2.0/tests/test_raster_backups.py +31 -0
  60. geoserver_dataops-0.2.0/tests/test_raster_sync_integration.py +44 -0
  61. geoserver_dataops-0.2.0/tests/test_run_integration.py +133 -0
  62. geoserver_dataops-0.2.0/tests/test_schedule.py +20 -0
  63. geoserver_dataops-0.2.0/tests/test_stac.py +137 -0
  64. geoserver_dataops-0.2.0/tests/test_stac_integration.py +84 -0
  65. geoserver_dataops-0.2.0/tests/test_swap_integration.py +87 -0
  66. geoserver_dataops-0.2.0/tests/test_sync.py +175 -0
  67. geoserver_dataops-0.2.0/tests/test_sync_integration.py +251 -0
  68. geoserver_dataops-0.2.0/tests/test_validate.py +50 -0
  69. geoserver_dataops-0.2.0/tests/test_versions.py +49 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Nathan Leeks
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,151 @@
1
+ Metadata-Version: 2.4
2
+ Name: geoserver-dataops
3
+ Version: 0.2.0
4
+ Summary: The data GeoServer publishes, loaded into PostGIS and kept up to date from YAML files: ArcGIS, CKAN, STAC, Overture, OpenStreetMap, WFS, OGC API and more
5
+ License-Expression: MIT
6
+ Requires-Python: >=3.10
7
+ Description-Content-Type: text/markdown
8
+ License-File: LICENSE
9
+ Requires-Dist: pyyaml>=6
10
+ Requires-Dist: jsonschema>=4.18
11
+ Requires-Dist: requests>=2.28
12
+ Requires-Dist: psycopg2-binary>=2.9
13
+ Provides-Extra: dev
14
+ Requires-Dist: pytest; extra == "dev"
15
+ Requires-Dist: pyarrow>=14; extra == "dev"
16
+ Requires-Dist: shapely>=2; extra == "dev"
17
+ Provides-Extra: overture
18
+ Requires-Dist: pyarrow>=14; extra == "overture"
19
+ Requires-Dist: shapely>=2; extra == "overture"
20
+ Dynamic: license-file
21
+
22
+ <p align="center"><img src="docs/logo.svg" alt="GeoServer DataOps" width="120"></p>
23
+
24
+ <h1 align="center">GeoServer DataOps</h1>
25
+
26
+ <p align="center">The data GeoServer publishes, loaded into PostGIS and kept up to date, from YAML files.<br>For <a href="https://github.com/geoserver/geoserver-cloud">GeoServer Cloud</a> and standard GeoServer, beside <a href="https://github.com/Yibbzz/geoserver-configops">GeoServer ConfigOps</a>.</p>
27
+
28
+ <!-- badges -->
29
+ ![GeoServer Cloud 3.0.2.1](https://img.shields.io/badge/GeoServer_Cloud-3.0.2.1-brightgreen) ![GeoServer 3.0.2](https://img.shields.io/badge/GeoServer-3.0.2-brightgreen) ![GeoServer 2.28.6](https://img.shields.io/badge/GeoServer-2.28.6-brightgreen) ![PostGIS 18-3.7](https://img.shields.io/badge/PostGIS-18--3.7-blue)
30
+ <!-- /badges -->
31
+
32
+ <!-- tested-on -->
33
+ Tested on **GeoServer Cloud 3.0.2.1**, **GeoServer 3.0.2** and **GeoServer 2.28.6**, with **PostGIS 18-3.7**: on every pull request each one publishes what `dataops sync` loaded and serves it back, before and after a reload, and a daily job (`.github/workflows/versions.yaml`) tries each new release.
34
+ <!-- /tested-on -->
35
+
36
+ `dataops` keeps the open datasets you name in PostGIS, each at its newest version:
37
+
38
+ - **Find** data in the places it is published, worldwide: ArcGIS services, CKAN, Socrata and OpenDataSoft portals, INSPIRE and GeoNetwork catalogues (CSW, OGC API Records), WFS and OGC API Features, STAC imagery catalogues, Overture Maps, OpenStreetMap (Geofabrik), New Zealand's Koordinates services, the Ordnance Survey, or plain links. `discover` lists what a source offers and the layers inside a download.
39
+ - **Load** it safely: staged, checked (invalid geometries, row counts), then swapped in for the old tables in one transaction, which are kept as backups. A raster download becomes one Cloud-Optimised GeoTIFF.
40
+ - **Keep it up to date.** Each dataset has a schedule; `sync`, run nightly, checks those that are due and loads only a new version.
41
+
42
+ It loads the data; [GeoServer ConfigOps](https://github.com/Yibbzz/geoserver-configops) configures GeoServer to publish it, and `configops refresh` brings GeoServer up to date after each load.
43
+
44
+ > **Status: 0.x.** The file format may still change between releases. Formerly **geodata-sync**: the command was `geodata`, the files' `apiVersion` `geodata/v1alpha1` and the variables `GEODATA_*`; [docs/development.md](docs/development.md#renamed-from-geodata-sync) says what to change.
45
+
46
+ ## Install
47
+
48
+ ```bash
49
+ pip install geoserver-dataops # Python 3.10+, and GDAL's tools (apt install gdal-bin)
50
+ docker run --rm ghcr.io/yibbzz/geoserver-dataops --help # or the image, GDAL included
51
+ ```
52
+
53
+ ## Example
54
+
55
+ [`examples/quickstart/project.yaml`](examples/quickstart/project.yaml): Surrey Heath's conservation areas, from data.gov.uk, checked monthly.
56
+
57
+ ```yaml
58
+ apiVersion: dataops/v1alpha1
59
+ kind: Project
60
+ metadata: { name: quickstart }
61
+ spec:
62
+ database:
63
+ connection: { secret: database } # keys: host, port, database, username, password
64
+ crs: EPSG:27700
65
+ ---
66
+ apiVersion: dataops/v1alpha1
67
+ kind: Source
68
+ metadata: { name: data-gov-uk }
69
+ spec: { type: ckan, url: https://data.gov.uk }
70
+ ---
71
+ apiVersion: dataops/v1alpha1
72
+ kind: Dataset
73
+ metadata: { name: conservation-areas }
74
+ spec:
75
+ source: data-gov-uk
76
+ product: conservation-areas125 # `dataops discover data-gov-uk --search "conservation areas"` finds it
77
+ update: monthly
78
+ tables:
79
+ conservation_areas: {} # the layer in the download; table conservation_areas_conservation_areas
80
+ ```
81
+
82
+ ```console
83
+ $ docker compose -f examples/compose.yaml up -d --wait
84
+
85
+ $ cat > examples/quickstart/secrets.yaml <<'EOF'
86
+ database: { host: localhost, port: "5433", database: postgres, username: postgres, password: postgres }
87
+ EOF
88
+
89
+ $ dataops sync -C examples/quickstart
90
+ conservation-areas: loaded 2026-11-11 in 0s: data_layers.conservation_areas_conservation_areas (10 rows)
91
+ note: 1 invalid geometry left out
92
+ $ dataops status -C examples/quickstart
93
+ DATASET UPDATE LOADED AT ROWS NEXT CHECK LAST CHECK
94
+ conservation-areas monthly 2026-11-11 2026-11-13 08:51 10 2026-12-13 07:51 success
95
+ $ dataops sync -C examples/quickstart
96
+ conservation-areas: not due (monthly)
97
+ ```
98
+
99
+ `docker compose -f examples/compose.yaml down -v` removes the database. Two larger examples: [`examples/uk-basemap`](examples/uk-basemap/) (OS Open Roads, Boundary-Line and the Environment Agency's flood zones: 3 GB, 20 minutes) and [`examples/terrain`](examples/terrain/) (OS Terrain 50 as one Cloud-Optimised GeoTIFF).
100
+
101
+ ## Common tasks
102
+
103
+ **Find a dataset** and the layers inside it:
104
+
105
+ ```bash
106
+ dataops discover os-downloads # what a source type offers
107
+ dataops discover data-gov-uk --search "flood zones" -C my-data # search a Source of the project
108
+ dataops discover data-gov-uk <product> -C my-data # its files, and their layers
109
+ ```
110
+
111
+ **Load now** rather than when due: `dataops sync -C my-data <dataset>`; `--force` loads even when the version hasn't changed.
112
+
113
+ **Refresh GeoServer after a load.** `sync --loaded` lists what it loaded, and `configops refresh` works out those layers' bounds again and clears their cached tiles:
114
+
115
+ ```bash
116
+ dataops sync -C my-data --loaded loaded.txt
117
+ configops refresh -C my-geoserver --from loaded.txt
118
+ ```
119
+
120
+ **Keep logins out of the files.** A file names a Secret (`connection: { secret: database }`); its value comes from a Kubernetes Secret, a `DATAOPS_<SECRET>_<KEY>` environment variable or a git-ignored `secrets.yaml`. Credentials in addresses are masked in errors and the ingest log.
121
+
122
+ **Run it in Kubernetes** as a nightly CronJob that loads, then refreshes GeoServer: [docs/kubernetes.md](docs/kubernetes.md).
123
+
124
+ ## Built to be safe to run
125
+
126
+ - **A load that fails changes nothing.** Live tables are only swapped once every table of the dataset has loaded and passed its checks.
127
+ - **The old data is kept.** Each swap moves the old table to a backup schema; the newest `backups` are kept.
128
+ - **What reads the tables keeps working.** The new table gets the old one's grants, and views on it are moved across.
129
+ - **One sync at a time.** A run takes a lock in PostgreSQL, so a nightly job and a person can't load through each other.
130
+ - **Downloads aren't trusted.** GDAL may only open them as data formats, and what an archive unpacks to is limited.
131
+ - **Checked before it runs.** `dataops validate` reads the files without a network; problems come out as `file:line`.
132
+
133
+ ## What to know
134
+
135
+ - Big layers over WFS or OGC API Features don't load reliably: use a file download when there is one ([docs/sources-notes.md](docs/sources-notes.md)).
136
+ - A WFS or OGC API Features source has no version to compare, so it is loaded in full each time it is due; so is an ArcGIS layer that doesn't report its last edit.
137
+ - Overture reads GeoParquet with pyarrow: `pip install 'geoserver-dataops[overture]'` (the image has it). Koordinates needs an API key.
138
+ - A materialized view on a live table stops its load: drop it before and create it again after.
139
+
140
+ ## Documentation
141
+
142
+ | | |
143
+ | --- | --- |
144
+ | [docs/spec.md](docs/spec.md) | The file format and every command |
145
+ | [docs/kubernetes.md](docs/kubernetes.md) | Running it nightly in a cluster |
146
+ | [docs/sources-notes.md](docs/sources-notes.md) | What we learned about the sources |
147
+ | [docs/development.md](docs/development.md) | Tests, CI, new GeoServer versions, releasing |
148
+
149
+ ## Licence
150
+
151
+ MIT (see `LICENSE`). Data you load keeps its own licence: OS OpenData is under the [Open Government Licence v3.0](https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/) and needs "Contains OS data © Crown copyright and database right &lt;year&gt;" wherever it's shown.
@@ -0,0 +1,130 @@
1
+ <p align="center"><img src="docs/logo.svg" alt="GeoServer DataOps" width="120"></p>
2
+
3
+ <h1 align="center">GeoServer DataOps</h1>
4
+
5
+ <p align="center">The data GeoServer publishes, loaded into PostGIS and kept up to date, from YAML files.<br>For <a href="https://github.com/geoserver/geoserver-cloud">GeoServer Cloud</a> and standard GeoServer, beside <a href="https://github.com/Yibbzz/geoserver-configops">GeoServer ConfigOps</a>.</p>
6
+
7
+ <!-- badges -->
8
+ ![GeoServer Cloud 3.0.2.1](https://img.shields.io/badge/GeoServer_Cloud-3.0.2.1-brightgreen) ![GeoServer 3.0.2](https://img.shields.io/badge/GeoServer-3.0.2-brightgreen) ![GeoServer 2.28.6](https://img.shields.io/badge/GeoServer-2.28.6-brightgreen) ![PostGIS 18-3.7](https://img.shields.io/badge/PostGIS-18--3.7-blue)
9
+ <!-- /badges -->
10
+
11
+ <!-- tested-on -->
12
+ Tested on **GeoServer Cloud 3.0.2.1**, **GeoServer 3.0.2** and **GeoServer 2.28.6**, with **PostGIS 18-3.7**: on every pull request each one publishes what `dataops sync` loaded and serves it back, before and after a reload, and a daily job (`.github/workflows/versions.yaml`) tries each new release.
13
+ <!-- /tested-on -->
14
+
15
+ `dataops` keeps the open datasets you name in PostGIS, each at its newest version:
16
+
17
+ - **Find** data in the places it is published, worldwide: ArcGIS services, CKAN, Socrata and OpenDataSoft portals, INSPIRE and GeoNetwork catalogues (CSW, OGC API Records), WFS and OGC API Features, STAC imagery catalogues, Overture Maps, OpenStreetMap (Geofabrik), New Zealand's Koordinates services, the Ordnance Survey, or plain links. `discover` lists what a source offers and the layers inside a download.
18
+ - **Load** it safely: staged, checked (invalid geometries, row counts), then swapped in for the old tables in one transaction, which are kept as backups. A raster download becomes one Cloud-Optimised GeoTIFF.
19
+ - **Keep it up to date.** Each dataset has a schedule; `sync`, run nightly, checks those that are due and loads only a new version.
20
+
21
+ It loads the data; [GeoServer ConfigOps](https://github.com/Yibbzz/geoserver-configops) configures GeoServer to publish it, and `configops refresh` brings GeoServer up to date after each load.
22
+
23
+ > **Status: 0.x.** The file format may still change between releases. Formerly **geodata-sync**: the command was `geodata`, the files' `apiVersion` `geodata/v1alpha1` and the variables `GEODATA_*`; [docs/development.md](docs/development.md#renamed-from-geodata-sync) says what to change.
24
+
25
+ ## Install
26
+
27
+ ```bash
28
+ pip install geoserver-dataops # Python 3.10+, and GDAL's tools (apt install gdal-bin)
29
+ docker run --rm ghcr.io/yibbzz/geoserver-dataops --help # or the image, GDAL included
30
+ ```
31
+
32
+ ## Example
33
+
34
+ [`examples/quickstart/project.yaml`](examples/quickstart/project.yaml): Surrey Heath's conservation areas, from data.gov.uk, checked monthly.
35
+
36
+ ```yaml
37
+ apiVersion: dataops/v1alpha1
38
+ kind: Project
39
+ metadata: { name: quickstart }
40
+ spec:
41
+ database:
42
+ connection: { secret: database } # keys: host, port, database, username, password
43
+ crs: EPSG:27700
44
+ ---
45
+ apiVersion: dataops/v1alpha1
46
+ kind: Source
47
+ metadata: { name: data-gov-uk }
48
+ spec: { type: ckan, url: https://data.gov.uk }
49
+ ---
50
+ apiVersion: dataops/v1alpha1
51
+ kind: Dataset
52
+ metadata: { name: conservation-areas }
53
+ spec:
54
+ source: data-gov-uk
55
+ product: conservation-areas125 # `dataops discover data-gov-uk --search "conservation areas"` finds it
56
+ update: monthly
57
+ tables:
58
+ conservation_areas: {} # the layer in the download; table conservation_areas_conservation_areas
59
+ ```
60
+
61
+ ```console
62
+ $ docker compose -f examples/compose.yaml up -d --wait
63
+
64
+ $ cat > examples/quickstart/secrets.yaml <<'EOF'
65
+ database: { host: localhost, port: "5433", database: postgres, username: postgres, password: postgres }
66
+ EOF
67
+
68
+ $ dataops sync -C examples/quickstart
69
+ conservation-areas: loaded 2026-11-11 in 0s: data_layers.conservation_areas_conservation_areas (10 rows)
70
+ note: 1 invalid geometry left out
71
+ $ dataops status -C examples/quickstart
72
+ DATASET UPDATE LOADED AT ROWS NEXT CHECK LAST CHECK
73
+ conservation-areas monthly 2026-11-11 2026-11-13 08:51 10 2026-12-13 07:51 success
74
+ $ dataops sync -C examples/quickstart
75
+ conservation-areas: not due (monthly)
76
+ ```
77
+
78
+ `docker compose -f examples/compose.yaml down -v` removes the database. Two larger examples: [`examples/uk-basemap`](examples/uk-basemap/) (OS Open Roads, Boundary-Line and the Environment Agency's flood zones: 3 GB, 20 minutes) and [`examples/terrain`](examples/terrain/) (OS Terrain 50 as one Cloud-Optimised GeoTIFF).
79
+
80
+ ## Common tasks
81
+
82
+ **Find a dataset** and the layers inside it:
83
+
84
+ ```bash
85
+ dataops discover os-downloads # what a source type offers
86
+ dataops discover data-gov-uk --search "flood zones" -C my-data # search a Source of the project
87
+ dataops discover data-gov-uk <product> -C my-data # its files, and their layers
88
+ ```
89
+
90
+ **Load now** rather than when due: `dataops sync -C my-data <dataset>`; `--force` loads even when the version hasn't changed.
91
+
92
+ **Refresh GeoServer after a load.** `sync --loaded` lists what it loaded, and `configops refresh` works out those layers' bounds again and clears their cached tiles:
93
+
94
+ ```bash
95
+ dataops sync -C my-data --loaded loaded.txt
96
+ configops refresh -C my-geoserver --from loaded.txt
97
+ ```
98
+
99
+ **Keep logins out of the files.** A file names a Secret (`connection: { secret: database }`); its value comes from a Kubernetes Secret, a `DATAOPS_<SECRET>_<KEY>` environment variable or a git-ignored `secrets.yaml`. Credentials in addresses are masked in errors and the ingest log.
100
+
101
+ **Run it in Kubernetes** as a nightly CronJob that loads, then refreshes GeoServer: [docs/kubernetes.md](docs/kubernetes.md).
102
+
103
+ ## Built to be safe to run
104
+
105
+ - **A load that fails changes nothing.** Live tables are only swapped once every table of the dataset has loaded and passed its checks.
106
+ - **The old data is kept.** Each swap moves the old table to a backup schema; the newest `backups` are kept.
107
+ - **What reads the tables keeps working.** The new table gets the old one's grants, and views on it are moved across.
108
+ - **One sync at a time.** A run takes a lock in PostgreSQL, so a nightly job and a person can't load through each other.
109
+ - **Downloads aren't trusted.** GDAL may only open them as data formats, and what an archive unpacks to is limited.
110
+ - **Checked before it runs.** `dataops validate` reads the files without a network; problems come out as `file:line`.
111
+
112
+ ## What to know
113
+
114
+ - Big layers over WFS or OGC API Features don't load reliably: use a file download when there is one ([docs/sources-notes.md](docs/sources-notes.md)).
115
+ - A WFS or OGC API Features source has no version to compare, so it is loaded in full each time it is due; so is an ArcGIS layer that doesn't report its last edit.
116
+ - Overture reads GeoParquet with pyarrow: `pip install 'geoserver-dataops[overture]'` (the image has it). Koordinates needs an API key.
117
+ - A materialized view on a live table stops its load: drop it before and create it again after.
118
+
119
+ ## Documentation
120
+
121
+ | | |
122
+ | --- | --- |
123
+ | [docs/spec.md](docs/spec.md) | The file format and every command |
124
+ | [docs/kubernetes.md](docs/kubernetes.md) | Running it nightly in a cluster |
125
+ | [docs/sources-notes.md](docs/sources-notes.md) | What we learned about the sources |
126
+ | [docs/development.md](docs/development.md) | Tests, CI, new GeoServer versions, releasing |
127
+
128
+ ## Licence
129
+
130
+ MIT (see `LICENSE`). Data you load keeps its own licence: OS OpenData is under the [Open Government Licence v3.0](https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/) and needs "Contains OS data © Crown copyright and database right &lt;year&gt;" wherever it's shown.
@@ -0,0 +1,4 @@
1
+ """geoserver-dataops: open GIS data loaded into PostGIS, and kept up to date, from YAML files."""
2
+
3
+ __version__ = "0.2.0"
4
+ API_VERSION = "dataops/v1alpha1"
@@ -0,0 +1,3 @@
1
+ from .cli import main
2
+
3
+ raise SystemExit(main())
@@ -0,0 +1,141 @@
1
+ """The dataops command."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ import os
8
+ import sys
9
+ from pathlib import Path
10
+
11
+ from . import __version__
12
+ from .discover import DiscoverError
13
+ from .discover import run as discover
14
+ from .status import run as status
15
+ from .sync import SyncError
16
+ from .sync import run as sync
17
+ from .loader import load
18
+ from .validate import validate
19
+
20
+ def shown(path: Path) -> str:
21
+ """A path relative to the current directory when that's shorter."""
22
+ try:
23
+ rel = os.path.relpath(path)
24
+ except ValueError:
25
+ return str(path)
26
+ return rel if len(rel) < len(str(path)) else str(path)
27
+
28
+
29
+ def cmd_validate(args) -> int:
30
+ root = Path(args.dir)
31
+ if not root.is_dir():
32
+ print(f"dataops: {root} is not a directory", file=sys.stderr)
33
+ return 2
34
+ project = load(root)
35
+ problems = validate(project)
36
+ errors = [p for p in problems if p.severity == "error"]
37
+
38
+ if args.format == "json":
39
+ print(json.dumps([
40
+ {"file": shown(p.file), "line": p.line, "severity": p.severity, "resource": p.resource, "message": p.message}
41
+ for p in problems
42
+ ], indent=2))
43
+ else:
44
+ for p in problems:
45
+ where = f"{shown(p.file)}:{p.line}" if p.line else shown(p.file)
46
+ resource = f"{p.resource}: " if p.resource else ""
47
+ print(f"{where}: {p.severity}: {resource}{p.message}")
48
+ warnings = len(problems) - len(errors)
49
+ summary = f"{len(project.resources)} resources in {len(project.files)} files"
50
+ if problems:
51
+ print(f"\n{len(errors)} error(s), {warnings} warning(s); {summary}")
52
+ else:
53
+ print(f"Valid: {summary}")
54
+ return 1 if errors else 0
55
+
56
+
57
+ def cmd_discover(args) -> int:
58
+ try:
59
+ return discover(
60
+ args.source, args.product, Path(args.dir), fmt=args.format,
61
+ layers=not args.no_layers, use_cache=not args.refresh,
62
+ search=args.search, limit=args.limit, resource=args.resource, url=args.url,
63
+ )
64
+ except DiscoverError as e:
65
+ print(f"dataops discover: {e}", file=sys.stderr)
66
+ return e.code
67
+
68
+
69
+ def cmd_status(args) -> int:
70
+ try:
71
+ return status(Path(args.dir), fmt=args.format)
72
+ except SyncError as e:
73
+ print(f"dataops status: {e}", file=sys.stderr)
74
+ return e.code
75
+
76
+
77
+ def cmd_sync(args) -> int:
78
+ try:
79
+ return sync(Path(args.dir), args.datasets, force=args.force, loaded_file=args.loaded)
80
+ except SyncError as e:
81
+ print(f"dataops sync: {e}", file=sys.stderr)
82
+ return e.code
83
+
84
+
85
+ def main(argv: list[str] | None = None) -> int:
86
+ parser = argparse.ArgumentParser(prog="dataops", description="GeoServer DataOps: the data GeoServer publishes, loaded into PostGIS and kept up to date, from YAML files.")
87
+ parser.add_argument("--version", action="version", version=f"dataops {__version__}")
88
+ commands = parser.add_subparsers(dest="command", metavar="command")
89
+
90
+ v = commands.add_parser("validate", help="check the files against the spec (no network)")
91
+ v.add_argument("dir", nargs="?", default=".", help="project directory (default: current)")
92
+ v.add_argument("--format", choices=["text", "json"], default="text")
93
+ v.set_defaults(func=cmd_validate)
94
+
95
+ d = commands.add_parser(
96
+ "discover", help="list what a source offers, or the layers inside one product",
97
+ description="List the products a source offers, or the downloads and layers of one product. "
98
+ "SOURCE is a Source in the project (e.g. os) or a source type: os-downloads, ckan, wfs, ogc-api-features, arcgis, stac, ogc-api-records, csw, socrata, opendatasoft or koordinates (with --url), overture, geofabrik, "
99
+ "or url (the product is then a file's address).",
100
+ )
101
+ d.add_argument("source", help="a Source name from the project, or a source type")
102
+ d.add_argument("product", nargs="?", help="a product ID (a CKAN package, a file's URL), to list its downloads and layers")
103
+ d.add_argument("--search", metavar="WORDS", help="list only products matching these words")
104
+ d.add_argument("--limit", type=int, metavar="N", help="list at most N products (CKAN default: 20)")
105
+ d.add_argument("--resource", metavar="ID", help="read this download of the product (a CKAN resource, or a catalogue record's link: its ID, name or address)")
106
+ d.add_argument("--url", help="the source's address, e.g. a CKAN portal (overrides the Source's spec.url)")
107
+ d.add_argument("-C", "--dir", default=".", help="project directory, for Sources and secrets (default: current)")
108
+ d.add_argument("--format", choices=["text", "json"], default="text")
109
+ d.add_argument("--no-layers", action="store_true", help="list downloads only; don't read layers from the file")
110
+ d.add_argument("--refresh", action="store_true", help="read layers again instead of using the cache")
111
+ d.set_defaults(func=cmd_discover)
112
+
113
+ st = commands.add_parser("status", help="datasets: loaded, when, next check",
114
+ description="Show the project's datasets from the ingest log. Changes nothing.")
115
+ st.add_argument("-C", "--dir", default=".", help="project directory (default: current)")
116
+ st.add_argument("--format", choices=["text", "json"], default="text")
117
+ st.set_defaults(func=cmd_status)
118
+
119
+ s = commands.add_parser(
120
+ "sync", help="load new versions of datasets into PostGIS",
121
+ description="Check each due Dataset for a new version and load it: stage, check, back up, swap in. "
122
+ "Named datasets are checked whatever their schedule. The database comes from the Project's connection secret.",
123
+ )
124
+ s.add_argument("datasets", nargs="*", help="only these Datasets, due or not")
125
+ s.add_argument("-C", "--dir", default=".", help="project directory (default: current)")
126
+ s.add_argument("--force", action="store_true", help="load even when the version hasn't changed")
127
+ s.add_argument("--loaded", type=Path, metavar="FILE",
128
+ help="write what was loaded to FILE, one per line (schema.table, or a raster's path), for `configops refresh --from FILE`")
129
+ s.set_defaults(func=cmd_sync)
130
+
131
+ argv = sys.argv[1:] if argv is None else argv
132
+ if argv[:1] == ["discover"]:
133
+ # options may come between the source and the product
134
+ args = d.parse_intermixed_args(argv[1:])
135
+ args.command = "discover"
136
+ else:
137
+ args = parser.parse_args(argv)
138
+ if not args.command:
139
+ parser.print_help()
140
+ return 2
141
+ return args.func(args)
@@ -0,0 +1,21 @@
1
+ """Coordinate reference system names, in the short form the spec uses (EPSG:27700)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+
7
+ # urn:ogc:def:crs:EPSG::27700, urn:x-ogc:def:crs:EPSG:27700, EPSG:27700,
8
+ # http://www.opengis.net/def/crs/EPSG/0/27700, http://www.opengis.net/gml/srs/epsg.xml#27700
9
+ EPSG = re.compile(r"(?:EPSG(?::[\d.]*)?:|/EPSG/[\d.]+/|epsg\.xml#)(\d+)$", re.IGNORECASE)
10
+ CRS84 = re.compile(r"(?:OGC/1\.3/|OGC:(?:1\.3:)?)CRS84$", re.IGNORECASE)
11
+
12
+
13
+ def short(name: str | None) -> str | None:
14
+ """EPSG:<code> or CRS84 when recognised, else the name as given."""
15
+ if not name:
16
+ return None
17
+ name = name.strip()
18
+ if CRS84.search(name):
19
+ return "CRS84"
20
+ m = EPSG.search(name)
21
+ return f"EPSG:{m.group(1)}" if m else name