geoserver-dataops 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- geoserver_dataops/__init__.py +4 -0
- geoserver_dataops/__main__.py +3 -0
- geoserver_dataops/cli.py +141 -0
- geoserver_dataops/crs.py +21 -0
- geoserver_dataops/discover.py +324 -0
- geoserver_dataops/ingest.py +509 -0
- geoserver_dataops/loader.py +132 -0
- geoserver_dataops/raster.py +165 -0
- geoserver_dataops/remote.py +260 -0
- geoserver_dataops/schedule.py +96 -0
- geoserver_dataops/schema/v1alpha1.json +346 -0
- geoserver_dataops/secrets.py +65 -0
- geoserver_dataops/sources/__init__.py +121 -0
- geoserver_dataops/sources/arcgis.py +219 -0
- geoserver_dataops/sources/catalogue.py +259 -0
- geoserver_dataops/sources/ckan.py +135 -0
- geoserver_dataops/sources/formats.py +104 -0
- geoserver_dataops/sources/geofabrik.py +102 -0
- geoserver_dataops/sources/koordinates.py +127 -0
- geoserver_dataops/sources/ogcapi.py +115 -0
- geoserver_dataops/sources/opendata.py +152 -0
- geoserver_dataops/sources/os_downloads.py +134 -0
- geoserver_dataops/sources/overture.py +205 -0
- geoserver_dataops/sources/stac.py +184 -0
- geoserver_dataops/sources/url.py +99 -0
- geoserver_dataops/sources/wfs.py +123 -0
- geoserver_dataops/status.py +90 -0
- geoserver_dataops/sync.py +315 -0
- geoserver_dataops/validate.py +212 -0
- geoserver_dataops-0.2.0.dist-info/METADATA +151 -0
- geoserver_dataops-0.2.0.dist-info/RECORD +35 -0
- geoserver_dataops-0.2.0.dist-info/WHEEL +5 -0
- geoserver_dataops-0.2.0.dist-info/entry_points.txt +2 -0
- geoserver_dataops-0.2.0.dist-info/licenses/LICENSE +21 -0
- geoserver_dataops-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,509 @@
|
|
|
1
|
+
"""Loading tables into PostGIS: stage, check, then swap in, with backups.
|
|
2
|
+
|
|
3
|
+
Every layer is loaded with ogr2ogr into the staging schema under a unique
|
|
4
|
+
name, checked (invalid geometries dropped, row count), then all of a
|
|
5
|
+
dataset's tables go live in one transaction: each old table is renamed into
|
|
6
|
+
the backup schema and the new one renamed into place. Nothing is copied,
|
|
7
|
+
and if anything fails before the swap, every live table stays as it was.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import hashlib
|
|
13
|
+
import os
|
|
14
|
+
import re
|
|
15
|
+
import shutil
|
|
16
|
+
import subprocess
|
|
17
|
+
import time
|
|
18
|
+
import uuid
|
|
19
|
+
import zipfile
|
|
20
|
+
from dataclasses import dataclass, field
|
|
21
|
+
from datetime import datetime, timedelta
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
import psycopg2
|
|
25
|
+
import psycopg2.errors
|
|
26
|
+
import requests
|
|
27
|
+
|
|
28
|
+
from . import remote
|
|
29
|
+
|
|
30
|
+
LOG_TABLE = "public.os_ingest_log" # the name the first ingest pipeline gave it; kept so history carries on
|
|
31
|
+
TIMEOUT = 120
|
|
32
|
+
|
|
33
|
+
# A date PostgreSQL rejects (shapefiles store "no date" as 0000/00/00): load dates as text instead
|
|
34
|
+
BAD_DATE = re.compile(r"date/time field value out of range|invalid input syntax for type (date|time|timestamp)|column \w+: \"0000", re.IGNORECASE)
|
|
35
|
+
DATES_AS_TEXT = ["-mapFieldType", "Date=String,Time=String,DateTime=String"]
|
|
36
|
+
|
|
37
|
+
BOOTSTRAP_SQL = f"""
|
|
38
|
+
CREATE TABLE IF NOT EXISTS {LOG_TABLE} (
|
|
39
|
+
id SERIAL PRIMARY KEY,
|
|
40
|
+
dataset_id TEXT NOT NULL,
|
|
41
|
+
version TEXT NOT NULL,
|
|
42
|
+
format_used TEXT,
|
|
43
|
+
status TEXT NOT NULL, -- success | failed | skipped
|
|
44
|
+
rows_loaded BIGINT,
|
|
45
|
+
rows_filtered BIGINT, -- invalid geometries left out
|
|
46
|
+
sha256 TEXT,
|
|
47
|
+
error TEXT,
|
|
48
|
+
started_at TIMESTAMP NOT NULL DEFAULT now(),
|
|
49
|
+
finished_at TIMESTAMP
|
|
50
|
+
);
|
|
51
|
+
ALTER TABLE {LOG_TABLE} ADD COLUMN IF NOT EXISTS tables TEXT;
|
|
52
|
+
-- every run and `dataops status` look up each dataset's latest rows
|
|
53
|
+
CREATE INDEX IF NOT EXISTS os_ingest_log_dataset_started ON {LOG_TABLE} (dataset_id, started_at);
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
# Most a download may unpack to, so a zip bomb can't fill the disk (DATAOPS_MAX_UNPACKED_BYTES overrides)
|
|
58
|
+
MAX_UNPACKED = 100 * 1000**3
|
|
59
|
+
MAX_UNPACKED_ENV = "DATAOPS_MAX_UNPACKED_BYTES"
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class LoadError(Exception):
|
|
63
|
+
"""A load failed in a way the user can act on; the message says how."""
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass
|
|
67
|
+
class Database:
|
|
68
|
+
host: str
|
|
69
|
+
port: int
|
|
70
|
+
database: str
|
|
71
|
+
username: str
|
|
72
|
+
password: str
|
|
73
|
+
live: str = "data_layers"
|
|
74
|
+
staging: str = "staging"
|
|
75
|
+
backup: str = "backup"
|
|
76
|
+
|
|
77
|
+
def connect(self):
|
|
78
|
+
return psycopg2.connect(host=self.host, port=self.port, dbname=self.database, user=self.username, password=self.password)
|
|
79
|
+
|
|
80
|
+
def ogr_target(self) -> str:
|
|
81
|
+
"""For ogr2ogr; the password goes in the environment, not the command line."""
|
|
82
|
+
return f"PG:host={self.host} port={self.port} dbname={self.database} user={self.username}"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@dataclass
|
|
86
|
+
class Layer:
|
|
87
|
+
"""One layer to load: where GDAL reads it, and the live table it becomes."""
|
|
88
|
+
|
|
89
|
+
source: str # a file path, or a connection such as WFS:<url>
|
|
90
|
+
layer: str
|
|
91
|
+
table: str
|
|
92
|
+
crs: str | None = None # assigned when given (the file declares none, or the Dataset overrides it)
|
|
93
|
+
gdal_layer: str | None = None # GDAL's name for the layer, when `layer` is another (ArcGIS's are all ESRIJSON)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass
|
|
97
|
+
class Staged:
|
|
98
|
+
layer: Layer
|
|
99
|
+
staging: str
|
|
100
|
+
rows: int = 0
|
|
101
|
+
dropped: int = 0
|
|
102
|
+
note: str = ""
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
@dataclass
|
|
106
|
+
class LastLoad:
|
|
107
|
+
version: str
|
|
108
|
+
sha256: str | None
|
|
109
|
+
tables: list[str] = field(default_factory=list)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def stamp(when: datetime | None = None) -> str:
|
|
113
|
+
return (when or datetime.now()).strftime("%Y%m%d_%H%M%S")
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def suffixed(table: str, suffix: str) -> str:
|
|
117
|
+
"""<table>_<suffix> within PostgreSQL's 63-character limit."""
|
|
118
|
+
return f"{table[: 62 - len(suffix)]}_{suffix}"
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
# The end of every staging table's name: _<YYYYmmdd_HHMMSS>_<6 hex digits> (staging_name)
|
|
122
|
+
STAGING_TAG = "_[0-9]{8}_[0-9]{6}_[0-9a-f]{6}$"
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def hold_lock(db: Database):
|
|
126
|
+
"""
|
|
127
|
+
A connection holding the lock that lets one sync at a time load through `db`'s staging schema,
|
|
128
|
+
or None if another run holds it. The lock is PostgreSQL's (an advisory lock), so it works across
|
|
129
|
+
machines and is let go when the connection closes, however the run ends.
|
|
130
|
+
"""
|
|
131
|
+
conn = db.connect()
|
|
132
|
+
conn.autocommit = True
|
|
133
|
+
with conn.cursor() as cur:
|
|
134
|
+
cur.execute("SELECT pg_try_advisory_lock(hashtext('geoserver-dataops'), hashtext(%s))", (db.staging,))
|
|
135
|
+
if cur.fetchone()[0]:
|
|
136
|
+
return conn
|
|
137
|
+
conn.close()
|
|
138
|
+
return None
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def drop_orphans(conn, db: Database) -> list[str]:
|
|
142
|
+
"""
|
|
143
|
+
Drop staging tables a run that was killed (out of memory, a node lost) left behind; returns their
|
|
144
|
+
names. Only call with the lock held: then no other run is loading, so any staging table is an orphan.
|
|
145
|
+
Only tables named as staging_name names them are touched.
|
|
146
|
+
"""
|
|
147
|
+
with conn.cursor() as cur:
|
|
148
|
+
cur.execute("SELECT table_name FROM information_schema.tables WHERE table_schema = %s AND table_name ~ %s "
|
|
149
|
+
"ORDER BY table_name", (db.staging, STAGING_TAG))
|
|
150
|
+
orphans = [row[0] for row in cur.fetchall()]
|
|
151
|
+
for name in orphans:
|
|
152
|
+
cur.execute(f'DROP TABLE IF EXISTS "{db.staging}"."{name}"')
|
|
153
|
+
conn.commit()
|
|
154
|
+
return orphans
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def staging_name(table: str) -> str:
|
|
158
|
+
"""
|
|
159
|
+
A name for this load's staging table, unique to the load. ogr2ogr names
|
|
160
|
+
the table's key, index and sequence after it (…_ogc_fid_seq is the
|
|
161
|
+
longest), and they move with the table into live and backup, so it is
|
|
162
|
+
kept short enough that those names aren't cut short and stay unique.
|
|
163
|
+
"""
|
|
164
|
+
tag = f"{stamp()}_{uuid.uuid4().hex[:6]}"
|
|
165
|
+
return f"{table[: 63 - len('_ogc_fid_seq') - len(tag) - 1]}_{tag}"
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
# --- the ingest log
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def bootstrap(conn) -> None:
|
|
172
|
+
with conn.cursor() as cur:
|
|
173
|
+
cur.execute(BOOTSTRAP_SQL)
|
|
174
|
+
conn.commit()
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def last_load(conn, dataset: str) -> LastLoad | None:
|
|
178
|
+
"""The version, checksum and tables of what is live: the last load, or a later check that found the same
|
|
179
|
+
data under a new version (a portal's metadata changed, its file didn't), so that one isn't downloaded again."""
|
|
180
|
+
with conn.cursor() as cur:
|
|
181
|
+
cur.execute(
|
|
182
|
+
f"SELECT version, sha256, tables FROM {LOG_TABLE} WHERE dataset_id = %s AND status IN ('success', 'skipped') "
|
|
183
|
+
"AND tables IS NOT NULL "
|
|
184
|
+
"ORDER BY finished_at DESC LIMIT 1",
|
|
185
|
+
(dataset,),
|
|
186
|
+
)
|
|
187
|
+
row = cur.fetchone()
|
|
188
|
+
if row is None:
|
|
189
|
+
return None
|
|
190
|
+
return LastLoad(row[0], row[1], [t for t in (row[2] or "").split(",") if t])
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def last_check(conn, dataset: str) -> datetime | None:
|
|
194
|
+
"""When the dataset was last checked; a failed attempt doesn't count, so the next run tries again."""
|
|
195
|
+
with conn.cursor() as cur:
|
|
196
|
+
cur.execute(f"SELECT max(started_at) FROM {LOG_TABLE} WHERE dataset_id = %s AND status <> 'failed'", (dataset,))
|
|
197
|
+
return cur.fetchone()[0]
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def record(conn, dataset: str, version: str, fmt: str | None, status: str, started_at: datetime,
|
|
201
|
+
rows: int | None = None, dropped: int | None = None, sha: str | None = None,
|
|
202
|
+
error: str | None = None, tables: list[str] | None = None) -> None:
|
|
203
|
+
with conn.cursor() as cur:
|
|
204
|
+
cur.execute(
|
|
205
|
+
f"INSERT INTO {LOG_TABLE} (dataset_id, version, format_used, status, rows_loaded, rows_filtered, "
|
|
206
|
+
"sha256, error, started_at, finished_at, tables) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, now(), %s)",
|
|
207
|
+
(dataset, version, fmt, status, rows, dropped, sha, error[:1000] if error else None, started_at,
|
|
208
|
+
",".join(tables) if tables else None),
|
|
209
|
+
)
|
|
210
|
+
conn.commit()
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def tables_exist(conn, schema: str, tables: list[str]) -> bool:
|
|
214
|
+
with conn.cursor() as cur:
|
|
215
|
+
cur.execute(
|
|
216
|
+
"SELECT count(*) FROM information_schema.tables WHERE table_schema = %s AND table_name = ANY(%s)",
|
|
217
|
+
(schema, tables),
|
|
218
|
+
)
|
|
219
|
+
return cur.fetchone()[0] == len(set(tables))
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
# --- getting the data
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
# A connection lost part way through a download is tried again, from where it stopped when the
|
|
226
|
+
# server can send part of a file: one dropped connection shouldn't cost a night's load of a large product
|
|
227
|
+
DOWNLOAD_ATTEMPTS = 4
|
|
228
|
+
DOWNLOAD_PAUSE = 10 # seconds, times the attempt
|
|
229
|
+
DROPPED = (requests.ConnectionError, requests.Timeout, requests.exceptions.ChunkedEncodingError)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def download(url: str, dest: Path, md5: str | None = None, session: requests.Session | None = None) -> str:
|
|
233
|
+
"""Stream `url` to `dest`; check the MD5 when the source gives one; return the SHA256."""
|
|
234
|
+
sha, md5sum, written = hashlib.sha256(), hashlib.md5(), 0
|
|
235
|
+
for attempt in range(1, DOWNLOAD_ATTEMPTS + 1):
|
|
236
|
+
try:
|
|
237
|
+
headers = {"Range": f"bytes={written}-"} if written else {}
|
|
238
|
+
with (session or requests).get(url, stream=True, timeout=TIMEOUT, headers=headers) as r:
|
|
239
|
+
r.raise_for_status()
|
|
240
|
+
resumed = bool(written) and r.status_code == 206 and \
|
|
241
|
+
r.headers.get("Content-Range", "").startswith(f"bytes {written}-")
|
|
242
|
+
if not resumed: # from the start: the first try, or a server that sends whole files only
|
|
243
|
+
sha, md5sum, written = hashlib.sha256(), hashlib.md5(), 0
|
|
244
|
+
with open(dest, "ab" if resumed else "wb") as f:
|
|
245
|
+
for chunk in r.iter_content(1 << 20):
|
|
246
|
+
f.write(chunk)
|
|
247
|
+
sha.update(chunk)
|
|
248
|
+
md5sum.update(chunk)
|
|
249
|
+
written += len(chunk)
|
|
250
|
+
break
|
|
251
|
+
except DROPPED:
|
|
252
|
+
if attempt == DOWNLOAD_ATTEMPTS:
|
|
253
|
+
raise
|
|
254
|
+
time.sleep(DOWNLOAD_PAUSE * attempt)
|
|
255
|
+
if md5 and md5sum.hexdigest().lower() != md5.lower():
|
|
256
|
+
raise LoadError(f"the download's MD5 is {md5sum.hexdigest()}, not {md5} as the source says; try again later")
|
|
257
|
+
return sha.hexdigest()
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def unpack_limit() -> int:
|
|
261
|
+
value = os.environ.get(MAX_UNPACKED_ENV, "")
|
|
262
|
+
return int(value) if value.isdigit() else MAX_UNPACKED
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def check_unpacked_size(size: int) -> None:
|
|
266
|
+
"""Refuse an archive that would unpack to more than the limit. A zip's entries can't unpack to
|
|
267
|
+
more than the sizes it declares (zipfile stops there), so their sum is a true upper bound."""
|
|
268
|
+
if size > unpack_limit():
|
|
269
|
+
raise LoadError(f"the download would unpack to {size:,} bytes, more than the limit of {unpack_limit():,}; "
|
|
270
|
+
f"if that is right for this dataset, raise {MAX_UNPACKED_ENV}")
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def file_layers(path: Path, work: Path) -> list[tuple[str, str, str | None]]:
|
|
274
|
+
"""(GDAL path, layer, CRS) of every layer in a downloaded file, unzipping it first."""
|
|
275
|
+
if zipfile.is_zipfile(path):
|
|
276
|
+
out = work / "extracted"
|
|
277
|
+
with zipfile.ZipFile(path) as z:
|
|
278
|
+
check_unpacked_size(sum(i.file_size for i in z.infolist()))
|
|
279
|
+
z.extractall(out)
|
|
280
|
+
files = [str(out / f) for f in remote.data_files([i.filename for i in z.infolist() if not i.is_dir()])]
|
|
281
|
+
if not files:
|
|
282
|
+
raise LoadError("the download holds no data files GDAL can read")
|
|
283
|
+
else:
|
|
284
|
+
files = [str(path)]
|
|
285
|
+
found = []
|
|
286
|
+
for f in files:
|
|
287
|
+
for layer in remote.to_layers(remote.ogrinfo(f), f):
|
|
288
|
+
found.append((f, layer.name, layer.crs))
|
|
289
|
+
return found
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
# --- loading
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def ensure_schemas(conn, *schemas: str) -> None:
|
|
296
|
+
with conn.cursor() as cur:
|
|
297
|
+
for schema in schemas:
|
|
298
|
+
cur.execute(f'CREATE SCHEMA IF NOT EXISTS "{schema}"')
|
|
299
|
+
conn.commit()
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def gdal_errors(stderr: str) -> str:
|
|
303
|
+
"""The lines of GDAL's output that say what went wrong, without its boilerplate."""
|
|
304
|
+
lines = []
|
|
305
|
+
for line in stderr.splitlines():
|
|
306
|
+
line = line.strip()
|
|
307
|
+
if not line or "Terminating translation" in line or "use -skipfailures" in line or line in lines:
|
|
308
|
+
continue
|
|
309
|
+
if line.startswith(("ERROR", "CONTEXT", "DETAIL")):
|
|
310
|
+
lines.append(line.removeprefix("ERROR 1: ").removeprefix("ERROR:").strip())
|
|
311
|
+
return "; ".join(lines[-3:]) or (stderr.strip().splitlines() or ["no details"])[-1]
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def load(db: Database, layer: Layer, staging: str) -> str:
|
|
315
|
+
"""ogr2ogr one layer into <staging schema>.<staging>; returns a note for the user, or ""."""
|
|
316
|
+
if shutil.which("ogr2ogr") is None:
|
|
317
|
+
raise LoadError("loading needs GDAL's ogr2ogr on the PATH")
|
|
318
|
+
cmd = [
|
|
319
|
+
"ogr2ogr", "-f", "PostgreSQL", *remote.input_drivers(layer.source), db.ogr_target(), layer.source, layer.gdal_layer or layer.layer,
|
|
320
|
+
"-nln", f"{db.staging}.{staging}",
|
|
321
|
+
"-lco", "GEOMETRY_NAME=geom", "-lco", "PRECISION=NO",
|
|
322
|
+
"-nlt", "PROMOTE_TO_MULTI", "-gt", "65536",
|
|
323
|
+
"--config", "PG_USE_COPY", "YES",
|
|
324
|
+
]
|
|
325
|
+
if layer.crs:
|
|
326
|
+
cmd += ["-a_srs", layer.crs]
|
|
327
|
+
env = {**remote.GDAL_ENV, **os.environ, "PGPASSWORD": db.password}
|
|
328
|
+
result = subprocess.run(cmd + ["-overwrite"], capture_output=True, text=True, env=env)
|
|
329
|
+
if result.returncode != 0 and BAD_DATE.search(result.stderr):
|
|
330
|
+
result = subprocess.run(cmd + ["-overwrite"] + DATES_AS_TEXT, capture_output=True, text=True, env=env)
|
|
331
|
+
if result.returncode == 0:
|
|
332
|
+
return f"{layer.table}: dates loaded as text, since some aren't valid dates"
|
|
333
|
+
if result.returncode != 0:
|
|
334
|
+
raise LoadError(f"ogr2ogr couldn't load layer {layer.layer}: {gdal_errors(result.stderr)}")
|
|
335
|
+
return ""
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def geometry_column(conn, schema: str, table: str) -> str | None:
|
|
339
|
+
with conn.cursor() as cur:
|
|
340
|
+
cur.execute("SELECT f_geometry_column FROM geometry_columns WHERE f_table_schema = %s AND f_table_name = %s LIMIT 1",
|
|
341
|
+
(schema, table))
|
|
342
|
+
row = cur.fetchone()
|
|
343
|
+
return row[0] if row else None
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def check(conn, db: Database, staged: Staged, drop_invalid: bool, min_rows: int) -> None:
|
|
347
|
+
"""Drop invalid geometries if asked, count rows, and refuse a table below `min_rows`."""
|
|
348
|
+
name = f'"{db.staging}"."{staged.staging}"'
|
|
349
|
+
with conn.cursor() as cur:
|
|
350
|
+
geom = geometry_column(conn, db.staging, staged.staging)
|
|
351
|
+
if drop_invalid and geom:
|
|
352
|
+
cur.execute(f'DELETE FROM {name} WHERE NOT ST_IsValid("{geom}")')
|
|
353
|
+
staged.dropped = cur.rowcount
|
|
354
|
+
cur.execute(f"SELECT count(*) FROM {name}")
|
|
355
|
+
staged.rows = cur.fetchone()[0]
|
|
356
|
+
conn.commit()
|
|
357
|
+
if staged.rows < min_rows:
|
|
358
|
+
raise LoadError(f"{staged.layer.table} would have {staged.rows:,} rows, fewer than minRows ({min_rows}); "
|
|
359
|
+
"the live table is unchanged")
|
|
360
|
+
with conn.cursor() as cur:
|
|
361
|
+
cur.execute(f"ANALYZE {name}")
|
|
362
|
+
conn.commit()
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
LOCK_TIMEOUT = "10s" # how long a swap waits for readers of the live table before stepping back
|
|
366
|
+
LOCK_ATTEMPTS = 6
|
|
367
|
+
LOCK_PAUSE = 5 # seconds between attempts
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
def dependent_views(cur, table: str) -> list[tuple[int, str, str, str, list[str]]]:
|
|
371
|
+
"""(oid, kind, name, definition, options) of the views that read `table` ("schema"."table") directly.
|
|
372
|
+
Call with an empty search_path, so the names and definitions come fully qualified."""
|
|
373
|
+
cur.execute(
|
|
374
|
+
"SELECT DISTINCT v.oid, v.relkind, v.oid::regclass::text, pg_get_viewdef(v.oid), coalesce(v.reloptions, '{}') "
|
|
375
|
+
"FROM pg_depend d JOIN pg_rewrite r ON r.oid = d.objid JOIN pg_class v ON v.oid = r.ev_class "
|
|
376
|
+
"WHERE d.classid = 'pg_rewrite'::regclass AND d.refclassid = 'pg_class'::regclass "
|
|
377
|
+
"AND d.refobjid = to_regclass(%s) AND v.oid <> d.refobjid ORDER BY v.oid", (table,))
|
|
378
|
+
return cur.fetchall()
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def copy_grants(cur, old: str, new: str) -> None:
|
|
382
|
+
"""Give `new` the privileges granted on `old`: a table ogr2ogr just made has only its owner's."""
|
|
383
|
+
cur.execute(
|
|
384
|
+
"SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE a.grantee::regrole::text END, a.privilege_type, a.is_grantable "
|
|
385
|
+
"FROM pg_class c, aclexplode(c.relacl) a WHERE c.oid = to_regclass(%s) AND a.grantee <> c.relowner", (old,))
|
|
386
|
+
for role, privilege, grantable in cur.fetchall():
|
|
387
|
+
cur.execute(f"GRANT {privilege} ON {new} TO {role}" + (" WITH GRANT OPTION" if grantable else ""))
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def swap_once(conn, db: Database, staged: list[Staged]) -> None:
|
|
391
|
+
with conn.cursor() as cur:
|
|
392
|
+
# an empty search_path makes PostgreSQL write view definitions with every name qualified
|
|
393
|
+
cur.execute("SET LOCAL search_path = ''")
|
|
394
|
+
cur.execute("SET LOCAL lock_timeout = %s", (LOCK_TIMEOUT,))
|
|
395
|
+
views: dict[int, tuple[str, str, list[str]]] = {}
|
|
396
|
+
for s in staged:
|
|
397
|
+
live = f'"{db.live}"."{s.layer.table}"'
|
|
398
|
+
for oid, kind, name, definition, options in dependent_views(cur, live):
|
|
399
|
+
if kind != "v":
|
|
400
|
+
raise LoadError(
|
|
401
|
+
f"materialized view {name} reads {db.live}.{s.layer.table}, and can't be moved onto the new table: "
|
|
402
|
+
"it would go on reading the old one. Drop it before the load and create it again after; "
|
|
403
|
+
"the live table is unchanged")
|
|
404
|
+
views[oid] = (name, definition, options)
|
|
405
|
+
for s in staged:
|
|
406
|
+
table = s.layer.table
|
|
407
|
+
cur.execute("SELECT to_regclass(%s)", (f'"{db.live}"."{table}"',))
|
|
408
|
+
if cur.fetchone()[0] is not None:
|
|
409
|
+
copy_grants(cur, f'"{db.live}"."{table}"', f'"{db.staging}"."{s.staging}"')
|
|
410
|
+
backup = free_backup_name(cur, db, table)
|
|
411
|
+
cur.execute(f'ALTER TABLE "{db.live}"."{table}" RENAME TO "{backup}"')
|
|
412
|
+
cur.execute(f'ALTER TABLE "{db.live}"."{backup}" SET SCHEMA "{db.backup}"')
|
|
413
|
+
cur.execute(f'ALTER TABLE "{db.staging}"."{s.staging}" SET SCHEMA "{db.live}"')
|
|
414
|
+
cur.execute(f'ALTER TABLE "{db.live}"."{s.staging}" RENAME TO "{table}"')
|
|
415
|
+
# A view is tied to the table it was made on, not to its name: left alone it would follow the
|
|
416
|
+
# old table into the backups and serve old data. Its definition, taken before the renames,
|
|
417
|
+
# names the live table, so replacing the view with it points it at the new one; the view
|
|
418
|
+
# keeps its owner and grants.
|
|
419
|
+
for name, definition, options in views.values():
|
|
420
|
+
with_options = f" WITH ({', '.join(options)})" if options else ""
|
|
421
|
+
try:
|
|
422
|
+
cur.execute(f"CREATE OR REPLACE VIEW {name}{with_options} AS {definition}")
|
|
423
|
+
except psycopg2.errors.LockNotAvailable:
|
|
424
|
+
raise
|
|
425
|
+
except psycopg2.Error as e:
|
|
426
|
+
raise LoadError(f"view {name} can't be moved onto the new table ({str(e).strip().splitlines()[0]}); "
|
|
427
|
+
"the live table is unchanged") from None
|
|
428
|
+
conn.commit()
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def swap(conn, db: Database, staged: list[Staged], keep: int) -> None:
|
|
432
|
+
"""In one transaction: each live table to the backup schema, each staged table into its place,
|
|
433
|
+
with the old table's grants and the views that read it."""
|
|
434
|
+
for attempt in range(1, LOCK_ATTEMPTS + 1):
|
|
435
|
+
try:
|
|
436
|
+
swap_once(conn, db, staged)
|
|
437
|
+
break
|
|
438
|
+
except psycopg2.errors.LockNotAvailable:
|
|
439
|
+
# Waiting on behind a long query would hold up every other reader of the table too
|
|
440
|
+
conn.rollback()
|
|
441
|
+
if attempt == LOCK_ATTEMPTS:
|
|
442
|
+
raise LoadError(f"the live tables stayed in use for {LOCK_ATTEMPTS} tries of {LOCK_TIMEOUT}; "
|
|
443
|
+
"they are unchanged, and the next run tries again") from None
|
|
444
|
+
time.sleep(LOCK_PAUSE)
|
|
445
|
+
for s in staged:
|
|
446
|
+
# the new data is live: a backup that can't be dropped mustn't make the load a failure
|
|
447
|
+
problem = prune(conn, db, s.layer.table, keep)
|
|
448
|
+
if problem:
|
|
449
|
+
s.note = "; ".join(x for x in (s.note, problem) if x)
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def free_backup_name(cur, db: Database, table: str) -> str:
|
|
453
|
+
"""<table>_<now>, a second later if two loads land in the same second."""
|
|
454
|
+
when = datetime.now()
|
|
455
|
+
while True:
|
|
456
|
+
name = suffixed(table, stamp(when))
|
|
457
|
+
cur.execute("SELECT to_regclass(%s)", (f'"{db.backup}"."{name}"',))
|
|
458
|
+
if cur.fetchone()[0] is None:
|
|
459
|
+
return name
|
|
460
|
+
when += timedelta(seconds=1)
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def prune(conn, db: Database, table: str, keep: int) -> str:
|
|
464
|
+
"""Drop all but the newest `keep` backups of a table; returns what couldn't be dropped, or ""."""
|
|
465
|
+
prefix = table[: 62 - len("YYYYmmdd_HHMMSS")]
|
|
466
|
+
kept = []
|
|
467
|
+
with conn.cursor() as cur:
|
|
468
|
+
cur.execute(
|
|
469
|
+
"SELECT table_name FROM information_schema.tables WHERE table_schema = %s AND table_name ~ %s "
|
|
470
|
+
"ORDER BY table_name DESC",
|
|
471
|
+
(db.backup, f"^{prefix}_[0-9]{{8}}_[0-9]{{6}}$"),
|
|
472
|
+
)
|
|
473
|
+
for (old,) in cur.fetchall()[keep:]:
|
|
474
|
+
try:
|
|
475
|
+
cur.execute("SAVEPOINT prune")
|
|
476
|
+
cur.execute(f'DROP TABLE "{db.backup}"."{old}"')
|
|
477
|
+
except psycopg2.Error as e:
|
|
478
|
+
cur.execute("ROLLBACK TO SAVEPOINT prune")
|
|
479
|
+
kept.append(f"old backup {db.backup}.{old} was kept: {str(e).strip().splitlines()[0]}")
|
|
480
|
+
conn.commit()
|
|
481
|
+
return "; ".join(kept)
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def drop_staged(conn, db: Database, staged: list[Staged]) -> None:
|
|
485
|
+
conn.rollback()
|
|
486
|
+
with conn.cursor() as cur:
|
|
487
|
+
for s in staged:
|
|
488
|
+
cur.execute(f'DROP TABLE IF EXISTS "{db.staging}"."{s.staging}"')
|
|
489
|
+
conn.commit()
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
def load_all(conn, db: Database, layers: list[Layer], drop_invalid: bool = True, min_rows: int = 1, keep: int = 3) -> list[Staged]:
|
|
493
|
+
"""Stage and check every layer, then swap them all in; on any failure, live tables are untouched."""
|
|
494
|
+
ensure_schemas(conn, db.live, db.staging, db.backup)
|
|
495
|
+
staged: list[Staged] = []
|
|
496
|
+
try:
|
|
497
|
+
for layer in layers:
|
|
498
|
+
s = Staged(layer, staging_name(layer.table))
|
|
499
|
+
staged.append(s)
|
|
500
|
+
s.note = load(db, layer, s.staging)
|
|
501
|
+
check(conn, db, s, drop_invalid, min_rows)
|
|
502
|
+
swap(conn, db, staged, keep)
|
|
503
|
+
except BaseException:
|
|
504
|
+
try:
|
|
505
|
+
drop_staged(conn, db, staged)
|
|
506
|
+
except psycopg2.Error:
|
|
507
|
+
pass # the connection is gone: let the error that got us here be the one reported
|
|
508
|
+
raise
|
|
509
|
+
return staged
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Read every resource in a project directory, keeping where each one came from."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import re
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
import yaml
|
|
12
|
+
|
|
13
|
+
# Holds secret values (spec: "Secrets"); never read as configuration
|
|
14
|
+
SECRET_FILES = {"secrets.yaml", "secrets.yml"}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def default_table(dataset: str, layer: str) -> str:
|
|
18
|
+
"""A Dataset layer's table when `tables` doesn't name it: <dataset>_<layer>, lowercase, a-z 0-9 _ only."""
|
|
19
|
+
name = re.sub(r"[^a-z0-9]+", "_", f"{dataset}_{layer}".lower()).strip("_")
|
|
20
|
+
if not name or name[0].isdigit():
|
|
21
|
+
name = f"t_{name}"
|
|
22
|
+
return name[:63]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class Problem:
|
|
27
|
+
"""Something wrong with the files. `line` is 1-based; 0 means the whole file."""
|
|
28
|
+
|
|
29
|
+
file: Path
|
|
30
|
+
line: int
|
|
31
|
+
message: str
|
|
32
|
+
resource: str = "" # e.g. "Layer road-links"
|
|
33
|
+
severity: str = "error" # error | warning
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass
|
|
37
|
+
class Resource:
|
|
38
|
+
data: Any
|
|
39
|
+
file: Path
|
|
40
|
+
node: yaml.Node = field(repr=False)
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def kind(self) -> str | None:
|
|
44
|
+
return self.data.get("kind") if isinstance(self.data, dict) else None
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def name(self) -> str | None:
|
|
48
|
+
meta = self.data.get("metadata") if isinstance(self.data, dict) else None
|
|
49
|
+
return meta.get("name") if isinstance(meta, dict) else None
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def spec(self) -> dict:
|
|
53
|
+
spec = self.data.get("spec") if isinstance(self.data, dict) else None
|
|
54
|
+
return spec if isinstance(spec, dict) else {}
|
|
55
|
+
|
|
56
|
+
@property
|
|
57
|
+
def label(self) -> str:
|
|
58
|
+
return f"{self.kind or '?'} {self.name or '?'}"
|
|
59
|
+
|
|
60
|
+
def line(self, path: tuple = ()) -> int:
|
|
61
|
+
"""1-based line of the value at `path` (keys and list indexes), or of its nearest parent."""
|
|
62
|
+
node = self.node
|
|
63
|
+
for step in path:
|
|
64
|
+
child = None
|
|
65
|
+
if isinstance(node, yaml.MappingNode):
|
|
66
|
+
child = next((v for k, v in node.value if k.value == step), None)
|
|
67
|
+
elif isinstance(node, yaml.SequenceNode) and isinstance(step, int) and step < len(node.value):
|
|
68
|
+
child = node.value[step]
|
|
69
|
+
if child is None:
|
|
70
|
+
break
|
|
71
|
+
node = child
|
|
72
|
+
return node.start_mark.line + 1
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass
|
|
76
|
+
class Project:
|
|
77
|
+
root: Path
|
|
78
|
+
resources: list[Resource]
|
|
79
|
+
files: list[Path]
|
|
80
|
+
problems: list[Problem] # files that couldn't be read
|
|
81
|
+
|
|
82
|
+
def of_kind(self, kind: str) -> list[Resource]:
|
|
83
|
+
return [r for r in self.resources if r.kind == kind]
|
|
84
|
+
|
|
85
|
+
def named(self, kind: str) -> dict[str, Resource]:
|
|
86
|
+
"""First resource of each name (duplicates are reported by validate)."""
|
|
87
|
+
out: dict[str, Resource] = {}
|
|
88
|
+
for r in self.of_kind(kind):
|
|
89
|
+
if r.name is not None:
|
|
90
|
+
out.setdefault(r.name, r)
|
|
91
|
+
return out
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def find_files(root: Path) -> list[Path]:
|
|
95
|
+
"""
|
|
96
|
+
Every YAML file under `root`, hidden ones left out. Symlinked folders are
|
|
97
|
+
followed: Kubernetes mounts a ConfigMap as symlinks into a hidden ..data
|
|
98
|
+
folder, so a project's datasets/ is a link to a directory there.
|
|
99
|
+
"""
|
|
100
|
+
files = []
|
|
101
|
+
for folder, dirs, names in os.walk(root, followlinks=True):
|
|
102
|
+
dirs[:] = [d for d in dirs if not d.startswith(".")]
|
|
103
|
+
for name in names:
|
|
104
|
+
path = Path(folder) / name
|
|
105
|
+
if (not name.startswith(".") and path.suffix in (".yaml", ".yml") and name not in SECRET_FILES
|
|
106
|
+
and path.is_file()):
|
|
107
|
+
files.append(path)
|
|
108
|
+
return sorted(files)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def load(root: Path) -> Project:
|
|
112
|
+
root = Path(root)
|
|
113
|
+
resources: list[Resource] = []
|
|
114
|
+
problems: list[Problem] = []
|
|
115
|
+
files = find_files(root)
|
|
116
|
+
for path in files:
|
|
117
|
+
try:
|
|
118
|
+
text = path.read_text(encoding="utf-8")
|
|
119
|
+
for node in yaml.compose_all(text, Loader=yaml.SafeLoader):
|
|
120
|
+
if node is None:
|
|
121
|
+
continue
|
|
122
|
+
data = yaml.SafeLoader(text).construct_document(node)
|
|
123
|
+
if data is None:
|
|
124
|
+
continue
|
|
125
|
+
resources.append(Resource(data=data, file=path, node=node))
|
|
126
|
+
except yaml.YAMLError as e:
|
|
127
|
+
mark = getattr(e, "problem_mark", None)
|
|
128
|
+
reason = getattr(e, "problem", None) or str(e)
|
|
129
|
+
problems.append(Problem(path, mark.line + 1 if mark else 0, f"not valid YAML: {reason}"))
|
|
130
|
+
except UnicodeDecodeError:
|
|
131
|
+
problems.append(Problem(path, 0, "not UTF-8 text"))
|
|
132
|
+
return Project(root=root, resources=resources, files=files, problems=problems)
|