geoserver-dataops 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. geoserver_dataops/__init__.py +4 -0
  2. geoserver_dataops/__main__.py +3 -0
  3. geoserver_dataops/cli.py +141 -0
  4. geoserver_dataops/crs.py +21 -0
  5. geoserver_dataops/discover.py +324 -0
  6. geoserver_dataops/ingest.py +509 -0
  7. geoserver_dataops/loader.py +132 -0
  8. geoserver_dataops/raster.py +165 -0
  9. geoserver_dataops/remote.py +260 -0
  10. geoserver_dataops/schedule.py +96 -0
  11. geoserver_dataops/schema/v1alpha1.json +346 -0
  12. geoserver_dataops/secrets.py +65 -0
  13. geoserver_dataops/sources/__init__.py +121 -0
  14. geoserver_dataops/sources/arcgis.py +219 -0
  15. geoserver_dataops/sources/catalogue.py +259 -0
  16. geoserver_dataops/sources/ckan.py +135 -0
  17. geoserver_dataops/sources/formats.py +104 -0
  18. geoserver_dataops/sources/geofabrik.py +102 -0
  19. geoserver_dataops/sources/koordinates.py +127 -0
  20. geoserver_dataops/sources/ogcapi.py +115 -0
  21. geoserver_dataops/sources/opendata.py +152 -0
  22. geoserver_dataops/sources/os_downloads.py +134 -0
  23. geoserver_dataops/sources/overture.py +205 -0
  24. geoserver_dataops/sources/stac.py +184 -0
  25. geoserver_dataops/sources/url.py +99 -0
  26. geoserver_dataops/sources/wfs.py +123 -0
  27. geoserver_dataops/status.py +90 -0
  28. geoserver_dataops/sync.py +315 -0
  29. geoserver_dataops/validate.py +212 -0
  30. geoserver_dataops-0.2.0.dist-info/METADATA +151 -0
  31. geoserver_dataops-0.2.0.dist-info/RECORD +35 -0
  32. geoserver_dataops-0.2.0.dist-info/WHEEL +5 -0
  33. geoserver_dataops-0.2.0.dist-info/entry_points.txt +2 -0
  34. geoserver_dataops-0.2.0.dist-info/licenses/LICENSE +21 -0
  35. geoserver_dataops-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,509 @@
1
+ """Loading tables into PostGIS: stage, check, then swap in, with backups.
2
+
3
+ Every layer is loaded with ogr2ogr into the staging schema under a unique
4
+ name, checked (invalid geometries dropped, row count), then all of a
5
+ dataset's tables go live in one transaction: each old table is renamed into
6
+ the backup schema and the new one renamed into place. Nothing is copied,
7
+ and if anything fails before the swap, every live table stays as it was.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import hashlib
13
+ import os
14
+ import re
15
+ import shutil
16
+ import subprocess
17
+ import time
18
+ import uuid
19
+ import zipfile
20
+ from dataclasses import dataclass, field
21
+ from datetime import datetime, timedelta
22
+ from pathlib import Path
23
+
24
+ import psycopg2
25
+ import psycopg2.errors
26
+ import requests
27
+
28
+ from . import remote
29
+
30
+ LOG_TABLE = "public.os_ingest_log" # the name the first ingest pipeline gave it; kept so history carries on
31
+ TIMEOUT = 120
32
+
33
+ # A date PostgreSQL rejects (shapefiles store "no date" as 0000/00/00): load dates as text instead
34
+ BAD_DATE = re.compile(r"date/time field value out of range|invalid input syntax for type (date|time|timestamp)|column \w+: \"0000", re.IGNORECASE)
35
+ DATES_AS_TEXT = ["-mapFieldType", "Date=String,Time=String,DateTime=String"]
36
+
37
+ BOOTSTRAP_SQL = f"""
38
+ CREATE TABLE IF NOT EXISTS {LOG_TABLE} (
39
+ id SERIAL PRIMARY KEY,
40
+ dataset_id TEXT NOT NULL,
41
+ version TEXT NOT NULL,
42
+ format_used TEXT,
43
+ status TEXT NOT NULL, -- success | failed | skipped
44
+ rows_loaded BIGINT,
45
+ rows_filtered BIGINT, -- invalid geometries left out
46
+ sha256 TEXT,
47
+ error TEXT,
48
+ started_at TIMESTAMP NOT NULL DEFAULT now(),
49
+ finished_at TIMESTAMP
50
+ );
51
+ ALTER TABLE {LOG_TABLE} ADD COLUMN IF NOT EXISTS tables TEXT;
52
+ -- every run and `dataops status` look up each dataset's latest rows
53
+ CREATE INDEX IF NOT EXISTS os_ingest_log_dataset_started ON {LOG_TABLE} (dataset_id, started_at);
54
+ """
55
+
56
+
57
+ # Most a download may unpack to, so a zip bomb can't fill the disk (DATAOPS_MAX_UNPACKED_BYTES overrides)
58
+ MAX_UNPACKED = 100 * 1000**3
59
+ MAX_UNPACKED_ENV = "DATAOPS_MAX_UNPACKED_BYTES"
60
+
61
+
62
+ class LoadError(Exception):
63
+ """A load failed in a way the user can act on; the message says how."""
64
+
65
+
66
+ @dataclass
67
+ class Database:
68
+ host: str
69
+ port: int
70
+ database: str
71
+ username: str
72
+ password: str
73
+ live: str = "data_layers"
74
+ staging: str = "staging"
75
+ backup: str = "backup"
76
+
77
+ def connect(self):
78
+ return psycopg2.connect(host=self.host, port=self.port, dbname=self.database, user=self.username, password=self.password)
79
+
80
+ def ogr_target(self) -> str:
81
+ """For ogr2ogr; the password goes in the environment, not the command line."""
82
+ return f"PG:host={self.host} port={self.port} dbname={self.database} user={self.username}"
83
+
84
+
85
+ @dataclass
86
+ class Layer:
87
+ """One layer to load: where GDAL reads it, and the live table it becomes."""
88
+
89
+ source: str # a file path, or a connection such as WFS:<url>
90
+ layer: str
91
+ table: str
92
+ crs: str | None = None # assigned when given (the file declares none, or the Dataset overrides it)
93
+ gdal_layer: str | None = None # GDAL's name for the layer, when `layer` is another (ArcGIS's are all ESRIJSON)
94
+
95
+
96
+ @dataclass
97
+ class Staged:
98
+ layer: Layer
99
+ staging: str
100
+ rows: int = 0
101
+ dropped: int = 0
102
+ note: str = ""
103
+
104
+
105
+ @dataclass
106
+ class LastLoad:
107
+ version: str
108
+ sha256: str | None
109
+ tables: list[str] = field(default_factory=list)
110
+
111
+
112
+ def stamp(when: datetime | None = None) -> str:
113
+ return (when or datetime.now()).strftime("%Y%m%d_%H%M%S")
114
+
115
+
116
+ def suffixed(table: str, suffix: str) -> str:
117
+ """<table>_<suffix> within PostgreSQL's 63-character limit."""
118
+ return f"{table[: 62 - len(suffix)]}_{suffix}"
119
+
120
+
121
+ # The end of every staging table's name: _<YYYYmmdd_HHMMSS>_<6 hex digits> (staging_name)
122
+ STAGING_TAG = "_[0-9]{8}_[0-9]{6}_[0-9a-f]{6}$"
123
+
124
+
125
+ def hold_lock(db: Database):
126
+ """
127
+ A connection holding the lock that lets one sync at a time load through `db`'s staging schema,
128
+ or None if another run holds it. The lock is PostgreSQL's (an advisory lock), so it works across
129
+ machines and is let go when the connection closes, however the run ends.
130
+ """
131
+ conn = db.connect()
132
+ conn.autocommit = True
133
+ with conn.cursor() as cur:
134
+ cur.execute("SELECT pg_try_advisory_lock(hashtext('geoserver-dataops'), hashtext(%s))", (db.staging,))
135
+ if cur.fetchone()[0]:
136
+ return conn
137
+ conn.close()
138
+ return None
139
+
140
+
141
+ def drop_orphans(conn, db: Database) -> list[str]:
142
+ """
143
+ Drop staging tables a run that was killed (out of memory, a node lost) left behind; returns their
144
+ names. Only call with the lock held: then no other run is loading, so any staging table is an orphan.
145
+ Only tables named as staging_name names them are touched.
146
+ """
147
+ with conn.cursor() as cur:
148
+ cur.execute("SELECT table_name FROM information_schema.tables WHERE table_schema = %s AND table_name ~ %s "
149
+ "ORDER BY table_name", (db.staging, STAGING_TAG))
150
+ orphans = [row[0] for row in cur.fetchall()]
151
+ for name in orphans:
152
+ cur.execute(f'DROP TABLE IF EXISTS "{db.staging}"."{name}"')
153
+ conn.commit()
154
+ return orphans
155
+
156
+
157
+ def staging_name(table: str) -> str:
158
+ """
159
+ A name for this load's staging table, unique to the load. ogr2ogr names
160
+ the table's key, index and sequence after it (…_ogc_fid_seq is the
161
+ longest), and they move with the table into live and backup, so it is
162
+ kept short enough that those names aren't cut short and stay unique.
163
+ """
164
+ tag = f"{stamp()}_{uuid.uuid4().hex[:6]}"
165
+ return f"{table[: 63 - len('_ogc_fid_seq') - len(tag) - 1]}_{tag}"
166
+
167
+
168
+ # --- the ingest log
169
+
170
+
171
+ def bootstrap(conn) -> None:
172
+ with conn.cursor() as cur:
173
+ cur.execute(BOOTSTRAP_SQL)
174
+ conn.commit()
175
+
176
+
177
+ def last_load(conn, dataset: str) -> LastLoad | None:
178
+ """The version, checksum and tables of what is live: the last load, or a later check that found the same
179
+ data under a new version (a portal's metadata changed, its file didn't), so that one isn't downloaded again."""
180
+ with conn.cursor() as cur:
181
+ cur.execute(
182
+ f"SELECT version, sha256, tables FROM {LOG_TABLE} WHERE dataset_id = %s AND status IN ('success', 'skipped') "
183
+ "AND tables IS NOT NULL "
184
+ "ORDER BY finished_at DESC LIMIT 1",
185
+ (dataset,),
186
+ )
187
+ row = cur.fetchone()
188
+ if row is None:
189
+ return None
190
+ return LastLoad(row[0], row[1], [t for t in (row[2] or "").split(",") if t])
191
+
192
+
193
+ def last_check(conn, dataset: str) -> datetime | None:
194
+ """When the dataset was last checked; a failed attempt doesn't count, so the next run tries again."""
195
+ with conn.cursor() as cur:
196
+ cur.execute(f"SELECT max(started_at) FROM {LOG_TABLE} WHERE dataset_id = %s AND status <> 'failed'", (dataset,))
197
+ return cur.fetchone()[0]
198
+
199
+
200
+ def record(conn, dataset: str, version: str, fmt: str | None, status: str, started_at: datetime,
201
+ rows: int | None = None, dropped: int | None = None, sha: str | None = None,
202
+ error: str | None = None, tables: list[str] | None = None) -> None:
203
+ with conn.cursor() as cur:
204
+ cur.execute(
205
+ f"INSERT INTO {LOG_TABLE} (dataset_id, version, format_used, status, rows_loaded, rows_filtered, "
206
+ "sha256, error, started_at, finished_at, tables) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, now(), %s)",
207
+ (dataset, version, fmt, status, rows, dropped, sha, error[:1000] if error else None, started_at,
208
+ ",".join(tables) if tables else None),
209
+ )
210
+ conn.commit()
211
+
212
+
213
+ def tables_exist(conn, schema: str, tables: list[str]) -> bool:
214
+ with conn.cursor() as cur:
215
+ cur.execute(
216
+ "SELECT count(*) FROM information_schema.tables WHERE table_schema = %s AND table_name = ANY(%s)",
217
+ (schema, tables),
218
+ )
219
+ return cur.fetchone()[0] == len(set(tables))
220
+
221
+
222
+ # --- getting the data
223
+
224
+
225
+ # A connection lost part way through a download is tried again, from where it stopped when the
226
+ # server can send part of a file: one dropped connection shouldn't cost a night's load of a large product
227
+ DOWNLOAD_ATTEMPTS = 4
228
+ DOWNLOAD_PAUSE = 10 # seconds, times the attempt
229
+ DROPPED = (requests.ConnectionError, requests.Timeout, requests.exceptions.ChunkedEncodingError)
230
+
231
+
232
+ def download(url: str, dest: Path, md5: str | None = None, session: requests.Session | None = None) -> str:
233
+ """Stream `url` to `dest`; check the MD5 when the source gives one; return the SHA256."""
234
+ sha, md5sum, written = hashlib.sha256(), hashlib.md5(), 0
235
+ for attempt in range(1, DOWNLOAD_ATTEMPTS + 1):
236
+ try:
237
+ headers = {"Range": f"bytes={written}-"} if written else {}
238
+ with (session or requests).get(url, stream=True, timeout=TIMEOUT, headers=headers) as r:
239
+ r.raise_for_status()
240
+ resumed = bool(written) and r.status_code == 206 and \
241
+ r.headers.get("Content-Range", "").startswith(f"bytes {written}-")
242
+ if not resumed: # from the start: the first try, or a server that sends whole files only
243
+ sha, md5sum, written = hashlib.sha256(), hashlib.md5(), 0
244
+ with open(dest, "ab" if resumed else "wb") as f:
245
+ for chunk in r.iter_content(1 << 20):
246
+ f.write(chunk)
247
+ sha.update(chunk)
248
+ md5sum.update(chunk)
249
+ written += len(chunk)
250
+ break
251
+ except DROPPED:
252
+ if attempt == DOWNLOAD_ATTEMPTS:
253
+ raise
254
+ time.sleep(DOWNLOAD_PAUSE * attempt)
255
+ if md5 and md5sum.hexdigest().lower() != md5.lower():
256
+ raise LoadError(f"the download's MD5 is {md5sum.hexdigest()}, not {md5} as the source says; try again later")
257
+ return sha.hexdigest()
258
+
259
+
260
+ def unpack_limit() -> int:
261
+ value = os.environ.get(MAX_UNPACKED_ENV, "")
262
+ return int(value) if value.isdigit() else MAX_UNPACKED
263
+
264
+
265
+ def check_unpacked_size(size: int) -> None:
266
+ """Refuse an archive that would unpack to more than the limit. A zip's entries can't unpack to
267
+ more than the sizes it declares (zipfile stops there), so their sum is a true upper bound."""
268
+ if size > unpack_limit():
269
+ raise LoadError(f"the download would unpack to {size:,} bytes, more than the limit of {unpack_limit():,}; "
270
+ f"if that is right for this dataset, raise {MAX_UNPACKED_ENV}")
271
+
272
+
273
+ def file_layers(path: Path, work: Path) -> list[tuple[str, str, str | None]]:
274
+ """(GDAL path, layer, CRS) of every layer in a downloaded file, unzipping it first."""
275
+ if zipfile.is_zipfile(path):
276
+ out = work / "extracted"
277
+ with zipfile.ZipFile(path) as z:
278
+ check_unpacked_size(sum(i.file_size for i in z.infolist()))
279
+ z.extractall(out)
280
+ files = [str(out / f) for f in remote.data_files([i.filename for i in z.infolist() if not i.is_dir()])]
281
+ if not files:
282
+ raise LoadError("the download holds no data files GDAL can read")
283
+ else:
284
+ files = [str(path)]
285
+ found = []
286
+ for f in files:
287
+ for layer in remote.to_layers(remote.ogrinfo(f), f):
288
+ found.append((f, layer.name, layer.crs))
289
+ return found
290
+
291
+
292
+ # --- loading
293
+
294
+
295
+ def ensure_schemas(conn, *schemas: str) -> None:
296
+ with conn.cursor() as cur:
297
+ for schema in schemas:
298
+ cur.execute(f'CREATE SCHEMA IF NOT EXISTS "{schema}"')
299
+ conn.commit()
300
+
301
+
302
+ def gdal_errors(stderr: str) -> str:
303
+ """The lines of GDAL's output that say what went wrong, without its boilerplate."""
304
+ lines = []
305
+ for line in stderr.splitlines():
306
+ line = line.strip()
307
+ if not line or "Terminating translation" in line or "use -skipfailures" in line or line in lines:
308
+ continue
309
+ if line.startswith(("ERROR", "CONTEXT", "DETAIL")):
310
+ lines.append(line.removeprefix("ERROR 1: ").removeprefix("ERROR:").strip())
311
+ return "; ".join(lines[-3:]) or (stderr.strip().splitlines() or ["no details"])[-1]
312
+
313
+
314
+ def load(db: Database, layer: Layer, staging: str) -> str:
315
+ """ogr2ogr one layer into <staging schema>.<staging>; returns a note for the user, or ""."""
316
+ if shutil.which("ogr2ogr") is None:
317
+ raise LoadError("loading needs GDAL's ogr2ogr on the PATH")
318
+ cmd = [
319
+ "ogr2ogr", "-f", "PostgreSQL", *remote.input_drivers(layer.source), db.ogr_target(), layer.source, layer.gdal_layer or layer.layer,
320
+ "-nln", f"{db.staging}.{staging}",
321
+ "-lco", "GEOMETRY_NAME=geom", "-lco", "PRECISION=NO",
322
+ "-nlt", "PROMOTE_TO_MULTI", "-gt", "65536",
323
+ "--config", "PG_USE_COPY", "YES",
324
+ ]
325
+ if layer.crs:
326
+ cmd += ["-a_srs", layer.crs]
327
+ env = {**remote.GDAL_ENV, **os.environ, "PGPASSWORD": db.password}
328
+ result = subprocess.run(cmd + ["-overwrite"], capture_output=True, text=True, env=env)
329
+ if result.returncode != 0 and BAD_DATE.search(result.stderr):
330
+ result = subprocess.run(cmd + ["-overwrite"] + DATES_AS_TEXT, capture_output=True, text=True, env=env)
331
+ if result.returncode == 0:
332
+ return f"{layer.table}: dates loaded as text, since some aren't valid dates"
333
+ if result.returncode != 0:
334
+ raise LoadError(f"ogr2ogr couldn't load layer {layer.layer}: {gdal_errors(result.stderr)}")
335
+ return ""
336
+
337
+
338
+ def geometry_column(conn, schema: str, table: str) -> str | None:
339
+ with conn.cursor() as cur:
340
+ cur.execute("SELECT f_geometry_column FROM geometry_columns WHERE f_table_schema = %s AND f_table_name = %s LIMIT 1",
341
+ (schema, table))
342
+ row = cur.fetchone()
343
+ return row[0] if row else None
344
+
345
+
346
+ def check(conn, db: Database, staged: Staged, drop_invalid: bool, min_rows: int) -> None:
347
+ """Drop invalid geometries if asked, count rows, and refuse a table below `min_rows`."""
348
+ name = f'"{db.staging}"."{staged.staging}"'
349
+ with conn.cursor() as cur:
350
+ geom = geometry_column(conn, db.staging, staged.staging)
351
+ if drop_invalid and geom:
352
+ cur.execute(f'DELETE FROM {name} WHERE NOT ST_IsValid("{geom}")')
353
+ staged.dropped = cur.rowcount
354
+ cur.execute(f"SELECT count(*) FROM {name}")
355
+ staged.rows = cur.fetchone()[0]
356
+ conn.commit()
357
+ if staged.rows < min_rows:
358
+ raise LoadError(f"{staged.layer.table} would have {staged.rows:,} rows, fewer than minRows ({min_rows}); "
359
+ "the live table is unchanged")
360
+ with conn.cursor() as cur:
361
+ cur.execute(f"ANALYZE {name}")
362
+ conn.commit()
363
+
364
+
365
+ LOCK_TIMEOUT = "10s" # how long a swap waits for readers of the live table before stepping back
366
+ LOCK_ATTEMPTS = 6
367
+ LOCK_PAUSE = 5 # seconds between attempts
368
+
369
+
370
+ def dependent_views(cur, table: str) -> list[tuple[int, str, str, str, list[str]]]:
371
+ """(oid, kind, name, definition, options) of the views that read `table` ("schema"."table") directly.
372
+ Call with an empty search_path, so the names and definitions come fully qualified."""
373
+ cur.execute(
374
+ "SELECT DISTINCT v.oid, v.relkind, v.oid::regclass::text, pg_get_viewdef(v.oid), coalesce(v.reloptions, '{}') "
375
+ "FROM pg_depend d JOIN pg_rewrite r ON r.oid = d.objid JOIN pg_class v ON v.oid = r.ev_class "
376
+ "WHERE d.classid = 'pg_rewrite'::regclass AND d.refclassid = 'pg_class'::regclass "
377
+ "AND d.refobjid = to_regclass(%s) AND v.oid <> d.refobjid ORDER BY v.oid", (table,))
378
+ return cur.fetchall()
379
+
380
+
381
+ def copy_grants(cur, old: str, new: str) -> None:
382
+ """Give `new` the privileges granted on `old`: a table ogr2ogr just made has only its owner's."""
383
+ cur.execute(
384
+ "SELECT CASE WHEN a.grantee = 0 THEN 'PUBLIC' ELSE a.grantee::regrole::text END, a.privilege_type, a.is_grantable "
385
+ "FROM pg_class c, aclexplode(c.relacl) a WHERE c.oid = to_regclass(%s) AND a.grantee <> c.relowner", (old,))
386
+ for role, privilege, grantable in cur.fetchall():
387
+ cur.execute(f"GRANT {privilege} ON {new} TO {role}" + (" WITH GRANT OPTION" if grantable else ""))
388
+
389
+
390
+ def swap_once(conn, db: Database, staged: list[Staged]) -> None:
391
+ with conn.cursor() as cur:
392
+ # an empty search_path makes PostgreSQL write view definitions with every name qualified
393
+ cur.execute("SET LOCAL search_path = ''")
394
+ cur.execute("SET LOCAL lock_timeout = %s", (LOCK_TIMEOUT,))
395
+ views: dict[int, tuple[str, str, list[str]]] = {}
396
+ for s in staged:
397
+ live = f'"{db.live}"."{s.layer.table}"'
398
+ for oid, kind, name, definition, options in dependent_views(cur, live):
399
+ if kind != "v":
400
+ raise LoadError(
401
+ f"materialized view {name} reads {db.live}.{s.layer.table}, and can't be moved onto the new table: "
402
+ "it would go on reading the old one. Drop it before the load and create it again after; "
403
+ "the live table is unchanged")
404
+ views[oid] = (name, definition, options)
405
+ for s in staged:
406
+ table = s.layer.table
407
+ cur.execute("SELECT to_regclass(%s)", (f'"{db.live}"."{table}"',))
408
+ if cur.fetchone()[0] is not None:
409
+ copy_grants(cur, f'"{db.live}"."{table}"', f'"{db.staging}"."{s.staging}"')
410
+ backup = free_backup_name(cur, db, table)
411
+ cur.execute(f'ALTER TABLE "{db.live}"."{table}" RENAME TO "{backup}"')
412
+ cur.execute(f'ALTER TABLE "{db.live}"."{backup}" SET SCHEMA "{db.backup}"')
413
+ cur.execute(f'ALTER TABLE "{db.staging}"."{s.staging}" SET SCHEMA "{db.live}"')
414
+ cur.execute(f'ALTER TABLE "{db.live}"."{s.staging}" RENAME TO "{table}"')
415
+ # A view is tied to the table it was made on, not to its name: left alone it would follow the
416
+ # old table into the backups and serve old data. Its definition, taken before the renames,
417
+ # names the live table, so replacing the view with it points it at the new one; the view
418
+ # keeps its owner and grants.
419
+ for name, definition, options in views.values():
420
+ with_options = f" WITH ({', '.join(options)})" if options else ""
421
+ try:
422
+ cur.execute(f"CREATE OR REPLACE VIEW {name}{with_options} AS {definition}")
423
+ except psycopg2.errors.LockNotAvailable:
424
+ raise
425
+ except psycopg2.Error as e:
426
+ raise LoadError(f"view {name} can't be moved onto the new table ({str(e).strip().splitlines()[0]}); "
427
+ "the live table is unchanged") from None
428
+ conn.commit()
429
+
430
+
431
+ def swap(conn, db: Database, staged: list[Staged], keep: int) -> None:
432
+ """In one transaction: each live table to the backup schema, each staged table into its place,
433
+ with the old table's grants and the views that read it."""
434
+ for attempt in range(1, LOCK_ATTEMPTS + 1):
435
+ try:
436
+ swap_once(conn, db, staged)
437
+ break
438
+ except psycopg2.errors.LockNotAvailable:
439
+ # Waiting on behind a long query would hold up every other reader of the table too
440
+ conn.rollback()
441
+ if attempt == LOCK_ATTEMPTS:
442
+ raise LoadError(f"the live tables stayed in use for {LOCK_ATTEMPTS} tries of {LOCK_TIMEOUT}; "
443
+ "they are unchanged, and the next run tries again") from None
444
+ time.sleep(LOCK_PAUSE)
445
+ for s in staged:
446
+ # the new data is live: a backup that can't be dropped mustn't make the load a failure
447
+ problem = prune(conn, db, s.layer.table, keep)
448
+ if problem:
449
+ s.note = "; ".join(x for x in (s.note, problem) if x)
450
+
451
+
452
+ def free_backup_name(cur, db: Database, table: str) -> str:
453
+ """<table>_<now>, a second later if two loads land in the same second."""
454
+ when = datetime.now()
455
+ while True:
456
+ name = suffixed(table, stamp(when))
457
+ cur.execute("SELECT to_regclass(%s)", (f'"{db.backup}"."{name}"',))
458
+ if cur.fetchone()[0] is None:
459
+ return name
460
+ when += timedelta(seconds=1)
461
+
462
+
463
+ def prune(conn, db: Database, table: str, keep: int) -> str:
464
+ """Drop all but the newest `keep` backups of a table; returns what couldn't be dropped, or ""."""
465
+ prefix = table[: 62 - len("YYYYmmdd_HHMMSS")]
466
+ kept = []
467
+ with conn.cursor() as cur:
468
+ cur.execute(
469
+ "SELECT table_name FROM information_schema.tables WHERE table_schema = %s AND table_name ~ %s "
470
+ "ORDER BY table_name DESC",
471
+ (db.backup, f"^{prefix}_[0-9]{{8}}_[0-9]{{6}}$"),
472
+ )
473
+ for (old,) in cur.fetchall()[keep:]:
474
+ try:
475
+ cur.execute("SAVEPOINT prune")
476
+ cur.execute(f'DROP TABLE "{db.backup}"."{old}"')
477
+ except psycopg2.Error as e:
478
+ cur.execute("ROLLBACK TO SAVEPOINT prune")
479
+ kept.append(f"old backup {db.backup}.{old} was kept: {str(e).strip().splitlines()[0]}")
480
+ conn.commit()
481
+ return "; ".join(kept)
482
+
483
+
484
+ def drop_staged(conn, db: Database, staged: list[Staged]) -> None:
485
+ conn.rollback()
486
+ with conn.cursor() as cur:
487
+ for s in staged:
488
+ cur.execute(f'DROP TABLE IF EXISTS "{db.staging}"."{s.staging}"')
489
+ conn.commit()
490
+
491
+
492
+ def load_all(conn, db: Database, layers: list[Layer], drop_invalid: bool = True, min_rows: int = 1, keep: int = 3) -> list[Staged]:
493
+ """Stage and check every layer, then swap them all in; on any failure, live tables are untouched."""
494
+ ensure_schemas(conn, db.live, db.staging, db.backup)
495
+ staged: list[Staged] = []
496
+ try:
497
+ for layer in layers:
498
+ s = Staged(layer, staging_name(layer.table))
499
+ staged.append(s)
500
+ s.note = load(db, layer, s.staging)
501
+ check(conn, db, s, drop_invalid, min_rows)
502
+ swap(conn, db, staged, keep)
503
+ except BaseException:
504
+ try:
505
+ drop_staged(conn, db, staged)
506
+ except psycopg2.Error:
507
+ pass # the connection is gone: let the error that got us here be the one reported
508
+ raise
509
+ return staged
@@ -0,0 +1,132 @@
1
+ """Read every resource in a project directory, keeping where each one came from."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import re
7
+ from dataclasses import dataclass, field
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ import yaml
12
+
13
+ # Holds secret values (spec: "Secrets"); never read as configuration
14
+ SECRET_FILES = {"secrets.yaml", "secrets.yml"}
15
+
16
+
17
+ def default_table(dataset: str, layer: str) -> str:
18
+ """A Dataset layer's table when `tables` doesn't name it: <dataset>_<layer>, lowercase, a-z 0-9 _ only."""
19
+ name = re.sub(r"[^a-z0-9]+", "_", f"{dataset}_{layer}".lower()).strip("_")
20
+ if not name or name[0].isdigit():
21
+ name = f"t_{name}"
22
+ return name[:63]
23
+
24
+
25
+ @dataclass
26
+ class Problem:
27
+ """Something wrong with the files. `line` is 1-based; 0 means the whole file."""
28
+
29
+ file: Path
30
+ line: int
31
+ message: str
32
+ resource: str = "" # e.g. "Layer road-links"
33
+ severity: str = "error" # error | warning
34
+
35
+
36
+ @dataclass
37
+ class Resource:
38
+ data: Any
39
+ file: Path
40
+ node: yaml.Node = field(repr=False)
41
+
42
+ @property
43
+ def kind(self) -> str | None:
44
+ return self.data.get("kind") if isinstance(self.data, dict) else None
45
+
46
+ @property
47
+ def name(self) -> str | None:
48
+ meta = self.data.get("metadata") if isinstance(self.data, dict) else None
49
+ return meta.get("name") if isinstance(meta, dict) else None
50
+
51
+ @property
52
+ def spec(self) -> dict:
53
+ spec = self.data.get("spec") if isinstance(self.data, dict) else None
54
+ return spec if isinstance(spec, dict) else {}
55
+
56
+ @property
57
+ def label(self) -> str:
58
+ return f"{self.kind or '?'} {self.name or '?'}"
59
+
60
+ def line(self, path: tuple = ()) -> int:
61
+ """1-based line of the value at `path` (keys and list indexes), or of its nearest parent."""
62
+ node = self.node
63
+ for step in path:
64
+ child = None
65
+ if isinstance(node, yaml.MappingNode):
66
+ child = next((v for k, v in node.value if k.value == step), None)
67
+ elif isinstance(node, yaml.SequenceNode) and isinstance(step, int) and step < len(node.value):
68
+ child = node.value[step]
69
+ if child is None:
70
+ break
71
+ node = child
72
+ return node.start_mark.line + 1
73
+
74
+
75
+ @dataclass
76
+ class Project:
77
+ root: Path
78
+ resources: list[Resource]
79
+ files: list[Path]
80
+ problems: list[Problem] # files that couldn't be read
81
+
82
+ def of_kind(self, kind: str) -> list[Resource]:
83
+ return [r for r in self.resources if r.kind == kind]
84
+
85
+ def named(self, kind: str) -> dict[str, Resource]:
86
+ """First resource of each name (duplicates are reported by validate)."""
87
+ out: dict[str, Resource] = {}
88
+ for r in self.of_kind(kind):
89
+ if r.name is not None:
90
+ out.setdefault(r.name, r)
91
+ return out
92
+
93
+
94
+ def find_files(root: Path) -> list[Path]:
95
+ """
96
+ Every YAML file under `root`, hidden ones left out. Symlinked folders are
97
+ followed: Kubernetes mounts a ConfigMap as symlinks into a hidden ..data
98
+ folder, so a project's datasets/ is a link to a directory there.
99
+ """
100
+ files = []
101
+ for folder, dirs, names in os.walk(root, followlinks=True):
102
+ dirs[:] = [d for d in dirs if not d.startswith(".")]
103
+ for name in names:
104
+ path = Path(folder) / name
105
+ if (not name.startswith(".") and path.suffix in (".yaml", ".yml") and name not in SECRET_FILES
106
+ and path.is_file()):
107
+ files.append(path)
108
+ return sorted(files)
109
+
110
+
111
+ def load(root: Path) -> Project:
112
+ root = Path(root)
113
+ resources: list[Resource] = []
114
+ problems: list[Problem] = []
115
+ files = find_files(root)
116
+ for path in files:
117
+ try:
118
+ text = path.read_text(encoding="utf-8")
119
+ for node in yaml.compose_all(text, Loader=yaml.SafeLoader):
120
+ if node is None:
121
+ continue
122
+ data = yaml.SafeLoader(text).construct_document(node)
123
+ if data is None:
124
+ continue
125
+ resources.append(Resource(data=data, file=path, node=node))
126
+ except yaml.YAMLError as e:
127
+ mark = getattr(e, "problem_mark", None)
128
+ reason = getattr(e, "problem", None) or str(e)
129
+ problems.append(Problem(path, mark.line + 1 if mark else 0, f"not valid YAML: {reason}"))
130
+ except UnicodeDecodeError:
131
+ problems.append(Problem(path, 0, "not UTF-8 text"))
132
+ return Project(root=root, resources=resources, files=files, problems=problems)