caeli-data 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- caeli_data/__init__.py +11 -0
- caeli_data/checks.py +125 -0
- caeli_data/loader.py +269 -0
- caeli_data-0.1.0.dist-info/METADATA +44 -0
- caeli_data-0.1.0.dist-info/RECORD +7 -0
- caeli_data-0.1.0.dist-info/WHEEL +5 -0
- caeli_data-0.1.0.dist-info/top_level.txt +1 -0
caeli_data/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""caeli_data: the Python client of the Caeli data repository (https://data.caeli.space).
|
|
2
|
+
|
|
3
|
+
from caeli_data import Repository
|
|
4
|
+
repo = Repository() # CAELI_API_URL, CAELI_API_KEY from the environment
|
|
5
|
+
repo.whoami()
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from .loader import ApiError, IngestFailed, Part, Repository, TableRejected, sha256_of # noqa: F401
|
|
9
|
+
|
|
10
|
+
__version__ = "0.1.0"
|
|
11
|
+
__all__ = ["Repository", "Part", "ApiError", "TableRejected", "IngestFailed", "sha256_of", "__version__"]
|
caeli_data/checks.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""The checks on a table file (conventions v0.7 section 6), the same code the ingest worker runs
|
|
2
|
+
(worker/parquet_checks.py in caeli-docker-ingest-worker; keep the two the same): the footer first (columns, types,
|
|
3
|
+
row count), then one pass over the rows (region, month, tile, nulls, unique row_hash). Each check returns the problems
|
|
4
|
+
it found as sentences; an empty list means the file passes. Running them before the upload gives the answer in
|
|
5
|
+
seconds instead of after a round trip through the queue."""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from datetime import datetime
|
|
10
|
+
|
|
11
|
+
import pyarrow as pa
|
|
12
|
+
import pyarrow.compute as pc
|
|
13
|
+
import pyarrow.parquet as pq
|
|
14
|
+
|
|
15
|
+
STANDARD = [("pt_region", "string"), ("pt_tile", "int"), ("idx_utc", "timestamp"), ("idx_lon", "double"), ("idx_lat", "double"), ("row_hash", "bigint")]
|
|
16
|
+
STANDARD_NAMES = [n for n, _ in STANDARD]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def arrow_type_name(t: pa.DataType) -> str:
|
|
20
|
+
"""The Athena name of an Arrow type; anything else comes back as Arrow spells it, which never matches."""
|
|
21
|
+
if pa.types.is_string(t) or pa.types.is_large_string(t):
|
|
22
|
+
return "string"
|
|
23
|
+
if pa.types.is_int32(t):
|
|
24
|
+
return "int"
|
|
25
|
+
if pa.types.is_int64(t):
|
|
26
|
+
return "bigint"
|
|
27
|
+
if pa.types.is_float32(t):
|
|
28
|
+
return "float"
|
|
29
|
+
if pa.types.is_float64(t):
|
|
30
|
+
return "double"
|
|
31
|
+
if pa.types.is_boolean(t):
|
|
32
|
+
return "boolean"
|
|
33
|
+
if pa.types.is_timestamp(t):
|
|
34
|
+
return "timestamp" if t.tz is None else f"timestamp with zone {t.tz}"
|
|
35
|
+
if pa.types.is_date32(t) or pa.types.is_date64(t):
|
|
36
|
+
return "date"
|
|
37
|
+
return str(t)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def col(batch, name: str):
|
|
41
|
+
"""A column of a RecordBatch by name (older pyarrow has no batch[name])."""
|
|
42
|
+
return batch.column(batch.schema.get_field_index(name))
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def month_bounds(period: str) -> tuple[datetime, datetime]:
|
|
46
|
+
"""[start, end) of a month yyyy-mm."""
|
|
47
|
+
y, m = int(period[:4]), int(period[5:7])
|
|
48
|
+
return datetime(y, m, 1), datetime(y + (m == 12), 1 if m == 12 else m + 1, 1)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def footer_problems(pf: pq.ParquetFile, expected: dict[str, str], row_count: int) -> list[str]:
|
|
52
|
+
given = {f.name: arrow_type_name(f.type) for f in pf.schema_arrow}
|
|
53
|
+
problems = []
|
|
54
|
+
if missing := sorted(n for n in expected if n not in given):
|
|
55
|
+
problems.append(f"missing column(s) {', '.join(missing)}")
|
|
56
|
+
if extra := sorted(n for n in given if n not in expected):
|
|
57
|
+
problems.append(f"extra column(s) {', '.join(extra)}: a schema change first (POST /datasets/{{key}}/schema), then load again")
|
|
58
|
+
for n in expected:
|
|
59
|
+
if n in given and given[n] != expected[n]:
|
|
60
|
+
problems.append(f"{n} is {given[n]} in the file, {expected[n]} in the schema")
|
|
61
|
+
if pf.metadata.num_rows != row_count:
|
|
62
|
+
problems.append(f"{pf.metadata.num_rows} rows in the file, {row_count} in the manifest")
|
|
63
|
+
return problems
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def row_problems(pf: pq.ParquetFile, area: str, period: str, batch_size: int = 500_000) -> list[str]:
|
|
67
|
+
"""One pass over the six standard columns. Stops at the first batch with a problem; the uniqueness of row_hash is
|
|
68
|
+
checked over the whole file at the end."""
|
|
69
|
+
start, end = month_bounds(period)
|
|
70
|
+
start_us, end_us = pa.scalar(start, type=pa.timestamp("us")), pa.scalar(end, type=pa.timestamp("us"))
|
|
71
|
+
hashes, problems, rows = [], [], 0
|
|
72
|
+
for batch in pf.iter_batches(batch_size=batch_size, columns=STANDARD_NAMES):
|
|
73
|
+
for name in STANDARD_NAMES:
|
|
74
|
+
if col(batch, name).null_count:
|
|
75
|
+
problems.append(f"{name} has null values")
|
|
76
|
+
lon, lat = col(batch, "idx_lon"), col(batch, "idx_lat")
|
|
77
|
+
if pc.any(pc.is_nan(lon)).as_py() or pc.any(pc.is_nan(lat)).as_py():
|
|
78
|
+
problems.append("idx_lon or idx_lat is NaN for some rows")
|
|
79
|
+
if problems:
|
|
80
|
+
break
|
|
81
|
+
if not pc.all(pc.equal(col(batch, "pt_region"), area)).as_py():
|
|
82
|
+
problems.append(f"pt_region other than {area} found")
|
|
83
|
+
ts = pc.cast(col(batch, "idx_utc"), pa.timestamp("us"), safe=False)
|
|
84
|
+
if not pc.all(pc.and_(pc.greater_equal(ts, start_us), pc.less(ts, end_us))).as_py():
|
|
85
|
+
problems.append(f"idx_utc outside {period} found")
|
|
86
|
+
ty = pc.min_element_wise(pc.cast(pc.floor(pc.divide(pc.add(lat, 90.0), 5.0)), pa.int32()), 35)
|
|
87
|
+
tx = pc.min_element_wise(pc.cast(pc.floor(pc.divide(pc.add(lon, 180.0), 5.0)), pa.int32()), 71)
|
|
88
|
+
tile = pc.add(pc.multiply(ty, 72), tx)
|
|
89
|
+
if not pc.all(pc.equal(tile, pc.cast(col(batch, "pt_tile"), pa.int32()))).as_py():
|
|
90
|
+
problems.append("pt_tile does not match floor((lat+90)/5)*72 + floor((lon+180)/5) for some rows")
|
|
91
|
+
if problems:
|
|
92
|
+
break
|
|
93
|
+
hashes.append(col(batch, "row_hash"))
|
|
94
|
+
rows += batch.num_rows
|
|
95
|
+
if not problems and hashes:
|
|
96
|
+
unique = len(pc.unique(pa.chunked_array(hashes)))
|
|
97
|
+
if unique != rows:
|
|
98
|
+
problems.append(f"row_hash is not unique in the file: {rows - unique} duplicate(s)")
|
|
99
|
+
return problems
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def row_group_batches(pf: pq.ParquetFile, target_bytes: int = 256 * 1024 * 1024) -> list[list[int]]:
|
|
103
|
+
"""Row groups bundled so that each bundle is at most about target_bytes uncompressed (one Iceberg write each)."""
|
|
104
|
+
batches, current, size = [], [], 0
|
|
105
|
+
for i in range(pf.metadata.num_row_groups):
|
|
106
|
+
rg = pf.metadata.row_group(i).total_byte_size
|
|
107
|
+
if current and size + rg > target_bytes:
|
|
108
|
+
batches.append(current)
|
|
109
|
+
current, size = [], 0
|
|
110
|
+
current.append(i)
|
|
111
|
+
size += rg
|
|
112
|
+
if current:
|
|
113
|
+
batches.append(current)
|
|
114
|
+
return batches
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def conform(table: pa.Table, target: pa.Schema) -> pa.Table:
|
|
118
|
+
"""The file's columns in the Iceberg table's order and exact Arrow types (timestamps to microseconds)."""
|
|
119
|
+
arrays = []
|
|
120
|
+
for f in target:
|
|
121
|
+
col = table[f.name]
|
|
122
|
+
if col.type != f.type:
|
|
123
|
+
col = pc.cast(col, f.type, safe=not pa.types.is_timestamp(f.type))
|
|
124
|
+
arrays.append(col)
|
|
125
|
+
return pa.Table.from_arrays(arrays, schema=target)
|
caeli_data/loader.py
ADDED
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
"""The client of the Caeli data repository API, table side (the seed of the whole caeli_data library).
|
|
2
|
+
|
|
3
|
+
from caeli_data import Repository, TableRejected, IngestFailed
|
|
4
|
+
|
|
5
|
+
repo = Repository() # CAELI_API_URL (default https://data.caeli.space), CAELI_API_KEY
|
|
6
|
+
repo = Repository.from_secret("caeli/data-platform/api-key-inventory-platform") # a service: key from Secrets Manager
|
|
7
|
+
parts = repo.split_table_file("prepared.parquet", "out/") # one file per partition region and month
|
|
8
|
+
for part in parts:
|
|
9
|
+
job = repo.load_table(part.path, dataset="cdse-sentinel-5p-l2-no2", area=part.area, period=part.period,
|
|
10
|
+
software={"inventory-platform": "2.3.1"}, source_ref="download 4711")
|
|
11
|
+
print(job["status"], job["message"]) # "done", "... item N_EU__2026-09__v1 now 123456 rows ..."
|
|
12
|
+
|
|
13
|
+
load_table checks the file locally first (the same checks as the worker), announces the load, uploads to the link the
|
|
14
|
+
API hands out and, with wait=True, follows the job until the worker is done: it returns the job when done and raises
|
|
15
|
+
TableRejected (the file broke a rule; fix the file, new job) or IngestFailed (ours; try again later). With wait=False it
|
|
16
|
+
returns the queued job at once; repo.job(job_id) or repo.wait(job_id) tell the rest later. The job id is derived from
|
|
17
|
+
dataset, area, period, mode and the file's sha256, so sending the same file twice is the same job and never a double
|
|
18
|
+
load. Needs httpx and pyarrow (boto3 for from_secret); runs on Python 3.9 and later."""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import hashlib
|
|
23
|
+
import os
|
|
24
|
+
import time
|
|
25
|
+
import uuid
|
|
26
|
+
from dataclasses import dataclass
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
import httpx
|
|
30
|
+
|
|
31
|
+
JOB_NAMESPACE = uuid.UUID("5b9c6b4e-2c1e-4a3f-9d8a-0ae110000001") # fixed: the same file always gives the same job id
|
|
32
|
+
FINAL = ("done", "rejected", "failed")
|
|
33
|
+
MAX_PART_BYTES = 900 * 1024 * 1024 # the API refuses files over 1 GiB; keep a margin
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class ApiError(Exception):
|
|
37
|
+
"""The API refused the request (4xx) or is unavailable."""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class TableRejected(Exception):
|
|
41
|
+
"""The file broke a rule (section 6 of the conventions); the message says which."""
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class IngestFailed(Exception):
|
|
45
|
+
"""The worker could not load the file for a reason on our side; announce again later."""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass
|
|
49
|
+
class Part:
|
|
50
|
+
area: str
|
|
51
|
+
period: str
|
|
52
|
+
path: Path
|
|
53
|
+
row_count: int
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def sha256_of(path: Path) -> str:
|
|
57
|
+
h = hashlib.sha256()
|
|
58
|
+
with open(path, "rb") as f:
|
|
59
|
+
for chunk in iter(lambda: f.read(8 * 1024 * 1024), b""):
|
|
60
|
+
h.update(chunk)
|
|
61
|
+
return h.hexdigest()
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class Repository:
|
|
65
|
+
def __init__(self, api_url: str | None = None, api_key: str | None = None, timeout: float = 60.0):
|
|
66
|
+
self.api_url = (api_url or os.environ.get("CAELI_API_URL") or "https://data.caeli.space").rstrip("/")
|
|
67
|
+
key = api_key or os.environ.get("CAELI_API_KEY")
|
|
68
|
+
if not key:
|
|
69
|
+
raise ApiError("no API key: pass api_key or set CAELI_API_KEY")
|
|
70
|
+
self._client = httpx.Client(base_url=self.api_url, headers={"X-API-Key": key}, timeout=timeout)
|
|
71
|
+
self._schemas: dict[str, dict] = {}
|
|
72
|
+
|
|
73
|
+
# ---- small calls ------------------------------------------------------------------------------------------
|
|
74
|
+
def _call(self, method: str, path: str, **kwargs) -> httpx.Response:
|
|
75
|
+
try:
|
|
76
|
+
r = self._client.request(method, path, **kwargs)
|
|
77
|
+
except httpx.HTTPError as exc:
|
|
78
|
+
raise ApiError(f"{method} {path}: {exc}") from exc
|
|
79
|
+
if r.status_code >= 400:
|
|
80
|
+
try:
|
|
81
|
+
detail = r.json().get("detail", r.text)
|
|
82
|
+
except ValueError:
|
|
83
|
+
detail = r.text
|
|
84
|
+
raise ApiError(f"{method} {path} -> {r.status_code}: {detail}")
|
|
85
|
+
return r
|
|
86
|
+
|
|
87
|
+
@classmethod
|
|
88
|
+
def from_secret(cls, secret_id: str, profile: str | None = None, region: str = "eu-west-1", **kwargs) -> "Repository":
|
|
89
|
+
"""A Repository whose key comes from a Secrets Manager secret (JSON with api_key and, optionally, api_url), the
|
|
90
|
+
way services hold their keys. profile names an AWS profile on a laptop; services use their role."""
|
|
91
|
+
import json
|
|
92
|
+
import boto3
|
|
93
|
+
session = boto3.Session(profile_name=profile, region_name=region)
|
|
94
|
+
value = json.loads(session.client("secretsmanager").get_secret_value(SecretId=secret_id)["SecretString"])
|
|
95
|
+
return cls(api_url=kwargs.pop("api_url", None) or value.get("api_url"), api_key=value["api_key"], **kwargs)
|
|
96
|
+
|
|
97
|
+
def whoami(self) -> dict:
|
|
98
|
+
return self._call("GET", "/whoami").json()
|
|
99
|
+
|
|
100
|
+
def datasets(self) -> list[dict]:
|
|
101
|
+
return self._call("GET", "/datasets").json()
|
|
102
|
+
|
|
103
|
+
def areas(self) -> list[dict]:
|
|
104
|
+
return self._call("GET", "/areas").json()
|
|
105
|
+
|
|
106
|
+
def jobs(self, **params) -> list[dict]:
|
|
107
|
+
"""The caller's jobs (admins: everyone's); params as GET /jobs accepts them, e.g. status="queued"."""
|
|
108
|
+
return self._call("GET", "/jobs", params=params or None).json()
|
|
109
|
+
|
|
110
|
+
def dataset(self, dataset: str) -> dict:
|
|
111
|
+
return self._call("GET", f"/datasets/{dataset}").json()
|
|
112
|
+
|
|
113
|
+
def schema(self, dataset: str) -> dict:
|
|
114
|
+
"""The current schema version of a table dataset: {"version": n, "columns": {name: type}} (cached)."""
|
|
115
|
+
if dataset not in self._schemas:
|
|
116
|
+
s = self._call("GET", f"/datasets/{dataset}/schema").json()
|
|
117
|
+
current = next(v for v in s["versions"] if v["schema_version"] == s["current"])
|
|
118
|
+
columns = {c["name"]: c["type"] for c in s["standard_columns"]} | {c["name"]: c["type"] for c in current["columns"]}
|
|
119
|
+
self._schemas[dataset] = {"version": s["current"], "columns": columns}
|
|
120
|
+
return self._schemas[dataset]
|
|
121
|
+
|
|
122
|
+
def job(self, job_id: str) -> dict:
|
|
123
|
+
return self._call("GET", f"/jobs/{job_id}").json()
|
|
124
|
+
|
|
125
|
+
def wait(self, job_id: str, timeout: float = 3600.0, poll: float = 5.0) -> dict:
|
|
126
|
+
"""The job once the worker is done with it; raises when it was rejected or failed."""
|
|
127
|
+
t0 = time.time()
|
|
128
|
+
while True:
|
|
129
|
+
j = self.job(job_id)
|
|
130
|
+
if j["status"] in FINAL:
|
|
131
|
+
return _raise_if_not_done(j)
|
|
132
|
+
if time.time() - t0 > timeout:
|
|
133
|
+
raise IngestFailed(f"job {job_id} is still {j['status']} after {int(timeout)} s")
|
|
134
|
+
time.sleep(poll)
|
|
135
|
+
poll = min(poll * 1.5, 30.0)
|
|
136
|
+
|
|
137
|
+
# ---- files --------------------------------------------------------------------------------------------------
|
|
138
|
+
def check_table_file(self, path: str | Path, dataset: str, area: str, period: str) -> list[str]:
|
|
139
|
+
"""The worker's checks, run here: an empty list means the file will pass."""
|
|
140
|
+
import pyarrow.parquet as pq
|
|
141
|
+
try:
|
|
142
|
+
from . import checks # as part of the caeli_data package
|
|
143
|
+
except ImportError:
|
|
144
|
+
import checks # as a flat file next to a Glue job script
|
|
145
|
+
pf = pq.ParquetFile(str(path))
|
|
146
|
+
problems = checks.footer_problems(pf, self.schema(dataset)["columns"], pf.metadata.num_rows)
|
|
147
|
+
return problems or checks.row_problems(pf, area, period)
|
|
148
|
+
|
|
149
|
+
def split_table_file(self, path: str | Path, out_dir: str | Path, prefix: str | None = None,
|
|
150
|
+
max_bytes: int = MAX_PART_BYTES, drop_columns: tuple[str, ...] = ()) -> list[Part]:
|
|
151
|
+
"""Splits a prepared Parquet file into one file per (pt_region, month of idx_utc), the unit of a load, streaming
|
|
152
|
+
through the file batch by batch. The parts are named <prefix>__<area>__<period>.parquet. A part larger than
|
|
153
|
+
max_bytes (the API refuses files over 1 GiB) is cut once more, by rows, into pieces named ...__p01.parquet,
|
|
154
|
+
...__p02.parquet; each piece is a load of its own (append = accretion, so the month simply grows). Columns named
|
|
155
|
+
in drop_columns are left out of the parts (helper columns of a prepare step, such as idx_month)."""
|
|
156
|
+
import pyarrow as pa
|
|
157
|
+
import pyarrow.compute as pc
|
|
158
|
+
import pyarrow.parquet as pq
|
|
159
|
+
path, out_dir = Path(path), Path(out_dir)
|
|
160
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
161
|
+
prefix = prefix or path.stem
|
|
162
|
+
pf = pq.ParquetFile(str(path))
|
|
163
|
+
keep = [f.name for f in pf.schema_arrow if f.name not in drop_columns]
|
|
164
|
+
schema = pa.schema([f for f in pf.schema_arrow if f.name in keep], metadata=None)
|
|
165
|
+
writers: dict[tuple[str, str], tuple[pq.ParquetWriter, Path, int]] = {}
|
|
166
|
+
try:
|
|
167
|
+
for batch in pf.iter_batches(batch_size=500_000, columns=keep):
|
|
168
|
+
table = pa.Table.from_batches([batch])
|
|
169
|
+
ts = pc.cast(table["idx_utc"], pa.timestamp("us"))
|
|
170
|
+
ym = pc.add(pc.multiply(pc.cast(pc.year(ts), pa.int64()), 100), pc.cast(pc.month(ts), pa.int64()))
|
|
171
|
+
keys = pc.binary_join_element_wise(pc.cast(table["pt_region"], pa.string()), pc.cast(ym, pa.string()), "|")
|
|
172
|
+
for key in pc.unique(keys).to_pylist():
|
|
173
|
+
area, ym_text = key.split("|")
|
|
174
|
+
period = f"{int(ym_text) // 100:04d}-{int(ym_text) % 100:02d}"
|
|
175
|
+
piece = table.filter(pc.equal(keys, key))
|
|
176
|
+
if (area, period) not in writers:
|
|
177
|
+
target = out_dir / f"{prefix}__{area}__{period}.parquet"
|
|
178
|
+
writers[(area, period)] = (pq.ParquetWriter(str(target), schema, compression="zstd"), target, 0)
|
|
179
|
+
writer, target, n = writers[(area, period)]
|
|
180
|
+
writer.write_table(piece)
|
|
181
|
+
writers[(area, period)] = (writer, target, n + piece.num_rows)
|
|
182
|
+
finally:
|
|
183
|
+
for writer, _, _ in writers.values():
|
|
184
|
+
writer.close()
|
|
185
|
+
parts = []
|
|
186
|
+
for (area, period), (_, target, n) in writers.items():
|
|
187
|
+
parts.extend(_cut(target, area, period, max_bytes) if target.stat().st_size > max_bytes else [Part(area, period, target, n)])
|
|
188
|
+
return sorted(parts, key=lambda p: (p.area, p.period, p.path.name))
|
|
189
|
+
|
|
190
|
+
def load_table(self, path: str | Path, dataset: str, area: str, period: str, mode: str = "append", reason: str | None = None,
|
|
191
|
+
software: dict[str, str] | None = None, source_ref: str | None = None, schema_version: int | None = None,
|
|
192
|
+
check: bool = True, wait: bool = True, timeout: float = 3600.0, job_id: str | None = None,
|
|
193
|
+
row_count: int | None = None, poll: float = 5.0) -> dict:
|
|
194
|
+
"""Announces, uploads and (with wait) follows one load; returns the job. Raises TableRejected before the upload
|
|
195
|
+
when the local check finds a problem, TableRejected/IngestFailed after it when the worker does."""
|
|
196
|
+
path = Path(path)
|
|
197
|
+
if schema_version is None:
|
|
198
|
+
schema_version = self.schema(dataset)["version"]
|
|
199
|
+
if check:
|
|
200
|
+
problems = self.check_table_file(path, dataset, area, period)
|
|
201
|
+
if problems:
|
|
202
|
+
raise TableRejected(f"{path.name} would be refused: " + "; ".join(problems))
|
|
203
|
+
if row_count is None:
|
|
204
|
+
import pyarrow.parquet as pq
|
|
205
|
+
row_count = pq.ParquetFile(str(path)).metadata.num_rows
|
|
206
|
+
digest, size = sha256_of(path), path.stat().st_size
|
|
207
|
+
manifest = {"job_id": job_id or str(uuid.uuid5(JOB_NAMESPACE, f"{dataset}|{area}|{period}|{mode}|{digest}")),
|
|
208
|
+
"dataset": dataset, "area": area, "period": period, "schema_version": schema_version, "mode": mode,
|
|
209
|
+
"reason": reason, "row_count": row_count, "sha256": digest, "size_bytes": size,
|
|
210
|
+
"software": software or {"caeli_data": "0.1"}, "source_ref": source_ref}
|
|
211
|
+
job = self._call("POST", "/loads", json=manifest).json()
|
|
212
|
+
if job["status"] in FINAL: # the same file was sent before and is already through
|
|
213
|
+
return _raise_if_not_done(job)
|
|
214
|
+
if job.get("upload_link"): # queued and nothing received yet: upload
|
|
215
|
+
link = job["upload_link"]
|
|
216
|
+
for attempt in range(3):
|
|
217
|
+
try:
|
|
218
|
+
with open(path, "rb") as f:
|
|
219
|
+
r = httpx.put(link["url"], content=f, timeout=max(timeout, 600.0))
|
|
220
|
+
r.raise_for_status()
|
|
221
|
+
break
|
|
222
|
+
except httpx.HTTPError as exc:
|
|
223
|
+
if attempt == 2:
|
|
224
|
+
raise IngestFailed(f"upload of {path.name} failed three times: {exc}") from exc
|
|
225
|
+
time.sleep(5 * (attempt + 1))
|
|
226
|
+
link = self._call("GET", f"/jobs/{job['job_id']}/upload-link").json()["upload_link"]
|
|
227
|
+
return self.wait(job["job_id"], timeout, poll) if wait else job
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _cut(part: Path, area: str, period: str, max_bytes: int) -> list[Part]:
|
|
231
|
+
"""One part cut by rows into pieces of at most max_bytes each (aimed at 80 %, cut again when a piece still comes out
|
|
232
|
+
larger, which happens when the rows compress unevenly)."""
|
|
233
|
+
import pyarrow as pa
|
|
234
|
+
import pyarrow.parquet as pq
|
|
235
|
+
out, writer, written, k = [], None, 0, 0
|
|
236
|
+
with open(part, "rb") as f: # an explicit handle, closed before the unlink (Windows)
|
|
237
|
+
pf = pq.ParquetFile(f)
|
|
238
|
+
if pf.metadata.num_rows < 2: # cannot be cut any further
|
|
239
|
+
return [Part(area, period, part, pf.metadata.num_rows)]
|
|
240
|
+
pieces = max(2, -(-part.stat().st_size // int(max_bytes * 0.8)))
|
|
241
|
+
rows_per_piece = -(-pf.metadata.num_rows // pieces)
|
|
242
|
+
try:
|
|
243
|
+
for batch in pf.iter_batches(batch_size=min(200_000, rows_per_piece)):
|
|
244
|
+
if writer is None or written >= rows_per_piece:
|
|
245
|
+
if writer is not None:
|
|
246
|
+
writer.close()
|
|
247
|
+
k += 1
|
|
248
|
+
target = part.with_name(f"{part.stem}__p{k:02d}.parquet")
|
|
249
|
+
writer, written = pq.ParquetWriter(str(target), pf.schema_arrow, compression="zstd"), 0
|
|
250
|
+
out.append((target, 0))
|
|
251
|
+
writer.write_table(pa.Table.from_batches([batch]))
|
|
252
|
+
written += batch.num_rows
|
|
253
|
+
out[-1] = (out[-1][0], out[-1][1] + batch.num_rows)
|
|
254
|
+
finally:
|
|
255
|
+
if writer is not None:
|
|
256
|
+
writer.close()
|
|
257
|
+
part.unlink() # the pieces replace it
|
|
258
|
+
parts = []
|
|
259
|
+
for target, n in out:
|
|
260
|
+
parts.extend(_cut(target, area, period, max_bytes) if target.stat().st_size > max_bytes else [Part(area, period, target, n)])
|
|
261
|
+
return parts
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _raise_if_not_done(job: dict) -> dict:
|
|
265
|
+
if job["status"] == "rejected":
|
|
266
|
+
raise TableRejected(f"job {job['job_id']} rejected: {job.get('message')}")
|
|
267
|
+
if job["status"] == "failed":
|
|
268
|
+
raise IngestFailed(f"job {job['job_id']} failed: {job.get('message')}")
|
|
269
|
+
return job
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: caeli-data
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Python client of the Caeli data repository API (data.caeli.space): load tables, publish rasters, read the catalogue
|
|
5
|
+
Author-email: "Caeli B.V." <tim@caeli.nl>
|
|
6
|
+
License: Proprietary. Copyright Caeli B.V. The library is free to install; the data repository it talks to needs an API key from Caeli.
|
|
7
|
+
Project-URL: Homepage, https://data.caeli.space
|
|
8
|
+
Project-URL: Catalogue, https://catalog.caeli.space
|
|
9
|
+
Keywords: caeli,air quality,STAC,Iceberg,data repository
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: License :: Other/Proprietary License
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
|
|
14
|
+
Requires-Python: >=3.9
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
Requires-Dist: httpx>=0.27
|
|
17
|
+
Requires-Dist: pyarrow>=14
|
|
18
|
+
Provides-Extra: aws
|
|
19
|
+
Requires-Dist: boto3>=1.26; extra == "aws"
|
|
20
|
+
|
|
21
|
+
# caeli-data
|
|
22
|
+
|
|
23
|
+
The Python client of the Caeli data repository API (https://data.caeli.space). One door for every producer and reader
|
|
24
|
+
of datasets: the inventory platform, pipelines and colleagues all use this library; nothing writes to the tables or the
|
|
25
|
+
buckets directly.
|
|
26
|
+
|
|
27
|
+
pip install caeli-data # from the CodeArtifact repository caeli-pypi (see docs/library.md)
|
|
28
|
+
|
|
29
|
+
from caeli_data import Repository, TableRejected, IngestFailed
|
|
30
|
+
|
|
31
|
+
repo = Repository() # CAELI_API_URL (default https://data.caeli.space) and CAELI_API_KEY
|
|
32
|
+
repo.whoami() # {"name": "tim", "role": "admin", ...}
|
|
33
|
+
|
|
34
|
+
# a prepared Parquet file of a table dataset: split per partition region and month, load every part
|
|
35
|
+
for part in repo.split_table_file("prepared.parquet", "out/"):
|
|
36
|
+
job = repo.load_table(part.path, "cdse-sentinel-5p-l2-no2", part.area, part.period,
|
|
37
|
+
software={"my-pipeline": "1.0"}, source_ref="download 4711")
|
|
38
|
+
print(job["message"]) # "200000 rows appended in cdse_sentinel_5p_l2_no2, ... item N_EU__2026-09__v1 now ..."
|
|
39
|
+
|
|
40
|
+
A service takes its key from Secrets Manager: `Repository.from_secret("caeli/data-platform/api-key-inventory-platform")`.
|
|
41
|
+
|
|
42
|
+
Rasters, the read side (catalogue, versions, downloads) and the command line follow in the next versions.
|
|
43
|
+
The rules a table file must meet are in Caeli_data_repository_conventions (section 6); the library checks them before
|
|
44
|
+
uploading and says which rule a file breaks.
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
caeli_data/__init__.py,sha256=2z9nV07wCGf0pb4oQjilyFAD_ODy1wn0mEDq66-mWtk,480
|
|
2
|
+
caeli_data/checks.py,sha256=edR0XG3GZR4fA35P-JxhVMPJNNJ9pMeGgAeHU_zh6Os,5923
|
|
3
|
+
caeli_data/loader.py,sha256=T4Mq_qmZviZXWCuqD3Yuz5w3dakTLpeM74t7m4XFY8A,14839
|
|
4
|
+
caeli_data-0.1.0.dist-info/METADATA,sha256=YCs_EBUo1E5PkBiQJWhYtaPkDnvjhO7_szfE195ZtvQ,2455
|
|
5
|
+
caeli_data-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
6
|
+
caeli_data-0.1.0.dist-info/top_level.txt,sha256=utnYElINAqA_DkXy-U7IeS4eWmi-_Rbg1y1ERKty2zs,11
|
|
7
|
+
caeli_data-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
caeli_data
|