caeli-data 0.1.0__tar.gz

caeli_data-0.1.0/PKG-INFO

@@ -0,0 +1,44 @@
+Metadata-Version: 2.4
+Name: caeli-data
+Version: 0.1.0
+Summary: Python client of the Caeli data repository API (data.caeli.space): load tables, publish rasters, read the catalogue
+Author-email: "Caeli B.V." <tim@caeli.nl>
+License: Proprietary. Copyright Caeli B.V. The library is free to install; the data repository it talks to needs an API key from Caeli.
+Project-URL: Homepage, https://data.caeli.space
+Project-URL: Catalogue, https://catalog.caeli.space
+Keywords: caeli,air quality,STAC,Iceberg,data repository
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: Other/Proprietary License
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: httpx>=0.27
+Requires-Dist: pyarrow>=14
+Provides-Extra: aws
+Requires-Dist: boto3>=1.26; extra == "aws"
+
+# caeli-data
+
+The Python client of the Caeli data repository API (https://data.caeli.space). One door for every producer and reader
+of datasets: the inventory platform, pipelines and colleagues all use this library; nothing writes to the tables or the
+buckets directly.
+
+    pip install caeli-data            # from the CodeArtifact repository caeli-pypi (see docs/library.md)
+
+    from caeli_data import Repository, TableRejected, IngestFailed
+
+    repo = Repository()               # CAELI_API_URL (default https://data.caeli.space) and CAELI_API_KEY
+    repo.whoami()                     # {"name": "tim", "role": "admin", ...}
+
+    # a prepared Parquet file of a table dataset: split per partition region and month, load every part
+    for part in repo.split_table_file("prepared.parquet", "out/"):
+        job = repo.load_table(part.path, "cdse-sentinel-5p-l2-no2", part.area, part.period,
+                              software={"my-pipeline": "1.0"}, source_ref="download 4711")
+        print(job["message"])         # "200000 rows appended in cdse_sentinel_5p_l2_no2, ... item N_EU__2026-09__v1 now ..."
+
+A service takes its key from Secrets Manager: `Repository.from_secret("caeli/data-platform/api-key-inventory-platform")`.
+
+Rasters, the read side (catalogue, versions, downloads) and the command line follow in the next versions.
+The rules a table file must meet are in Caeli_data_repository_conventions (section 6); the library checks them before
+uploading and says which rule a file breaks.

caeli_data-0.1.0/README.md

@@ -0,0 +1,24 @@
+# caeli-data
+
+The Python client of the Caeli data repository API (https://data.caeli.space). One door for every producer and reader
+of datasets: the inventory platform, pipelines and colleagues all use this library; nothing writes to the tables or the
+buckets directly.
+
+    pip install caeli-data            # from the CodeArtifact repository caeli-pypi (see docs/library.md)
+
+    from caeli_data import Repository, TableRejected, IngestFailed
+
+    repo = Repository()               # CAELI_API_URL (default https://data.caeli.space) and CAELI_API_KEY
+    repo.whoami()                     # {"name": "tim", "role": "admin", ...}
+
+    # a prepared Parquet file of a table dataset: split per partition region and month, load every part
+    for part in repo.split_table_file("prepared.parquet", "out/"):
+        job = repo.load_table(part.path, "cdse-sentinel-5p-l2-no2", part.area, part.period,
+                              software={"my-pipeline": "1.0"}, source_ref="download 4711")
+        print(job["message"])         # "200000 rows appended in cdse_sentinel_5p_l2_no2, ... item N_EU__2026-09__v1 now ..."
+
+A service takes its key from Secrets Manager: `Repository.from_secret("caeli/data-platform/api-key-inventory-platform")`.
+
+Rasters, the read side (catalogue, versions, downloads) and the command line follow in the next versions.
+The rules a table file must meet are in Caeli_data_repository_conventions (section 6); the library checks them before
+uploading and says which rule a file breaks.

caeli_data-0.1.0/caeli_data/__init__.py

@@ -0,0 +1,11 @@
+"""caeli_data: the Python client of the Caeli data repository (https://data.caeli.space).
+
+    from caeli_data import Repository
+    repo = Repository()                       # CAELI_API_URL, CAELI_API_KEY from the environment
+    repo.whoami()
+"""
+
+from .loader import ApiError, IngestFailed, Part, Repository, TableRejected, sha256_of  # noqa: F401
+
+__version__ = "0.1.0"
+__all__ = ["Repository", "Part", "ApiError", "TableRejected", "IngestFailed", "sha256_of", "__version__"]

caeli_data-0.1.0/caeli_data/checks.py

@@ -0,0 +1,125 @@
+"""The checks on a table file (conventions v0.7 section 6), the same code the ingest worker runs
+(worker/parquet_checks.py in caeli-docker-ingest-worker; keep the two the same): the footer first (columns, types,
+row count), then one pass over the rows (region, month, tile, nulls, unique row_hash). Each check returns the problems
+it found as sentences; an empty list means the file passes. Running them before the upload gives the answer in
+seconds instead of after a round trip through the queue."""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+import pyarrow as pa
+import pyarrow.compute as pc
+import pyarrow.parquet as pq
+
+STANDARD = [("pt_region", "string"), ("pt_tile", "int"), ("idx_utc", "timestamp"), ("idx_lon", "double"), ("idx_lat", "double"), ("row_hash", "bigint")]
+STANDARD_NAMES = [n for n, _ in STANDARD]
+
+
+def arrow_type_name(t: pa.DataType) -> str:
+    """The Athena name of an Arrow type; anything else comes back as Arrow spells it, which never matches."""
+    if pa.types.is_string(t) or pa.types.is_large_string(t):
+        return "string"
+    if pa.types.is_int32(t):
+        return "int"
+    if pa.types.is_int64(t):
+        return "bigint"
+    if pa.types.is_float32(t):
+        return "float"
+    if pa.types.is_float64(t):
+        return "double"
+    if pa.types.is_boolean(t):
+        return "boolean"
+    if pa.types.is_timestamp(t):
+        return "timestamp" if t.tz is None else f"timestamp with zone {t.tz}"
+    if pa.types.is_date32(t) or pa.types.is_date64(t):
+        return "date"
+    return str(t)
+
+
+def col(batch, name: str):
+    """A column of a RecordBatch by name (older pyarrow has no batch[name])."""
+    return batch.column(batch.schema.get_field_index(name))
+
+
+def month_bounds(period: str) -> tuple[datetime, datetime]:
+    """[start, end) of a month yyyy-mm."""
+    y, m = int(period[:4]), int(period[5:7])
+    return datetime(y, m, 1), datetime(y + (m == 12), 1 if m == 12 else m + 1, 1)
+
+
+def footer_problems(pf: pq.ParquetFile, expected: dict[str, str], row_count: int) -> list[str]:
+    given = {f.name: arrow_type_name(f.type) for f in pf.schema_arrow}
+    problems = []
+    if missing := sorted(n for n in expected if n not in given):
+        problems.append(f"missing column(s) {', '.join(missing)}")
+    if extra := sorted(n for n in given if n not in expected):
+        problems.append(f"extra column(s) {', '.join(extra)}: a schema change first (POST /datasets/{{key}}/schema), then load again")
+    for n in expected:
+        if n in given and given[n] != expected[n]:
+            problems.append(f"{n} is {given[n]} in the file, {expected[n]} in the schema")
+    if pf.metadata.num_rows != row_count:
+        problems.append(f"{pf.metadata.num_rows} rows in the file, {row_count} in the manifest")
+    return problems
+
+
+def row_problems(pf: pq.ParquetFile, area: str, period: str, batch_size: int = 500_000) -> list[str]:
+    """One pass over the six standard columns. Stops at the first batch with a problem; the uniqueness of row_hash is
+    checked over the whole file at the end."""
+    start, end = month_bounds(period)
+    start_us, end_us = pa.scalar(start, type=pa.timestamp("us")), pa.scalar(end, type=pa.timestamp("us"))
+    hashes, problems, rows = [], [], 0
+    for batch in pf.iter_batches(batch_size=batch_size, columns=STANDARD_NAMES):
+        for name in STANDARD_NAMES:
+            if col(batch, name).null_count:
+                problems.append(f"{name} has null values")
+        lon, lat = col(batch, "idx_lon"), col(batch, "idx_lat")
+        if pc.any(pc.is_nan(lon)).as_py() or pc.any(pc.is_nan(lat)).as_py():
+            problems.append("idx_lon or idx_lat is NaN for some rows")
+        if problems:
+            break
+        if not pc.all(pc.equal(col(batch, "pt_region"), area)).as_py():
+            problems.append(f"pt_region other than {area} found")
+        ts = pc.cast(col(batch, "idx_utc"), pa.timestamp("us"), safe=False)
+        if not pc.all(pc.and_(pc.greater_equal(ts, start_us), pc.less(ts, end_us))).as_py():
+            problems.append(f"idx_utc outside {period} found")
+        ty = pc.min_element_wise(pc.cast(pc.floor(pc.divide(pc.add(lat, 90.0), 5.0)), pa.int32()), 35)
+        tx = pc.min_element_wise(pc.cast(pc.floor(pc.divide(pc.add(lon, 180.0), 5.0)), pa.int32()), 71)
+        tile = pc.add(pc.multiply(ty, 72), tx)
+        if not pc.all(pc.equal(tile, pc.cast(col(batch, "pt_tile"), pa.int32()))).as_py():
+            problems.append("pt_tile does not match floor((lat+90)/5)*72 + floor((lon+180)/5) for some rows")
+        if problems:
+            break
+        hashes.append(col(batch, "row_hash"))
+        rows += batch.num_rows
+    if not problems and hashes:
+        unique = len(pc.unique(pa.chunked_array(hashes)))
+        if unique != rows:
+            problems.append(f"row_hash is not unique in the file: {rows - unique} duplicate(s)")
+    return problems
+
+
+def row_group_batches(pf: pq.ParquetFile, target_bytes: int = 256 * 1024 * 1024) -> list[list[int]]:
+    """Row groups bundled so that each bundle is at most about target_bytes uncompressed (one Iceberg write each)."""
+    batches, current, size = [], [], 0
+    for i in range(pf.metadata.num_row_groups):
+        rg = pf.metadata.row_group(i).total_byte_size
+        if current and size + rg > target_bytes:
+            batches.append(current)
+            current, size = [], 0
+        current.append(i)
+        size += rg
+    if current:
+        batches.append(current)
+    return batches
+
+
+def conform(table: pa.Table, target: pa.Schema) -> pa.Table:
+    """The file's columns in the Iceberg table's order and exact Arrow types (timestamps to microseconds)."""
+    arrays = []
+    for f in target:
+        col = table[f.name]
+        if col.type != f.type:
+            col = pc.cast(col, f.type, safe=not pa.types.is_timestamp(f.type))
+        arrays.append(col)
+    return pa.Table.from_arrays(arrays, schema=target)

caeli_data-0.1.0/caeli_data/loader.py

@@ -0,0 +1,269 @@
+"""The client of the Caeli data repository API, table side (the seed of the whole caeli_data library).
+
+    from caeli_data import Repository, TableRejected, IngestFailed
+
+    repo = Repository()                                   # CAELI_API_URL (default https://data.caeli.space), CAELI_API_KEY
+    repo = Repository.from_secret("caeli/data-platform/api-key-inventory-platform")   # a service: key from Secrets Manager
+    parts = repo.split_table_file("prepared.parquet", "out/")          # one file per partition region and month
+    for part in parts:
+        job = repo.load_table(part.path, dataset="cdse-sentinel-5p-l2-no2", area=part.area, period=part.period,
+                              software={"inventory-platform": "2.3.1"}, source_ref="download 4711")
+        print(job["status"], job["message"])              # "done", "... item N_EU__2026-09__v1 now 123456 rows ..."
+
+load_table checks the file locally first (the same checks as the worker), announces the load, uploads to the link the
+API hands out and, with wait=True, follows the job until the worker is done: it returns the job when done and raises
+TableRejected (the file broke a rule; fix the file, new job) or IngestFailed (ours; try again later). With wait=False it
+returns the queued job at once; repo.job(job_id) or repo.wait(job_id) tell the rest later. The job id is derived from
+dataset, area, period, mode and the file's sha256, so sending the same file twice is the same job and never a double
+load. Needs httpx and pyarrow (boto3 for from_secret); runs on Python 3.9 and later."""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import time
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+
+import httpx
+
+JOB_NAMESPACE = uuid.UUID("5b9c6b4e-2c1e-4a3f-9d8a-0ae110000001")   # fixed: the same file always gives the same job id
+FINAL = ("done", "rejected", "failed")
+MAX_PART_BYTES = 900 * 1024 * 1024                                  # the API refuses files over 1 GiB; keep a margin
+
+
+class ApiError(Exception):
+    """The API refused the request (4xx) or is unavailable."""
+
+
+class TableRejected(Exception):
+    """The file broke a rule (section 6 of the conventions); the message says which."""
+
+
+class IngestFailed(Exception):
+    """The worker could not load the file for a reason on our side; announce again later."""
+
+
+@dataclass
+class Part:
+    area: str
+    period: str
+    path: Path
+    row_count: int
+
+
+def sha256_of(path: Path) -> str:
+    h = hashlib.sha256()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(8 * 1024 * 1024), b""):
+            h.update(chunk)
+    return h.hexdigest()
+
+
+class Repository:
+    def __init__(self, api_url: str | None = None, api_key: str | None = None, timeout: float = 60.0):
+        self.api_url = (api_url or os.environ.get("CAELI_API_URL") or "https://data.caeli.space").rstrip("/")
+        key = api_key or os.environ.get("CAELI_API_KEY")
+        if not key:
+            raise ApiError("no API key: pass api_key or set CAELI_API_KEY")
+        self._client = httpx.Client(base_url=self.api_url, headers={"X-API-Key": key}, timeout=timeout)
+        self._schemas: dict[str, dict] = {}
+
+    # ---- small calls ------------------------------------------------------------------------------------------
+    def _call(self, method: str, path: str, **kwargs) -> httpx.Response:
+        try:
+            r = self._client.request(method, path, **kwargs)
+        except httpx.HTTPError as exc:
+            raise ApiError(f"{method} {path}: {exc}") from exc
+        if r.status_code >= 400:
+            try:
+                detail = r.json().get("detail", r.text)
+            except ValueError:
+                detail = r.text
+            raise ApiError(f"{method} {path} -> {r.status_code}: {detail}")
+        return r
+
+    @classmethod
+    def from_secret(cls, secret_id: str, profile: str | None = None, region: str = "eu-west-1", **kwargs) -> "Repository":
+        """A Repository whose key comes from a Secrets Manager secret (JSON with api_key and, optionally, api_url), the
+        way services hold their keys. profile names an AWS profile on a laptop; services use their role."""
+        import json
+        import boto3
+        session = boto3.Session(profile_name=profile, region_name=region)
+        value = json.loads(session.client("secretsmanager").get_secret_value(SecretId=secret_id)["SecretString"])
+        return cls(api_url=kwargs.pop("api_url", None) or value.get("api_url"), api_key=value["api_key"], **kwargs)
+
+    def whoami(self) -> dict:
+        return self._call("GET", "/whoami").json()
+
+    def datasets(self) -> list[dict]:
+        return self._call("GET", "/datasets").json()
+
+    def areas(self) -> list[dict]:
+        return self._call("GET", "/areas").json()
+
+    def jobs(self, **params) -> list[dict]:
+        """The caller's jobs (admins: everyone's); params as GET /jobs accepts them, e.g. status="queued"."""
+        return self._call("GET", "/jobs", params=params or None).json()
+
+    def dataset(self, dataset: str) -> dict:
+        return self._call("GET", f"/datasets/{dataset}").json()
+
+    def schema(self, dataset: str) -> dict:
+        """The current schema version of a table dataset: {"version": n, "columns": {name: type}} (cached)."""
+        if dataset not in self._schemas:
+            s = self._call("GET", f"/datasets/{dataset}/schema").json()
+            current = next(v for v in s["versions"] if v["schema_version"] == s["current"])
+            columns = {c["name"]: c["type"] for c in s["standard_columns"]} | {c["name"]: c["type"] for c in current["columns"]}
+            self._schemas[dataset] = {"version": s["current"], "columns": columns}
+        return self._schemas[dataset]
+
+    def job(self, job_id: str) -> dict:
+        return self._call("GET", f"/jobs/{job_id}").json()
+
+    def wait(self, job_id: str, timeout: float = 3600.0, poll: float = 5.0) -> dict:
+        """The job once the worker is done with it; raises when it was rejected or failed."""
+        t0 = time.time()
+        while True:
+            j = self.job(job_id)
+            if j["status"] in FINAL:
+                return _raise_if_not_done(j)
+            if time.time() - t0 > timeout:
+                raise IngestFailed(f"job {job_id} is still {j['status']} after {int(timeout)} s")
+            time.sleep(poll)
+            poll = min(poll * 1.5, 30.0)
+
+    # ---- files --------------------------------------------------------------------------------------------------
+    def check_table_file(self, path: str | Path, dataset: str, area: str, period: str) -> list[str]:
+        """The worker's checks, run here: an empty list means the file will pass."""
+        import pyarrow.parquet as pq
+        try:
+            from . import checks                      # as part of the caeli_data package
+        except ImportError:
+            import checks                             # as a flat file next to a Glue job script
+        pf = pq.ParquetFile(str(path))
+        problems = checks.footer_problems(pf, self.schema(dataset)["columns"], pf.metadata.num_rows)
+        return problems or checks.row_problems(pf, area, period)
+
+    def split_table_file(self, path: str | Path, out_dir: str | Path, prefix: str | None = None,
+                         max_bytes: int = MAX_PART_BYTES, drop_columns: tuple[str, ...] = ()) -> list[Part]:
+        """Splits a prepared Parquet file into one file per (pt_region, month of idx_utc), the unit of a load, streaming
+        through the file batch by batch. The parts are named <prefix>__<area>__<period>.parquet. A part larger than
+        max_bytes (the API refuses files over 1 GiB) is cut once more, by rows, into pieces named ...__p01.parquet,
+        ...__p02.parquet; each piece is a load of its own (append = accretion, so the month simply grows). Columns named
+        in drop_columns are left out of the parts (helper columns of a prepare step, such as idx_month)."""
+        import pyarrow as pa
+        import pyarrow.compute as pc
+        import pyarrow.parquet as pq
+        path, out_dir = Path(path), Path(out_dir)
+        out_dir.mkdir(parents=True, exist_ok=True)
+        prefix = prefix or path.stem
+        pf = pq.ParquetFile(str(path))
+        keep = [f.name for f in pf.schema_arrow if f.name not in drop_columns]
+        schema = pa.schema([f for f in pf.schema_arrow if f.name in keep], metadata=None)
+        writers: dict[tuple[str, str], tuple[pq.ParquetWriter, Path, int]] = {}
+        try:
+            for batch in pf.iter_batches(batch_size=500_000, columns=keep):
+                table = pa.Table.from_batches([batch])
+                ts = pc.cast(table["idx_utc"], pa.timestamp("us"))
+                ym = pc.add(pc.multiply(pc.cast(pc.year(ts), pa.int64()), 100), pc.cast(pc.month(ts), pa.int64()))
+                keys = pc.binary_join_element_wise(pc.cast(table["pt_region"], pa.string()), pc.cast(ym, pa.string()), "|")
+                for key in pc.unique(keys).to_pylist():
+                    area, ym_text = key.split("|")
+                    period = f"{int(ym_text) // 100:04d}-{int(ym_text) % 100:02d}"
+                    piece = table.filter(pc.equal(keys, key))
+                    if (area, period) not in writers:
+                        target = out_dir / f"{prefix}__{area}__{period}.parquet"
+                        writers[(area, period)] = (pq.ParquetWriter(str(target), schema, compression="zstd"), target, 0)
+                    writer, target, n = writers[(area, period)]
+                    writer.write_table(piece)
+                    writers[(area, period)] = (writer, target, n + piece.num_rows)
+        finally:
+            for writer, _, _ in writers.values():
+                writer.close()
+        parts = []
+        for (area, period), (_, target, n) in writers.items():
+            parts.extend(_cut(target, area, period, max_bytes) if target.stat().st_size > max_bytes else [Part(area, period, target, n)])
+        return sorted(parts, key=lambda p: (p.area, p.period, p.path.name))
+
+    def load_table(self, path: str | Path, dataset: str, area: str, period: str, mode: str = "append", reason: str | None = None,
+                   software: dict[str, str] | None = None, source_ref: str | None = None, schema_version: int | None = None,
+                   check: bool = True, wait: bool = True, timeout: float = 3600.0, job_id: str | None = None,
+                   row_count: int | None = None, poll: float = 5.0) -> dict:
+        """Announces, uploads and (with wait) follows one load; returns the job. Raises TableRejected before the upload
+        when the local check finds a problem, TableRejected/IngestFailed after it when the worker does."""
+        path = Path(path)
+        if schema_version is None:
+            schema_version = self.schema(dataset)["version"]
+        if check:
+            problems = self.check_table_file(path, dataset, area, period)
+            if problems:
+                raise TableRejected(f"{path.name} would be refused: " + "; ".join(problems))
+        if row_count is None:
+            import pyarrow.parquet as pq
+            row_count = pq.ParquetFile(str(path)).metadata.num_rows
+        digest, size = sha256_of(path), path.stat().st_size
+        manifest = {"job_id": job_id or str(uuid.uuid5(JOB_NAMESPACE, f"{dataset}|{area}|{period}|{mode}|{digest}")),
+                    "dataset": dataset, "area": area, "period": period, "schema_version": schema_version, "mode": mode,
+                    "reason": reason, "row_count": row_count, "sha256": digest, "size_bytes": size,
+                    "software": software or {"caeli_data": "0.1"}, "source_ref": source_ref}
+        job = self._call("POST", "/loads", json=manifest).json()
+        if job["status"] in FINAL:                       # the same file was sent before and is already through
+            return _raise_if_not_done(job)
+        if job.get("upload_link"):                       # queued and nothing received yet: upload
+            link = job["upload_link"]
+            for attempt in range(3):
+                try:
+                    with open(path, "rb") as f:
+                        r = httpx.put(link["url"], content=f, timeout=max(timeout, 600.0))
+                    r.raise_for_status()
+                    break
+                except httpx.HTTPError as exc:
+                    if attempt == 2:
+                        raise IngestFailed(f"upload of {path.name} failed three times: {exc}") from exc
+                    time.sleep(5 * (attempt + 1))
+                    link = self._call("GET", f"/jobs/{job['job_id']}/upload-link").json()["upload_link"]
+        return self.wait(job["job_id"], timeout, poll) if wait else job
+
+
+def _cut(part: Path, area: str, period: str, max_bytes: int) -> list[Part]:
+    """One part cut by rows into pieces of at most max_bytes each (aimed at 80 %, cut again when a piece still comes out
+    larger, which happens when the rows compress unevenly)."""
+    import pyarrow as pa
+    import pyarrow.parquet as pq
+    out, writer, written, k = [], None, 0, 0
+    with open(part, "rb") as f:                                       # an explicit handle, closed before the unlink (Windows)
+        pf = pq.ParquetFile(f)
+        if pf.metadata.num_rows < 2:                                  # cannot be cut any further
+            return [Part(area, period, part, pf.metadata.num_rows)]
+        pieces = max(2, -(-part.stat().st_size // int(max_bytes * 0.8)))
+        rows_per_piece = -(-pf.metadata.num_rows // pieces)
+        try:
+            for batch in pf.iter_batches(batch_size=min(200_000, rows_per_piece)):
+                if writer is None or written >= rows_per_piece:
+                    if writer is not None:
+                        writer.close()
+                    k += 1
+                    target = part.with_name(f"{part.stem}__p{k:02d}.parquet")
+                    writer, written = pq.ParquetWriter(str(target), pf.schema_arrow, compression="zstd"), 0
+                    out.append((target, 0))
+                writer.write_table(pa.Table.from_batches([batch]))
+                written += batch.num_rows
+                out[-1] = (out[-1][0], out[-1][1] + batch.num_rows)
+        finally:
+            if writer is not None:
+                writer.close()
+    part.unlink()                                                     # the pieces replace it
+    parts = []
+    for target, n in out:
+        parts.extend(_cut(target, area, period, max_bytes) if target.stat().st_size > max_bytes else [Part(area, period, target, n)])
+    return parts
+
+
+def _raise_if_not_done(job: dict) -> dict:
+    if job["status"] == "rejected":
+        raise TableRejected(f"job {job['job_id']} rejected: {job.get('message')}")
+    if job["status"] == "failed":
+        raise IngestFailed(f"job {job['job_id']} failed: {job.get('message')}")
+    return job

caeli_data-0.1.0/caeli_data.egg-info/PKG-INFO

@@ -0,0 +1,44 @@
+Metadata-Version: 2.4
+Name: caeli-data
+Version: 0.1.0
+Summary: Python client of the Caeli data repository API (data.caeli.space): load tables, publish rasters, read the catalogue
+Author-email: "Caeli B.V." <tim@caeli.nl>
+License: Proprietary. Copyright Caeli B.V. The library is free to install; the data repository it talks to needs an API key from Caeli.
+Project-URL: Homepage, https://data.caeli.space
+Project-URL: Catalogue, https://catalog.caeli.space
+Keywords: caeli,air quality,STAC,Iceberg,data repository
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: Other/Proprietary License
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Scientific/Engineering :: Atmospheric Science
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: httpx>=0.27
+Requires-Dist: pyarrow>=14
+Provides-Extra: aws
+Requires-Dist: boto3>=1.26; extra == "aws"
+
+# caeli-data
+
+The Python client of the Caeli data repository API (https://data.caeli.space). One door for every producer and reader
+of datasets: the inventory platform, pipelines and colleagues all use this library; nothing writes to the tables or the
+buckets directly.
+
+    pip install caeli-data            # from the CodeArtifact repository caeli-pypi (see docs/library.md)
+
+    from caeli_data import Repository, TableRejected, IngestFailed
+
+    repo = Repository()               # CAELI_API_URL (default https://data.caeli.space) and CAELI_API_KEY
+    repo.whoami()                     # {"name": "tim", "role": "admin", ...}
+
+    # a prepared Parquet file of a table dataset: split per partition region and month, load every part
+    for part in repo.split_table_file("prepared.parquet", "out/"):
+        job = repo.load_table(part.path, "cdse-sentinel-5p-l2-no2", part.area, part.period,
+                              software={"my-pipeline": "1.0"}, source_ref="download 4711")
+        print(job["message"])         # "200000 rows appended in cdse_sentinel_5p_l2_no2, ... item N_EU__2026-09__v1 now ..."
+
+A service takes its key from Secrets Manager: `Repository.from_secret("caeli/data-platform/api-key-inventory-platform")`.
+
+Rasters, the read side (catalogue, versions, downloads) and the command line follow in the next versions.
+The rules a table file must meet are in Caeli_data_repository_conventions (section 6); the library checks them before
+uploading and says which rule a file breaks.

caeli_data-0.1.0/caeli_data.egg-info/SOURCES.txt

@@ -0,0 +1,11 @@
+README.md
+pyproject.toml
+caeli_data/__init__.py
+caeli_data/checks.py
+caeli_data/loader.py
+caeli_data.egg-info/PKG-INFO
+caeli_data.egg-info/SOURCES.txt
+caeli_data.egg-info/dependency_links.txt
+caeli_data.egg-info/requires.txt
+caeli_data.egg-info/top_level.txt
+tests/test_loader.py

caeli_data-0.1.0/caeli_data.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

caeli_data-0.1.0/caeli_data.egg-info/requires.txt

@@ -0,0 +1,5 @@
+httpx>=0.27
+pyarrow>=14
+
+[aws]
+boto3>=1.26

caeli_data-0.1.0/caeli_data.egg-info/top_level.txt

@@ -0,0 +1 @@
+caeli_data

caeli_data-0.1.0/pyproject.toml

@@ -0,0 +1,36 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "caeli-data"
+dynamic = ["version"]
+description = "Python client of the Caeli data repository API (data.caeli.space): load tables, publish rasters, read the catalogue"
+readme = "README.md"
+requires-python = ">=3.9"
+license = { text = "Proprietary. Copyright Caeli B.V. The library is free to install; the data repository it talks to needs an API key from Caeli." }
+authors = [{ name = "Caeli B.V.", email = "tim@caeli.nl" }]
+keywords = ["caeli", "air quality", "STAC", "Iceberg", "data repository"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: Other/Proprietary License",
+    "Operating System :: OS Independent",
+    "Topic :: Scientific/Engineering :: Atmospheric Science",
+]
+dependencies = [
+    "httpx>=0.27",
+    "pyarrow>=14",
+]
+
+[project.optional-dependencies]
+aws = ["boto3>=1.26"]            # Repository.from_secret
+
+[project.urls]
+Homepage = "https://data.caeli.space"
+Catalogue = "https://catalog.caeli.space"
+
+[tool.setuptools]
+packages = ["caeli_data"]
+
+[tool.setuptools.dynamic]
+version = { attr = "caeli_data.__version__" }

caeli_data-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

caeli_data-0.1.0/tests/test_loader.py

@@ -0,0 +1,126 @@
+"""Unit tests of caeli_data.loader against a fake API (httpx MockTransport); no network, no AWS, no pyarrow needed.
+Run from the repository root:  python -m pytest tests   (or  python tests/test_loader.py)."""
+
+import json
+import sys
+from pathlib import Path
+
+import httpx
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from caeli_data import ApiError, IngestFailed, Repository, TableRejected  # noqa: E402
+
+
+class FakeApi:
+    """Enough of the data API for load_table: schema, POST /loads with the same-job rule, GET /jobs/{id}, upload link."""
+
+    def __init__(self, outcome="done"):
+        self.jobs, self.uploads, self.polls, self.outcome = {}, [], 0, outcome
+
+    def handler(self, request: httpx.Request) -> httpx.Response:
+        p, m = request.url.path, request.method
+        if p == "/whoami":
+            return httpx.Response(200, json={"name": "ingest", "role": "publisher"})
+        if p == "/datasets/zz-test-load/schema":
+            return httpx.Response(200, json={"current": 2, "standard_columns": [{"name": "pt_region", "type": "string"}],
+                                             "versions": [{"schema_version": 1, "columns": []}, {"schema_version": 2, "columns": [{"name": "no2", "type": "double"}]}]})
+        if p == "/loads" and m == "POST":
+            man = json.loads(request.content)
+            if man["dataset"] == "nope":
+                return httpx.Response(422, json={"detail": "unknown dataset nope"})
+            jid = man["job_id"]
+            if jid in self.jobs:
+                return httpx.Response(200, json=self.jobs[jid])
+            self.jobs[jid] = {"job_id": jid, "status": "queued", "manifest": man, "message": None, "upload_link": {"url": "https://s3.test/put/" + jid, "method": "PUT"}}
+            return httpx.Response(201, json=self.jobs[jid])
+        if p.startswith("/jobs/") and p.endswith("/upload-link"):
+            return httpx.Response(200, json={"upload_link": {"url": "https://s3.test/put/again"}})
+        if p.startswith("/jobs/"):
+            jid = p.split("/")[2]
+            j = dict(self.jobs[jid])
+            j.pop("upload_link", None)
+            self.polls += 1
+            if self.polls >= 2:
+                j["status"] = self.outcome
+                j["message"] = "1000 rows appended" if self.outcome == "done" else "pt_tile does not match"
+                self.jobs[jid].update(j)
+            return httpx.Response(200, json=j)
+        return httpx.Response(404, json={"detail": "no"})
+
+
+@pytest.fixture
+def api(monkeypatch):
+    fake = FakeApi()
+    monkeypatch.setattr(httpx, "put", lambda url, content=None, timeout=None: (fake.uploads.append(url), httpx.Response(200, request=httpx.Request("PUT", url)))[1])
+    return fake
+
+
+@pytest.fixture
+def repo(api):
+    r = Repository(api_url="https://api.test", api_key="ck_x")
+    r._client = httpx.Client(base_url="https://api.test", headers={"X-API-Key": "ck_x"}, transport=httpx.MockTransport(api.handler))
+    return r
+
+
+@pytest.fixture
+def parquet(tmp_path):
+    f = tmp_path / "part.parquet"
+    f.write_bytes(b"x" * 1234)
+    return f
+
+
+def test_schema_is_cached_with_standard_and_value_columns(repo):
+    assert repo.schema("zz-test-load") == {"version": 2, "columns": {"pt_region": "string", "no2": "double"}}
+    assert repo.schema("zz-test-load") is repo.schema("zz-test-load")
+
+
+def test_load_table_announces_uploads_and_waits(repo, api, parquet):
+    job = repo.load_table(parquet, "zz-test-load", "N_EU", "2026-09", software={"t": "1"}, source_ref="src", check=False, row_count=1000, timeout=5, poll=0.01)
+    man = job["manifest"]
+    assert man["schema_version"] == 2 and man["row_count"] == 1000 and man["size_bytes"] == 1234 and len(man["sha256"]) == 64
+    assert man["mode"] == "append" and man["software"] == {"t": "1"} and man["source_ref"] == "src"
+    assert job["status"] == "done" and api.uploads == ["https://s3.test/put/" + job["job_id"]]
+
+
+def test_same_file_is_the_same_job_without_a_second_upload(repo, api, parquet):
+    first = repo.load_table(parquet, "zz-test-load", "N_EU", "2026-09", check=False, row_count=1000, timeout=5, poll=0.01)
+    second = repo.load_table(parquet, "zz-test-load", "N_EU", "2026-09", check=False, row_count=1000, timeout=5, poll=0.01)
+    assert first["job_id"] == second["job_id"] and len(api.uploads) == 1
+
+
+def test_rejected_raises_with_the_reason(repo, api, parquet):
+    api.outcome = "rejected"
+    with pytest.raises(TableRejected, match="pt_tile does not match"):
+        repo.load_table(parquet, "zz-test-load", "N_EU", "2026-09", check=False, row_count=10, timeout=5, poll=0.01)
+
+
+def test_failed_raises_ingest_failed(repo, api, parquet):
+    api.outcome = "failed"
+    with pytest.raises(IngestFailed):
+        repo.load_table(parquet, "zz-test-load", "N_EU", "2026-09", check=False, row_count=10, timeout=5, poll=0.01)
+
+
+def test_api_refusal_raises_api_error_with_detail(repo, parquet):
+    with pytest.raises(ApiError, match="unknown dataset nope"):
+        repo.load_table(parquet, "nope", "N_EU", "2026-09", check=False, row_count=10, schema_version=1)
+
+
+def test_wait_false_returns_the_queued_job_with_its_link(repo, api, parquet):
+    job = repo.load_table(parquet, "zz-test-load", "N_EU", "2026-09", check=False, row_count=10, wait=False)
+    assert job["status"] == "queued" and job["upload_link"]["url"].startswith("https://s3.test/put/") and len(api.uploads) == 1
+
+
+def test_wait_times_out_as_ingest_failed(repo, api, parquet):
+    api.outcome = "queued"
+    with pytest.raises(IngestFailed, match="still queued"):
+        repo.wait(repo.load_table(parquet, "zz-test-load", "N_EU", "2026-09", check=False, row_count=10, wait=False)["job_id"], timeout=0.01, poll=0.001)
+
+
+def test_replace_needs_its_mode_in_the_manifest(repo, api, parquet):
+    job = repo.load_table(parquet, "zz-test-load", "N_EU", "2026-09", mode="replace", reason="re-import", check=False, row_count=10, timeout=5, poll=0.01)
+    assert job["manifest"]["mode"] == "replace" and job["manifest"]["reason"] == "re-import"
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main([__file__, "-q"]))