caeli-data 0.2.0__tar.gz → 0.2.2__tar.gz

{caeli_data-0.2.0 → caeli_data-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: caeli-data
-Version: 0.2.0
+Version: 0.2.2
 Summary: Python client of the Caeli data repository API (data.caeli.space): load tables, publish rasters, read the catalogue
 Author-email: "Caeli B.V." <tim@caeli.nl>
 License: Proprietary. Copyright Caeli B.V. The library is free to install; the data repository it talks to needs an API key from Caeli.
@@ -42,7 +42,7 @@ A complete month delivered again is a new version of that month: `mode="replace"
 A service takes its key from Secrets Manager: `Repository.from_secret("caeli/data-platform/api-key-inventory-platform")`.
 
 The rules a table file must meet are in Caeli_data_repository_conventions (section 6); the library checks them before
-uploading and says which rule a file breaks (`TableRejected`). `IngestFailed` means the platform failed; the same file
+uploading and says which rule a file breaks (`TableRejected`; for an extra column the message carries the schema change to make). `IngestFailed` means the platform failed; the same file
 sent again is the same job, never a double load. Python 3.9 and later.
 
 Rasters, the read side (catalogue, versions, downloads) and the command line follow in the next versions.

{caeli_data-0.2.0 → caeli_data-0.2.2}/README.md

@@ -22,7 +22,7 @@ A complete month delivered again is a new version of that month: `mode="replace"
 A service takes its key from Secrets Manager: `Repository.from_secret("caeli/data-platform/api-key-inventory-platform")`.
 
 The rules a table file must meet are in Caeli_data_repository_conventions (section 6); the library checks them before
-uploading and says which rule a file breaks (`TableRejected`). `IngestFailed` means the platform failed; the same file
+uploading and says which rule a file breaks (`TableRejected`; for an extra column the message carries the schema change to make). `IngestFailed` means the platform failed; the same file
 sent again is the same job, never a double load. Python 3.9 and later.
 
 Rasters, the read side (catalogue, versions, downloads) and the command line follow in the next versions.

{caeli_data-0.2.0 → caeli_data-0.2.2}/caeli_data/__init__.py

@@ -7,5 +7,5 @@
 
 from .loader import ApiError, IngestFailed, Part, Repository, TableRejected, sha256_of  # noqa: F401
 
-__version__ = "0.2.0"
+__version__ = "0.2.2"
 __all__ = ["Repository", "Part", "ApiError", "TableRejected", "IngestFailed", "sha256_of", "__version__"]

{caeli_data-0.2.0 → caeli_data-0.2.2}/caeli_data/loader.py

@@ -20,6 +20,7 @@ load. Needs httpx and pyarrow (boto3 for from_secret); runs on Python 3.9 and la
 from __future__ import annotations
 
 import hashlib
+import logging
 import os
 import time
 import uuid
@@ -28,9 +29,14 @@ from pathlib import Path
 
 import httpx
 
+log = logging.getLogger("caeli_data")                               # the caller's logging setup decides where it goes
+
 JOB_NAMESPACE = uuid.UUID("5b9c6b4e-2c1e-4a3f-9d8a-0ae110000001")   # fixed: the same file always gives the same job id
 FINAL = ("done", "rejected", "failed")
 MAX_PART_BYTES = 900 * 1024 * 1024                                  # the API refuses files over 1 GiB; keep a margin
+STANDARD_NAMES = ("pt_region", "pt_tile", "idx_utc", "idx_lon", "idx_lat", "row_hash")   # must be in every DataFrame
+ARROW_NAMES = {"string": "string", "int32": "int", "int64": "bigint", "float": "float", "double": "double", "bool": "boolean",
+               "timestamp[us]": "timestamp", "date32[day]": "date"}  # Arrow type as a string -> platform type
 
 
 class _ArrowTypes(dict):
@@ -47,7 +53,11 @@ ARROW_TYPES = _ArrowTypes()
 
 
 class ApiError(Exception):
-    """The API refused the request (4xx) or is unavailable."""
+    """The API refused the request (4xx, status holds the code) or is unavailable (status None)."""
+
+    def __init__(self, message: str, status: int | None = None):
+        super().__init__(message)
+        self.status = status
 
 
 class TableRejected(Exception):
@@ -94,7 +104,7 @@ class Repository:
                 detail = r.json().get("detail", r.text)
             except ValueError:
                 detail = r.text
-            raise ApiError(f"{method} {path} -> {r.status_code}: {detail}")
+            raise ApiError(f"{method} {path} -> {r.status_code}: {detail}", status=r.status_code)
         return r
 
     @classmethod
@@ -123,6 +133,20 @@ class Repository:
     def dataset(self, dataset: str) -> dict:
         return self._call("GET", f"/datasets/{dataset}").json()
 
+    def register_dataset(self, dataset: str, kind: str, title: str, cadence: str, columns: list | None = None, areas=(), **fields) -> dict:
+        """POST /datasets (a registrar or admin key): the dataset is created in the record and the catalogue, and a table
+        dataset gets its Iceberg table and schema version 1 from columns = [{"name", "type", "unit"?, "description"?}].
+        areas = ["N_EU", ...] (rasters: [{"area_code", "grid"}]). fields: description, access (internal), owner_email
+        (default: the owner of the key), steward_email, licence_note, platform_key, collection_key, product_key, extra."""
+        body = {"dataset_key": dataset, "kind": kind, "title": title, "cadence": cadence, "columns": columns or [],
+                "areas": [a if isinstance(a, dict) else {"area_code": a} for a in areas], **fields}
+        self._schemas.pop(dataset, None)
+        return self._call("POST", "/datasets", json=body).json()
+
+    def add_areas(self, dataset: str, areas) -> dict:
+        """POST /datasets/{key}/areas (registrar): more areas for a dataset, ["N_AM", ...] or [{"area_code", "grid"}]."""
+        return self._call("POST", f"/datasets/{dataset}/areas", json=[a if isinstance(a, dict) else {"area_code": a} for a in areas]).json()
+
     def schema(self, dataset: str, refresh: bool = False) -> dict:
         """The current schema version of a table dataset: {"version": n, "columns": {name: type}}. Cached for the life
         of the Repository; refresh=True fetches it again. The file and DataFrame checks refresh it by themselves once
@@ -279,8 +303,10 @@ class Repository:
     def table_from_dataframe(self, df, dataset: str, drop_columns: tuple[str, ...] = ("idx_month",)):
         """A pandas DataFrame as an Arrow table in exactly the dataset's current schema: columns in order, types cast
         (int64 to int for int columns, float64 to float for float columns, timestamps to UTC without time zone in
-        microseconds), drop_columns left out. Raises TableRejected naming missing or extra columns; an extra column
-        means a schema change first (POST /datasets/<key>/schema, an admin)."""
+        microseconds), drop_columns left out. A value column the DataFrame lacks (an older product version) is filled
+        with nulls and logged; the six standard columns must be there. Raises TableRejected for a missing standard
+        column, for a value that cannot be cast (text in a number column) and for an extra column, which means a
+        schema change first (an admin, change_schema); the message then carries the API's proposal for that change."""
         import pyarrow as pa
         import pyarrow.compute as pc
         present = [c for c in df.columns if c not in drop_columns]
@@ -290,23 +316,57 @@ class Repository:
         if (missing or extra) and cached:                # the schema may have changed since it was cached: look once more
             schema = self.arrow_schema(dataset, refresh=True)
             missing, extra = _column_mismatch(schema, present)
-        expected = [f.name for f in schema]
-        if missing or extra:
-            raise TableRejected(f"{dataset}: " + "; ".join(([f"missing column(s) {', '.join(missing)}"] if missing else [])
-                                                           + ([f"extra column(s) {', '.join(extra)}: a schema change first"] if extra else [])))
-        out = df[expected].copy()
+        missing_standard = [n for n in missing if n in STANDARD_NAMES]
+        if missing_standard:
+            raise TableRejected(f"{dataset}: missing standard column(s) {', '.join(missing_standard)}")
+        if extra:
+            raise TableRejected(f"{dataset}: extra column(s) {', '.join(extra)}: a schema change first; " + self._proposal(dataset, df, present, extra))
+        if missing:
+            log.warning("%s: value column(s) %s not in the DataFrame, filled with nulls", dataset, ", ".join(missing))
+        out = df[[n for n in (f.name for f in schema) if n not in missing]].copy()
         for f in schema:
-            if pa.types.is_timestamp(f.type) and getattr(getattr(out[f.name], "dt", None), "tz", None) is not None:
+            if f.name in out and pa.types.is_timestamp(f.type) and getattr(getattr(out[f.name], "dt", None), "tz", None) is not None:
                 out[f.name] = out[f.name].dt.tz_convert("UTC").dt.tz_localize(None)
-        table = pa.Table.from_pandas(out, preserve_index=False)
+        try:
+            table = pa.Table.from_pandas(out, preserve_index=False)
+        except (pa.ArrowInvalid, pa.ArrowTypeError) as exc:
+            raise TableRejected(f"{dataset}: a column holds values Arrow cannot read ({exc})") from exc
         arrays = []
         for f in schema:
+            if f.name in missing:
+                arrays.append(pa.nulls(table.num_rows, f.type))
+                continue
             col = table[f.name]
             if col.type != f.type:
-                col = pc.cast(col, f.type, safe=not (pa.types.is_floating(f.type) or pa.types.is_timestamp(f.type)))
+                try:
+                    col = pc.cast(col, f.type, safe=not (pa.types.is_floating(f.type) or pa.types.is_timestamp(f.type)))
+                except (pa.ArrowInvalid, pa.ArrowNotImplementedError, pa.ArrowTypeError) as exc:
+                    raise TableRejected(f"{dataset}: {f.name} is {col.type} in the DataFrame and cannot become {ARROW_NAMES.get(str(f.type), f.type)} "
+                                        f"as registered ({exc})") from exc
             arrays.append(col)
         return pa.Table.from_arrays(arrays, schema=schema)
 
+    def _proposal(self, dataset: str, df, present: list, extra: list) -> str:
+        """The schema change an admin would make for the extra columns, as the API proposes it (types read from the
+        DataFrame); built locally when the API cannot be asked."""
+        import pyarrow as pa
+        try:
+            from . import checks
+        except ImportError:
+            import checks
+        types = {}
+        for name in present:
+            try:
+                t = checks.arrow_type_name(pa.Table.from_pandas(df[[name]], preserve_index=False).schema[0].type)
+            except Exception:                            # noqa: BLE001  (mixed values; the admin decides)
+                t = "string"
+            types[name] = "timestamp" if t.startswith("timestamp") else (t if t in ARROW_NAMES.values() else "string")
+        try:
+            proposal = (self.check_schema(dataset, types).get("proposal") or {}).get("add") or []
+        except ApiError:
+            proposal = [{"name": n, "type": types[n]} for n in extra]
+        return f"proposal for an admin: change_schema({dataset!r}, note='why', add={proposal})"
+
     def write_table_file(self, df, dataset: str, path: str | Path, drop_columns: tuple[str, ...] = ("idx_month",),
                          row_group_size: int = 100_000) -> Path:
         """Writes a DataFrame as a Parquet file that meets the dataset's schema (zstd); the file to hand to load_file."""
@@ -341,7 +401,9 @@ class Repository:
                       widen: list[dict] | None = None) -> dict:
         """The next schema version (admin): add = [{"name", "type", "unit"?, "description"?}], drop = [names],
         widen = [{"name", "type"}] (int to bigint, float to double). Deliberate and recorded, never automatic."""
-        return self._call("POST", f"/datasets/{dataset}/schema", json={"add": add or [], "drop": drop or [], "widen": widen or [], "note": note}).json()
+        out = self._call("POST", f"/datasets/{dataset}/schema", json={"add": add or [], "drop": drop or [], "widen": widen or [], "note": note}).json()
+        self._schemas.pop(dataset, None)                 # the next load reads the new version
+        return out
 
 
 def _version() -> str:

{caeli_data-0.2.0 → caeli_data-0.2.2}/caeli_data.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: caeli-data
-Version: 0.2.0
+Version: 0.2.2
 Summary: Python client of the Caeli data repository API (data.caeli.space): load tables, publish rasters, read the catalogue
 Author-email: "Caeli B.V." <tim@caeli.nl>
 License: Proprietary. Copyright Caeli B.V. The library is free to install; the data repository it talks to needs an API key from Caeli.
@@ -42,7 +42,7 @@ A complete month delivered again is a new version of that month: `mode="replace"
 A service takes its key from Secrets Manager: `Repository.from_secret("caeli/data-platform/api-key-inventory-platform")`.
 
 The rules a table file must meet are in Caeli_data_repository_conventions (section 6); the library checks them before
-uploading and says which rule a file breaks (`TableRejected`). `IngestFailed` means the platform failed; the same file
+uploading and says which rule a file breaks (`TableRejected`; for an extra column the message carries the schema change to make). `IngestFailed` means the platform failed; the same file
 sent again is the same job, never a double load. Python 3.9 and later.
 
 Rasters, the read side (catalogue, versions, downloads) and the command line follow in the next versions.

{caeli_data-0.2.0 → caeli_data-0.2.2}/tests/test_loader.py

@@ -27,10 +27,22 @@ class FakeApi:
         p, m = request.url.path, request.method
         if p == "/whoami":
             return httpx.Response(200, json={"name": "ingest", "role": "publisher"})
-        if p == "/datasets/zz-test-load/schema":
+        if p == "/datasets/zz-test-load/schema/check" and m == "POST":        # the API's proposal for extra columns
+            given = {c["name"]: c["type"] for c in json.loads(request.content)["columns"]}
+            expected = dict(STANDARD + [(c["name"], c["type"]) for c in self.columns])
+            extra = sorted(n for n in given if n not in expected)
+            return httpx.Response(200, json={"ok": not extra, "extra": extra, "proposal": {"add": [{"name": n, "type": given[n]} for n in extra]} if extra else None})
+        if p == "/datasets/zz-test-load/schema" and m == "GET":
             self.schema_calls += 1
             return httpx.Response(200, json={"current": self.version, "standard_columns": [{"name": n, "type": t} for n, t in STANDARD],
                                              "versions": [{"schema_version": 1, "columns": []}, {"schema_version": self.version, "columns": self.columns}]})
+        if p == "/datasets" and m == "POST":
+            self.registered = json.loads(request.content)
+            return httpx.Response(201, json={"dataset_key": self.registered["dataset_key"], "kind": self.registered["kind"]})
+        if p == "/datasets/zz-test-load/schema" and m == "POST":
+            self.version += 1
+            self.columns = self.columns + json.loads(request.content)["add"]
+            return httpx.Response(201, json={"dataset_key": "zz-test-load", "schema_version": self.version})
         if p == "/loads" and m == "POST":
             man = json.loads(request.content)
             if man["dataset"] == "nope":
@@ -121,9 +133,33 @@ def test_dataframe_follows_a_schema_change_and_still_refuses_a_real_extra_column
     table = repo.table_from_dataframe(df, "zz-test-load")
     assert table.column_names == [n for n, _ in STANDARD] + ["no2", "o3"] and api.schema_calls == 2
     assert table["pt_tile"].type == pa.int32() and table["idx_utc"].type == pa.timestamp("us") and table["idx_utc"][0].as_py().hour == 12
-    with pytest.raises(TableRejected, match="extra column\\(s\\) o4"):
-        repo.table_from_dataframe(df.assign(o4=[1.0]), "zz-test-load")
+    with pytest.raises(TableRejected, match="extra column\\(s\\) o4") as e:
+        repo.table_from_dataframe(df.assign(o4=[1.0], o5=["x"]), "zz-test-load")
     assert api.schema_calls == 3                                           # looked once more, the column is really not there
+    assert "change_schema('zz-test-load', note='why', add=[{'name': 'o4', 'type': 'double'}, {'name': 'o5', 'type': 'string'}])" in str(e.value)
+
+
+def test_dataframe_missing_value_column_is_filled_with_nulls_missing_standard_column_refused(repo, api, caplog):
+    pa = pytest.importorskip("pyarrow")
+    pd = pytest.importorskip("pandas")
+    api.version, api.columns = 3, [{"name": "no2", "type": "double"}, {"name": "o3", "type": "double"}]
+    df = pd.DataFrame({"pt_region": ["N_EU"], "pt_tile": [2053], "idx_utc": [pd.Timestamp("2026-09-15 12:00")], "idx_lon": [5.0], "idx_lat": [52.0],
+                       "row_hash": [1], "no2": [1.5]})                                                   # an older product version: no o3
+    with caplog.at_level("INFO", logger="caeli_data"):
+        table = repo.table_from_dataframe(df, "zz-test-load")
+    assert table.column_names[-2:] == ["no2", "o3"] and table["o3"].null_count == 1 and table["o3"].type == pa.float64()
+    assert "o3 not in the DataFrame, filled with nulls" in caplog.text
+    with pytest.raises(TableRejected, match="missing standard column\\(s\\) row_hash"):
+        repo.table_from_dataframe(df.drop(columns=["row_hash"]), "zz-test-load")
+
+
+def test_dataframe_value_that_cannot_be_cast_is_refused_with_the_column_name(repo, api):
+    pytest.importorskip("pyarrow")
+    pd = pytest.importorskip("pandas")
+    df = pd.DataFrame({"pt_region": ["N_EU"], "pt_tile": [2053], "idx_utc": [pd.Timestamp("2026-09-15 12:00")], "idx_lon": [5.0], "idx_lat": [52.0],
+                       "row_hash": [1], "no2": ["twelve"]})
+    with pytest.raises(TableRejected, match="no2 is (large_)?string in the DataFrame and cannot become double"):
+        repo.table_from_dataframe(df, "zz-test-load")
 
 
 def test_load_table_announces_uploads_and_waits(repo, api, parquet):
@@ -152,9 +188,27 @@ def test_failed_raises_ingest_failed(repo, api, parquet):
         repo.load_table(parquet, "zz-test-load", "N_EU", "2026-09", check=False, row_count=10, timeout=5, poll=0.01)
 
 
-def test_api_refusal_raises_api_error_with_detail(repo, parquet):
-    with pytest.raises(ApiError, match="unknown dataset nope"):
+def test_api_refusal_raises_api_error_with_detail_and_status(repo, parquet):
+    with pytest.raises(ApiError, match="unknown dataset nope") as e:
         repo.load_table(parquet, "nope", "N_EU", "2026-09", check=False, row_count=10, schema_version=1)
+    assert e.value.status == 422
+    with pytest.raises(ApiError) as e:
+        repo.dataset("nope")
+    assert e.value.status == 404
+
+
+def test_register_dataset_posts_the_registration_and_change_schema_forgets_the_cache(repo, api):
+    out = repo.register_dataset("zz-test-new", "table", "A test", "orbital", columns=[{"name": "no2", "type": "double", "unit": "mol m-2"}],
+                                areas=["N_EU", {"area_code": "N_AM"}], description="d", platform_key="zz", collection_key="test", product_key="new",
+                                licence_note="none")
+    assert out["dataset_key"] == "zz-test-new"
+    assert api.registered == {"dataset_key": "zz-test-new", "kind": "table", "title": "A test", "cadence": "orbital",
+                              "columns": [{"name": "no2", "type": "double", "unit": "mol m-2"}], "areas": [{"area_code": "N_EU"}, {"area_code": "N_AM"}],
+                              "description": "d", "platform_key": "zz", "collection_key": "test", "product_key": "new", "licence_note": "none"}
+    assert "owner_email" not in api.registered                      # the key's owner, unless named
+    assert repo.schema("zz-test-load")["version"] == 2
+    repo.change_schema("zz-test-load", "o3 added", add=[{"name": "o3", "type": "double"}])
+    assert repo.schema("zz-test-load")["version"] == 3 and "o3" in repo.schema("zz-test-load")["columns"]
 
 
 def test_wait_false_returns_the_queued_job_with_its_link(repo, api, parquet):