pyiceberg-fxmacrodata 0.1.0__py3-none-any.whl

fxmacrodata_iceberg/__init__.py

@@ -0,0 +1,154 @@
+"""Load public FXMacroData responses into a customer-owned Apache Iceberg table."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import hashlib
+import json
+import math
+import re
+from typing import Any
+
+import pyarrow as pa
+from pyiceberg.schema import Schema
+from pyiceberg.types import DoubleType, NestedField, StringType
+
+from fxmacrodata_platform_core import COLUMNS as SOURCE_COLUMNS, read
+
+__version__ = "0.1.0"
+
+COLUMNS = tuple(SOURCE_COLUMNS) + ("content_hash",)
+_IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,127}\Z")
+
+
+class LoadError(RuntimeError):
+    """A catalog or table operation failed; inspect the destination before retrying."""
+
+
+@dataclass(frozen=True)
+class LoadResult:
+    identifier: str
+    rows: int
+    batches: int
+
+
+def arrow_schema() -> pa.Schema:
+    """Return the lossless Arrow schema: value is float64, everything else string."""
+    return pa.schema([
+        pa.field(name, pa.float64() if name == "value" else pa.string(), nullable=True)
+        for name in COLUMNS
+    ])
+
+
+def iceberg_schema() -> Schema:
+    """Return the Iceberg schema matching ``arrow_schema``; no identifier fields."""
+    return Schema(*(
+        NestedField(field_id=index, name=name,
+                    field_type=DoubleType() if name == "value" else StringType(),
+                    required=False)
+        for index, name in enumerate(COLUMNS, start=1)
+    ))
+
+
+def _rows(batch: Any) -> list[dict[str, Any]]:
+    rows = []
+    for source in batch.rows():
+        row = {name: source.get(name) for name in SOURCE_COLUMNS}
+        if not isinstance(row["operation"], str) or not isinstance(row["record_id"], str):
+            raise ValueError("every row requires string operation and record_id")
+        for name, value in row.items():
+            if name == "value":
+                if value is not None and (
+                    isinstance(value, bool) or not isinstance(value, (int, float)) or
+                    not math.isfinite(value)
+                ):
+                    raise ValueError("value must be a finite number or null")
+            elif value is not None and not isinstance(value, str):
+                raise ValueError(f"{name} must be a string or null")
+        encoded = json.dumps(row, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+        row["content_hash"] = hashlib.sha256(encoded.encode("utf-8")).hexdigest()
+        rows.append(row)
+    return rows
+
+
+def to_arrow(batch: Any) -> pa.Table:
+    """Project a Batch to an Arrow table with the eleven columns plus content_hash."""
+    return pa.Table.from_pylist(_rows(batch), schema=arrow_schema())
+
+
+def _identifier(identifier: Any) -> tuple[str, ...]:
+    if isinstance(identifier, (tuple, list)):
+        parts = tuple(identifier)
+    elif isinstance(identifier, str):
+        parts = tuple(identifier.split("."))
+    else:
+        raise ValueError("identifier must be a dotted namespace.table string or a tuple of parts")
+    if len(parts) < 2 or any(not isinstance(p, str) or not _IDENTIFIER.fullmatch(p) for p in parts):
+        raise ValueError("identifier must contain two or more simple dotted parts")
+    return parts
+
+
+def _check_batch_size(batch_size: Any) -> None:
+    if isinstance(batch_size, bool) or not isinstance(batch_size, int) or not 1 <= batch_size <= 10000:
+        raise ValueError("batch_size must be an integer between 1 and 10000")
+
+
+def ensure_table(catalog: Any, identifier: str | tuple[str, ...]) -> Any:
+    """Create the envelope table when it is missing and return it.
+
+    The namespace must already exist; namespaces are administered by you.
+    """
+    parts = _identifier(identifier)
+    try:
+        return catalog.create_table_if_not_exists(parts, schema=iceberg_schema())
+    except Exception:
+        raise LoadError("Iceberg table setup failed; check the namespace and catalog access.") from None
+
+
+def load(batch: Any, catalog: Any, identifier: str | tuple[str, ...], *,
+         batch_size: int = 1000, create_table: bool = True) -> LoadResult:
+    """Append bounded Arrow blocks through ``Table.append``.
+
+    Appends are at-least-once: a repeated load appends the same envelopes
+    again. Deduplicate downstream on ``operation``, ``record_id`` and
+    ``content_hash``. A failed block is never retried automatically.
+    """
+    parts = _identifier(identifier)
+    _check_batch_size(batch_size)
+    table_data = to_arrow(batch)
+    if create_table:
+        table = ensure_table(catalog, parts)
+    else:
+        try:
+            table = catalog.load_table(parts)
+        except Exception:
+            raise LoadError("Iceberg table lookup failed; create the table or enable create_table.") from None
+    count = 0
+    for start in range(0, table_data.num_rows, batch_size):
+        block = table_data.slice(start, batch_size)
+        try:
+            table.append(block)
+        except Exception:
+            raise LoadError(
+                f"Iceberg append block {count + 1} failed; earlier blocks may be committed. "
+                "Inspect the table snapshots before retrying."
+            ) from None
+        count += 1
+    return LoadResult(".".join(parts), table_data.num_rows, count)
+
+
+def load_operation(operation: str, args: dict[str, Any] | None, catalog: Any,
+                   identifier: str | tuple[str, ...], *, api_key_env: str | None = None,
+                   complete_history: bool = True, max_pages: int = 10000,
+                   client_factory: Any = None, batch_size: int = 1000,
+                   create_table: bool = True) -> LoadResult:
+    """Read one public operation and append its complete response envelope."""
+    _identifier(identifier)
+    _check_batch_size(batch_size)
+    batch = read(operation, args, api_key_env=api_key_env, complete_history=complete_history,
+                 max_pages=max_pages, client_factory=client_factory)
+    return load(batch, catalog, identifier, batch_size=batch_size, create_table=create_table)
+
+
+__all__ = ["COLUMNS", "LoadError", "LoadResult", "arrow_schema", "ensure_table",
+           "iceberg_schema", "load", "load_operation", "to_arrow"]

pyiceberg_fxmacrodata-0.1.0.dist-info/METADATA

@@ -0,0 +1,104 @@
+Metadata-Version: 2.4
+Name: pyiceberg-fxmacrodata
+Version: 0.1.0
+Summary: Load public FXMacroData responses into Apache Iceberg tables with PyIceberg
+License-Expression: MIT
+Project-URL: Documentation, https://fxmacrodata.com/documentation/reference?utm_source=pypi&utm_medium=referral&utm_campaign=pyiceberg-fxmacrodata&utm_content=package
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: fxmacrodata-platform-core==0.1.0
+Requires-Dist: pyiceberg<1,>=0.12.0
+Requires-Dist: pyarrow<27,>=25
+Provides-Extra: test
+Requires-Dist: pytest<9,>=8; extra == "test"
+Dynamic: license-file
+
+# FXMacroData for Apache Iceberg
+
+Load FXMacroData public responses into Apache Iceberg tables with PyIceberg.
+Keep macro announcements, calendars, FX and positioning data in the same
+catalog as your own tables while retaining the source payload and revision
+metadata.
+
+[FXMacroData documentation](https://fxmacrodata.com/documentation/reference?utm_source=github&utm_medium=referral&utm_campaign=pyiceberg-fxmacrodata&utm_content=readme)
+
+## Install
+
+Requires Python 3.10 or later and PyIceberg 0.12 or later. Install this
+package and its adjacent public client dependencies from the distribution
+supplied to you, together with the PyIceberg extras for your catalog and
+object store (for example `pyiceberg[sql-sqlite]`, `pyiceberg[glue]` or
+`pyiceberg[s3fs]`):
+
+```sh
+python -m pip install /path/to/fxmacrodata-public-client /path/to/fxmacrodata-platform-core /path/to/pyiceberg-fxmacrodata
+```
+
+## Example
+
+USD data is available without a key:
+
+```python
+from pyiceberg.catalog import load_catalog
+from fxmacrodata_platform_core import read
+from fxmacrodata_iceberg import load
+
+catalog = load_catalog("default")  # configured in ~/.pyiceberg.yaml
+batch = read("indicator_history", {"currency": "usd", "indicator": "policy_rate"})
+result = load(batch, catalog, "analytics.fxmacrodata", batch_size=1000)
+print(result.rows, "envelopes appended in", result.batches, "blocks")
+```
+
+The namespace (`analytics`) must already exist. By default the loader creates
+the table if absent through `catalog.create_table_if_not_exists`; set
+`create_table=False` to use a table provisioned by your administrator, and
+`iceberg_schema()` returns the schema it expects. Identifiers are dotted
+simple names with at least a namespace and a table (`analytics.fxmacrodata`),
+or a tuple of parts. Quoted names, SQL fragments and URL-like names are
+rejected before the catalog is contacted.
+
+## Credentials
+
+Set `FXMD_API_KEY` in your environment or secret manager and pass its name
+to `read` (or to `load_operation`) through `api_key_env`. Omit it for
+anonymous access. Catalog and object-store credentials belong to the
+PyIceberg catalog object you construct and pass in; this package never builds
+connection settings or reads destination secrets.
+
+## Data semantics
+
+Columns are `operation`, `source_url`, `record_id`, `currency`, `indicator`,
+`date`, `announcement_datetime`, `value`, `value_text`, `record_json`,
+`response_json` and the loader's `content_hash`. Only `value` is numeric
+(optional `double`); every other column is an optional `string`. The schema
+declares no identifier fields. `record_json` retains the original record and
+`response_json` retains the complete response, including pagination and
+metadata. An empty response produces one envelope row with
+`record_json = 'null'`, so absence of data and the server's metadata remain
+inspectable; it is not an observation. Dates and announcement times remain
+source strings. No publication time is inferred and no missing value is
+filled. `content_hash` is the SHA-256 of the canonical JSON of the ten
+source columns.
+
+## Delivery guarantee
+
+`load` appends bounded blocks of `batch_size` rows (1 to 10,000) through
+`Table.append`, each block as its own snapshot. Delivery is at-least-once:
+repeating a load appends the same envelopes again, and a failed block is not
+retried automatically; earlier blocks may already be committed. Deduplicate
+downstream with a `MERGE INTO` keyed on `operation`, `record_id` and
+`content_hash` in your query engine (Spark, Trino, Dremio, Flink) when you
+need exactly-once storage semantics. Changed record data or response
+metadata produces a distinct `content_hash` with the same `record_id`.
+
+`load_operation` reads one operation and loads it in one call. It accepts
+`api_key_env`, `complete_history`, `max_pages` and `client_factory` for the
+public reader; `max_pages` bounds history pagination, and an incomplete read
+raises before the destination is touched.
+
+## Development
+
+Run the offline tests with `python -m pytest tests -q`. They use an in-test
+fake catalog that records appended Arrow tables and never call the live
+service or a real catalog.

pyiceberg_fxmacrodata-0.1.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+fxmacrodata_iceberg/__init__.py,sha256=7mWodYRSkm5b0k0rqI6Yg4wZor3NDFwY_R1U-wRDWxw,6096
+fxmacrodata_iceberg/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pyiceberg_fxmacrodata-0.1.0.dist-info/licenses/LICENSE,sha256=rklK7Fl8aL1eWGubsrozZ7rCNTsN-MATw4M6LQIMv10,1092
+pyiceberg_fxmacrodata-0.1.0.dist-info/METADATA,sha256=0Gq-3nqkBd6dngML4CEXxy68FuIgfTz4IlzB0kSQgBs,4869
+pyiceberg_fxmacrodata-0.1.0.dist-info/WHEEL,sha256=16fc_9Rp854SLcdhaDEjr0u65p0e3FhZxihNgcnCE98,91
+pyiceberg_fxmacrodata-0.1.0.dist-info/top_level.txt,sha256=FS7XWqjtRav3mi-EiXkm0-lMh8yyDBuw3wQjGhhxPOk,20
+pyiceberg_fxmacrodata-0.1.0.dist-info/RECORD,,

pyiceberg_fxmacrodata-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.1.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

pyiceberg_fxmacrodata-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Robert Tidball
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pyiceberg_fxmacrodata-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+fxmacrodata_iceberg