pyiceberg-fxmacrodata 0.1.0__tar.gz

pyiceberg_fxmacrodata-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Robert Tidball
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pyiceberg_fxmacrodata-0.1.0/MANIFEST.in

@@ -0,0 +1,10 @@
+include LICENSE
+include README.md
+include pyproject.toml
+include PUBLIC_FILES.json
+include MANIFEST.in
+include src/fxmacrodata_iceberg/__init__.py
+include src/fxmacrodata_iceberg/py.typed
+include tests/test_loader.py
+global-exclude *.pyc *.pyo
+global-exclude __pycache__/*

pyiceberg_fxmacrodata-0.1.0/PKG-INFO

@@ -0,0 +1,104 @@
+Metadata-Version: 2.4
+Name: pyiceberg-fxmacrodata
+Version: 0.1.0
+Summary: Load public FXMacroData responses into Apache Iceberg tables with PyIceberg
+License-Expression: MIT
+Project-URL: Documentation, https://fxmacrodata.com/documentation/reference?utm_source=pypi&utm_medium=referral&utm_campaign=pyiceberg-fxmacrodata&utm_content=package
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: fxmacrodata-platform-core==0.1.0
+Requires-Dist: pyiceberg<1,>=0.12.0
+Requires-Dist: pyarrow<27,>=25
+Provides-Extra: test
+Requires-Dist: pytest<9,>=8; extra == "test"
+Dynamic: license-file
+
+# FXMacroData for Apache Iceberg
+
+Load FXMacroData public responses into Apache Iceberg tables with PyIceberg.
+Keep macro announcements, calendars, FX and positioning data in the same
+catalog as your own tables while retaining the source payload and revision
+metadata.
+
+[FXMacroData documentation](https://fxmacrodata.com/documentation/reference?utm_source=github&utm_medium=referral&utm_campaign=pyiceberg-fxmacrodata&utm_content=readme)
+
+## Install
+
+Requires Python 3.10 or later and PyIceberg 0.12 or later. Install this
+package and its adjacent public client dependencies from the distribution
+supplied to you, together with the PyIceberg extras for your catalog and
+object store (for example `pyiceberg[sql-sqlite]`, `pyiceberg[glue]` or
+`pyiceberg[s3fs]`):
+
+```sh
+python -m pip install /path/to/fxmacrodata-public-client /path/to/fxmacrodata-platform-core /path/to/pyiceberg-fxmacrodata
+```
+
+## Example
+
+USD data is available without a key:
+
+```python
+from pyiceberg.catalog import load_catalog
+from fxmacrodata_platform_core import read
+from fxmacrodata_iceberg import load
+
+catalog = load_catalog("default")  # configured in ~/.pyiceberg.yaml
+batch = read("indicator_history", {"currency": "usd", "indicator": "policy_rate"})
+result = load(batch, catalog, "analytics.fxmacrodata", batch_size=1000)
+print(result.rows, "envelopes appended in", result.batches, "blocks")
+```
+
+The namespace (`analytics`) must already exist. By default the loader creates
+the table if absent through `catalog.create_table_if_not_exists`; set
+`create_table=False` to use a table provisioned by your administrator, and
+`iceberg_schema()` returns the schema it expects. Identifiers are dotted
+simple names with at least a namespace and a table (`analytics.fxmacrodata`),
+or a tuple of parts. Quoted names, SQL fragments and URL-like names are
+rejected before the catalog is contacted.
+
+## Credentials
+
+Set `FXMD_API_KEY` in your environment or secret manager and pass its name
+to `read` (or to `load_operation`) through `api_key_env`. Omit it for
+anonymous access. Catalog and object-store credentials belong to the
+PyIceberg catalog object you construct and pass in; this package never builds
+connection settings or reads destination secrets.
+
+## Data semantics
+
+Columns are `operation`, `source_url`, `record_id`, `currency`, `indicator`,
+`date`, `announcement_datetime`, `value`, `value_text`, `record_json`,
+`response_json` and the loader's `content_hash`. Only `value` is numeric
+(optional `double`); every other column is an optional `string`. The schema
+declares no identifier fields. `record_json` retains the original record and
+`response_json` retains the complete response, including pagination and
+metadata. An empty response produces one envelope row with
+`record_json = 'null'`, so absence of data and the server's metadata remain
+inspectable; it is not an observation. Dates and announcement times remain
+source strings. No publication time is inferred and no missing value is
+filled. `content_hash` is the SHA-256 of the canonical JSON of the ten
+source columns.
+
+## Delivery guarantee
+
+`load` appends bounded blocks of `batch_size` rows (1 to 10,000) through
+`Table.append`, each block as its own snapshot. Delivery is at-least-once:
+repeating a load appends the same envelopes again, and a failed block is not
+retried automatically; earlier blocks may already be committed. Deduplicate
+downstream with a `MERGE INTO` keyed on `operation`, `record_id` and
+`content_hash` in your query engine (Spark, Trino, Dremio, Flink) when you
+need exactly-once storage semantics. Changed record data or response
+metadata produces a distinct `content_hash` with the same `record_id`.
+
+`load_operation` reads one operation and loads it in one call. It accepts
+`api_key_env`, `complete_history`, `max_pages` and `client_factory` for the
+public reader; `max_pages` bounds history pagination, and an incomplete read
+raises before the destination is touched.
+
+## Development
+
+Run the offline tests with `python -m pytest tests -q`. They use an in-test
+fake catalog that records appended Arrow tables and never call the live
+service or a real catalog.

pyiceberg_fxmacrodata-0.1.0/PUBLIC_FILES.json

@@ -0,0 +1,10 @@
+[
+  "LICENSE",
+  "README.md",
+  "pyproject.toml",
+  "PUBLIC_FILES.json",
+  "MANIFEST.in",
+  "src/fxmacrodata_iceberg/__init__.py",
+  "src/fxmacrodata_iceberg/py.typed",
+  "tests/test_loader.py"
+]

pyiceberg_fxmacrodata-0.1.0/README.md

@@ -0,0 +1,88 @@
+# FXMacroData for Apache Iceberg
+
+Load FXMacroData public responses into Apache Iceberg tables with PyIceberg.
+Keep macro announcements, calendars, FX and positioning data in the same
+catalog as your own tables while retaining the source payload and revision
+metadata.
+
+[FXMacroData documentation](https://fxmacrodata.com/documentation/reference?utm_source=github&utm_medium=referral&utm_campaign=pyiceberg-fxmacrodata&utm_content=readme)
+
+## Install
+
+Requires Python 3.10 or later and PyIceberg 0.12 or later. Install this
+package and its adjacent public client dependencies from the distribution
+supplied to you, together with the PyIceberg extras for your catalog and
+object store (for example `pyiceberg[sql-sqlite]`, `pyiceberg[glue]` or
+`pyiceberg[s3fs]`):
+
+```sh
+python -m pip install /path/to/fxmacrodata-public-client /path/to/fxmacrodata-platform-core /path/to/pyiceberg-fxmacrodata
+```
+
+## Example
+
+USD data is available without a key:
+
+```python
+from pyiceberg.catalog import load_catalog
+from fxmacrodata_platform_core import read
+from fxmacrodata_iceberg import load
+
+catalog = load_catalog("default")  # configured in ~/.pyiceberg.yaml
+batch = read("indicator_history", {"currency": "usd", "indicator": "policy_rate"})
+result = load(batch, catalog, "analytics.fxmacrodata", batch_size=1000)
+print(result.rows, "envelopes appended in", result.batches, "blocks")
+```
+
+The namespace (`analytics`) must already exist. By default the loader creates
+the table if absent through `catalog.create_table_if_not_exists`; set
+`create_table=False` to use a table provisioned by your administrator, and
+`iceberg_schema()` returns the schema it expects. Identifiers are dotted
+simple names with at least a namespace and a table (`analytics.fxmacrodata`),
+or a tuple of parts. Quoted names, SQL fragments and URL-like names are
+rejected before the catalog is contacted.
+
+## Credentials
+
+Set `FXMD_API_KEY` in your environment or secret manager and pass its name
+to `read` (or to `load_operation`) through `api_key_env`. Omit it for
+anonymous access. Catalog and object-store credentials belong to the
+PyIceberg catalog object you construct and pass in; this package never builds
+connection settings or reads destination secrets.
+
+## Data semantics
+
+Columns are `operation`, `source_url`, `record_id`, `currency`, `indicator`,
+`date`, `announcement_datetime`, `value`, `value_text`, `record_json`,
+`response_json` and the loader's `content_hash`. Only `value` is numeric
+(optional `double`); every other column is an optional `string`. The schema
+declares no identifier fields. `record_json` retains the original record and
+`response_json` retains the complete response, including pagination and
+metadata. An empty response produces one envelope row with
+`record_json = 'null'`, so absence of data and the server's metadata remain
+inspectable; it is not an observation. Dates and announcement times remain
+source strings. No publication time is inferred and no missing value is
+filled. `content_hash` is the SHA-256 of the canonical JSON of the ten
+source columns.
+
+## Delivery guarantee
+
+`load` appends bounded blocks of `batch_size` rows (1 to 10,000) through
+`Table.append`, each block as its own snapshot. Delivery is at-least-once:
+repeating a load appends the same envelopes again, and a failed block is not
+retried automatically; earlier blocks may already be committed. Deduplicate
+downstream with a `MERGE INTO` keyed on `operation`, `record_id` and
+`content_hash` in your query engine (Spark, Trino, Dremio, Flink) when you
+need exactly-once storage semantics. Changed record data or response
+metadata produces a distinct `content_hash` with the same `record_id`.
+
+`load_operation` reads one operation and loads it in one call. It accepts
+`api_key_env`, `complete_history`, `max_pages` and `client_factory` for the
+public reader; `max_pages` bounds history pagination, and an incomplete read
+raises before the destination is touched.
+
+## Development
+
+Run the offline tests with `python -m pytest tests -q`. They use an in-test
+fake catalog that records appended Arrow tables and never call the live
+service or a real catalog.

pyiceberg_fxmacrodata-0.1.0/pyproject.toml

@@ -0,0 +1,30 @@
+[build-system]
+requires = ["setuptools>=77", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "pyiceberg-fxmacrodata"
+version = "0.1.0"
+description = "Load public FXMacroData responses into Apache Iceberg tables with PyIceberg"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+dependencies = ["fxmacrodata-platform-core==0.1.0", "pyiceberg>=0.12.0,<1", "pyarrow>=25,<27"]
+
+[project.urls]
+Documentation = "https://fxmacrodata.com/documentation/reference?utm_source=pypi&utm_medium=referral&utm_campaign=pyiceberg-fxmacrodata&utm_content=package"
+
+[project.optional-dependencies]
+test = ["pytest>=8,<9"]
+
+[tool.setuptools]
+packages = ["fxmacrodata_iceberg"]
+package-dir = {"" = "src"}
+include-package-data = false
+
+[tool.setuptools.package-data]
+fxmacrodata_iceberg = ["py.typed"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-q"

pyiceberg_fxmacrodata-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

pyiceberg_fxmacrodata-0.1.0/src/fxmacrodata_iceberg/__init__.py

@@ -0,0 +1,154 @@
+"""Load public FXMacroData responses into a customer-owned Apache Iceberg table."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import hashlib
+import json
+import math
+import re
+from typing import Any
+
+import pyarrow as pa
+from pyiceberg.schema import Schema
+from pyiceberg.types import DoubleType, NestedField, StringType
+
+from fxmacrodata_platform_core import COLUMNS as SOURCE_COLUMNS, read
+
+__version__ = "0.1.0"
+
+COLUMNS = tuple(SOURCE_COLUMNS) + ("content_hash",)
+_IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,127}\Z")
+
+
+class LoadError(RuntimeError):
+    """A catalog or table operation failed; inspect the destination before retrying."""
+
+
+@dataclass(frozen=True)
+class LoadResult:
+    identifier: str
+    rows: int
+    batches: int
+
+
+def arrow_schema() -> pa.Schema:
+    """Return the lossless Arrow schema: value is float64, everything else string."""
+    return pa.schema([
+        pa.field(name, pa.float64() if name == "value" else pa.string(), nullable=True)
+        for name in COLUMNS
+    ])
+
+
+def iceberg_schema() -> Schema:
+    """Return the Iceberg schema matching ``arrow_schema``; no identifier fields."""
+    return Schema(*(
+        NestedField(field_id=index, name=name,
+                    field_type=DoubleType() if name == "value" else StringType(),
+                    required=False)
+        for index, name in enumerate(COLUMNS, start=1)
+    ))
+
+
+def _rows(batch: Any) -> list[dict[str, Any]]:
+    rows = []
+    for source in batch.rows():
+        row = {name: source.get(name) for name in SOURCE_COLUMNS}
+        if not isinstance(row["operation"], str) or not isinstance(row["record_id"], str):
+            raise ValueError("every row requires string operation and record_id")
+        for name, value in row.items():
+            if name == "value":
+                if value is not None and (
+                    isinstance(value, bool) or not isinstance(value, (int, float)) or
+                    not math.isfinite(value)
+                ):
+                    raise ValueError("value must be a finite number or null")
+            elif value is not None and not isinstance(value, str):
+                raise ValueError(f"{name} must be a string or null")
+        encoded = json.dumps(row, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+        row["content_hash"] = hashlib.sha256(encoded.encode("utf-8")).hexdigest()
+        rows.append(row)
+    return rows
+
+
+def to_arrow(batch: Any) -> pa.Table:
+    """Project a Batch to an Arrow table with the eleven columns plus content_hash."""
+    return pa.Table.from_pylist(_rows(batch), schema=arrow_schema())
+
+
+def _identifier(identifier: Any) -> tuple[str, ...]:
+    if isinstance(identifier, (tuple, list)):
+        parts = tuple(identifier)
+    elif isinstance(identifier, str):
+        parts = tuple(identifier.split("."))
+    else:
+        raise ValueError("identifier must be a dotted namespace.table string or a tuple of parts")
+    if len(parts) < 2 or any(not isinstance(p, str) or not _IDENTIFIER.fullmatch(p) for p in parts):
+        raise ValueError("identifier must contain two or more simple dotted parts")
+    return parts
+
+
+def _check_batch_size(batch_size: Any) -> None:
+    if isinstance(batch_size, bool) or not isinstance(batch_size, int) or not 1 <= batch_size <= 10000:
+        raise ValueError("batch_size must be an integer between 1 and 10000")
+
+
+def ensure_table(catalog: Any, identifier: str | tuple[str, ...]) -> Any:
+    """Create the envelope table when it is missing and return it.
+
+    The namespace must already exist; namespaces are administered by you.
+    """
+    parts = _identifier(identifier)
+    try:
+        return catalog.create_table_if_not_exists(parts, schema=iceberg_schema())
+    except Exception:
+        raise LoadError("Iceberg table setup failed; check the namespace and catalog access.") from None
+
+
+def load(batch: Any, catalog: Any, identifier: str | tuple[str, ...], *,
+         batch_size: int = 1000, create_table: bool = True) -> LoadResult:
+    """Append bounded Arrow blocks through ``Table.append``.
+
+    Appends are at-least-once: a repeated load appends the same envelopes
+    again. Deduplicate downstream on ``operation``, ``record_id`` and
+    ``content_hash``. A failed block is never retried automatically.
+    """
+    parts = _identifier(identifier)
+    _check_batch_size(batch_size)
+    table_data = to_arrow(batch)
+    if create_table:
+        table = ensure_table(catalog, parts)
+    else:
+        try:
+            table = catalog.load_table(parts)
+        except Exception:
+            raise LoadError("Iceberg table lookup failed; create the table or enable create_table.") from None
+    count = 0
+    for start in range(0, table_data.num_rows, batch_size):
+        block = table_data.slice(start, batch_size)
+        try:
+            table.append(block)
+        except Exception:
+            raise LoadError(
+                f"Iceberg append block {count + 1} failed; earlier blocks may be committed. "
+                "Inspect the table snapshots before retrying."
+            ) from None
+        count += 1
+    return LoadResult(".".join(parts), table_data.num_rows, count)
+
+
+def load_operation(operation: str, args: dict[str, Any] | None, catalog: Any,
+                   identifier: str | tuple[str, ...], *, api_key_env: str | None = None,
+                   complete_history: bool = True, max_pages: int = 10000,
+                   client_factory: Any = None, batch_size: int = 1000,
+                   create_table: bool = True) -> LoadResult:
+    """Read one public operation and append its complete response envelope."""
+    _identifier(identifier)
+    _check_batch_size(batch_size)
+    batch = read(operation, args, api_key_env=api_key_env, complete_history=complete_history,
+                 max_pages=max_pages, client_factory=client_factory)
+    return load(batch, catalog, identifier, batch_size=batch_size, create_table=create_table)
+
+
+__all__ = ["COLUMNS", "LoadError", "LoadResult", "arrow_schema", "ensure_table",
+           "iceberg_schema", "load", "load_operation", "to_arrow"]

pyiceberg_fxmacrodata-0.1.0/src/pyiceberg_fxmacrodata.egg-info/PKG-INFO

@@ -0,0 +1,104 @@
+Metadata-Version: 2.4
+Name: pyiceberg-fxmacrodata
+Version: 0.1.0
+Summary: Load public FXMacroData responses into Apache Iceberg tables with PyIceberg
+License-Expression: MIT
+Project-URL: Documentation, https://fxmacrodata.com/documentation/reference?utm_source=pypi&utm_medium=referral&utm_campaign=pyiceberg-fxmacrodata&utm_content=package
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: fxmacrodata-platform-core==0.1.0
+Requires-Dist: pyiceberg<1,>=0.12.0
+Requires-Dist: pyarrow<27,>=25
+Provides-Extra: test
+Requires-Dist: pytest<9,>=8; extra == "test"
+Dynamic: license-file
+
+# FXMacroData for Apache Iceberg
+
+Load FXMacroData public responses into Apache Iceberg tables with PyIceberg.
+Keep macro announcements, calendars, FX and positioning data in the same
+catalog as your own tables while retaining the source payload and revision
+metadata.
+
+[FXMacroData documentation](https://fxmacrodata.com/documentation/reference?utm_source=github&utm_medium=referral&utm_campaign=pyiceberg-fxmacrodata&utm_content=readme)
+
+## Install
+
+Requires Python 3.10 or later and PyIceberg 0.12 or later. Install this
+package and its adjacent public client dependencies from the distribution
+supplied to you, together with the PyIceberg extras for your catalog and
+object store (for example `pyiceberg[sql-sqlite]`, `pyiceberg[glue]` or
+`pyiceberg[s3fs]`):
+
+```sh
+python -m pip install /path/to/fxmacrodata-public-client /path/to/fxmacrodata-platform-core /path/to/pyiceberg-fxmacrodata
+```
+
+## Example
+
+USD data is available without a key:
+
+```python
+from pyiceberg.catalog import load_catalog
+from fxmacrodata_platform_core import read
+from fxmacrodata_iceberg import load
+
+catalog = load_catalog("default")  # configured in ~/.pyiceberg.yaml
+batch = read("indicator_history", {"currency": "usd", "indicator": "policy_rate"})
+result = load(batch, catalog, "analytics.fxmacrodata", batch_size=1000)
+print(result.rows, "envelopes appended in", result.batches, "blocks")
+```
+
+The namespace (`analytics`) must already exist. By default the loader creates
+the table if absent through `catalog.create_table_if_not_exists`; set
+`create_table=False` to use a table provisioned by your administrator, and
+`iceberg_schema()` returns the schema it expects. Identifiers are dotted
+simple names with at least a namespace and a table (`analytics.fxmacrodata`),
+or a tuple of parts. Quoted names, SQL fragments and URL-like names are
+rejected before the catalog is contacted.
+
+## Credentials
+
+Set `FXMD_API_KEY` in your environment or secret manager and pass its name
+to `read` (or to `load_operation`) through `api_key_env`. Omit it for
+anonymous access. Catalog and object-store credentials belong to the
+PyIceberg catalog object you construct and pass in; this package never builds
+connection settings or reads destination secrets.
+
+## Data semantics
+
+Columns are `operation`, `source_url`, `record_id`, `currency`, `indicator`,
+`date`, `announcement_datetime`, `value`, `value_text`, `record_json`,
+`response_json` and the loader's `content_hash`. Only `value` is numeric
+(optional `double`); every other column is an optional `string`. The schema
+declares no identifier fields. `record_json` retains the original record and
+`response_json` retains the complete response, including pagination and
+metadata. An empty response produces one envelope row with
+`record_json = 'null'`, so absence of data and the server's metadata remain
+inspectable; it is not an observation. Dates and announcement times remain
+source strings. No publication time is inferred and no missing value is
+filled. `content_hash` is the SHA-256 of the canonical JSON of the ten
+source columns.
+
+## Delivery guarantee
+
+`load` appends bounded blocks of `batch_size` rows (1 to 10,000) through
+`Table.append`, each block as its own snapshot. Delivery is at-least-once:
+repeating a load appends the same envelopes again, and a failed block is not
+retried automatically; earlier blocks may already be committed. Deduplicate
+downstream with a `MERGE INTO` keyed on `operation`, `record_id` and
+`content_hash` in your query engine (Spark, Trino, Dremio, Flink) when you
+need exactly-once storage semantics. Changed record data or response
+metadata produces a distinct `content_hash` with the same `record_id`.
+
+`load_operation` reads one operation and loads it in one call. It accepts
+`api_key_env`, `complete_history`, `max_pages` and `client_factory` for the
+public reader; `max_pages` bounds history pagination, and an incomplete read
+raises before the destination is touched.
+
+## Development
+
+Run the offline tests with `python -m pytest tests -q`. They use an in-test
+fake catalog that records appended Arrow tables and never call the live
+service or a real catalog.

pyiceberg_fxmacrodata-0.1.0/src/pyiceberg_fxmacrodata.egg-info/SOURCES.txt

@@ -0,0 +1,13 @@
+LICENSE
+MANIFEST.in
+PUBLIC_FILES.json
+README.md
+pyproject.toml
+src/fxmacrodata_iceberg/__init__.py
+src/fxmacrodata_iceberg/py.typed
+src/pyiceberg_fxmacrodata.egg-info/PKG-INFO
+src/pyiceberg_fxmacrodata.egg-info/SOURCES.txt
+src/pyiceberg_fxmacrodata.egg-info/dependency_links.txt
+src/pyiceberg_fxmacrodata.egg-info/requires.txt
+src/pyiceberg_fxmacrodata.egg-info/top_level.txt
+tests/test_loader.py

pyiceberg_fxmacrodata-0.1.0/src/pyiceberg_fxmacrodata.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

pyiceberg_fxmacrodata-0.1.0/src/pyiceberg_fxmacrodata.egg-info/requires.txt

@@ -0,0 +1,6 @@
+fxmacrodata-platform-core==0.1.0
+pyiceberg<1,>=0.12.0
+pyarrow<27,>=25
+
+[test]
+pytest<9,>=8

pyiceberg_fxmacrodata-0.1.0/src/pyiceberg_fxmacrodata.egg-info/top_level.txt

@@ -0,0 +1 @@
+fxmacrodata_iceberg

pyiceberg_fxmacrodata-0.1.0/tests/test_loader.py

@@ -0,0 +1,175 @@
+import json
+
+import pyarrow as pa
+import pytest
+from pyiceberg.types import DoubleType, StringType
+
+from fxmacrodata_platform_core import Batch
+import fxmacrodata_iceberg as adapter
+
+
+def compatible(schema):
+    """Name/type view of an Arrow schema; PyIceberg promotes string to large_string."""
+    return [(f.name, pa.large_string() if f.type == pa.string() else f.type, f.nullable) for f in schema]
+
+
+class Table:
+    """Records appended Arrow tables the way a catalog-backed table would."""
+
+    def __init__(self, identifier, schema, fail_block=None):
+        self.identifier = identifier
+        self.schema = schema
+        self.appended = []
+        self.fail_block = fail_block
+
+    def append(self, table):
+        if not isinstance(table, pa.Table):
+            raise TypeError("append expects a pyarrow Table")
+        if compatible(table.schema) != compatible(self.schema.as_arrow()):
+            raise TypeError("schema mismatch")
+        self.appended.append(table)
+        if len(self.appended) == self.fail_block:
+            raise RuntimeError("private destination diagnostic")
+
+    def rows(self):
+        return [row for block in self.appended for row in block.to_pylist()]
+
+
+class Catalog:
+    def __init__(self, fail_block=None, fail_create=False):
+        self.tables = {}
+        self.calls = []
+        self.fail_block = fail_block
+        self.fail_create = fail_create
+
+    def create_table_if_not_exists(self, identifier, schema, **kwargs):
+        self.calls.append(("create_table_if_not_exists", tuple(identifier)))
+        if self.fail_create:
+            raise RuntimeError("private catalog diagnostic")
+        key = tuple(identifier)
+        if key not in self.tables:
+            self.tables[key] = Table(key, schema, self.fail_block)
+        return self.tables[key]
+
+    def load_table(self, identifier):
+        self.calls.append(("load_table", tuple(identifier)))
+        return self.tables[tuple(identifier)]
+
+
+def batch(count=3, metadata=None):
+    return Batch("indicator_history", {
+        "data": [{"currency": "USD", "indicator": "policy_rate", "value": str(i),
+                  "date": "2026-10-01", "publication_time_status": "unknown"}
+                 for i in range(count)],
+        "metadata": metadata or {"revision": 1},
+    })
+
+
+def test_iceberg_schema_matches_arrow_schema():
+    schema = adapter.iceberg_schema()
+    assert [f.name for f in schema.fields] == list(adapter.COLUMNS)
+    assert isinstance(schema.find_field("value").field_type, DoubleType)
+    assert all(isinstance(f.field_type, StringType) for f in schema.fields if f.name != "value")
+    assert all(not f.required for f in schema.fields)
+    assert schema.identifier_field_ids == []
+    assert compatible(schema.as_arrow()) == compatible(adapter.arrow_schema())
+    assert adapter.arrow_schema().field("value").type == pa.float64()
+
+
+def test_append_preserves_full_payload_and_source_strings():
+    source, catalog = batch(), Catalog()
+    result = adapter.load(source, catalog, "analytics.fxmacrodata", batch_size=2)
+    assert (result.rows, result.batches, result.identifier) == (3, 2, "analytics.fxmacrodata")
+    table = catalog.tables[("analytics", "fxmacrodata")]
+    assert [b.num_rows for b in table.appended] == [2, 1]
+    row = table.rows()[0]
+    assert row["date"] == "2026-10-01"
+    assert row["announcement_datetime"] is None
+    assert row["value"] == 0.0 and row["value_text"] == "0"
+    assert json.loads(row["record_json"])["publication_time_status"] == "unknown"
+    assert json.loads(row["response_json"]) == source.payload
+    assert len(row["content_hash"]) == 64
+
+
+def test_empty_response_is_an_envelope_not_an_observation():
+    source, catalog = batch(0), Catalog()
+    result = adapter.load(source, catalog, ("ns", "fxmacrodata"))
+    row = catalog.tables[("ns", "fxmacrodata")].rows()[0]
+    assert result.rows == 1 and row["record_json"] == "null"
+    assert row["value"] is None and row["date"] is None
+    assert json.loads(row["response_json"]) == source.payload
+
+
+def test_repeated_loads_append_again_with_stable_keys():
+    catalog = Catalog()
+    adapter.load(batch(1), catalog, "ns.t")
+    adapter.load(batch(1), catalog, "ns.t")
+    adapter.load(batch(1, {"revision": 2}), catalog, "ns.t")
+    rows = catalog.tables[("ns", "t")].rows()
+    assert len(rows) == 3
+    assert rows[0]["record_id"] == rows[1]["record_id"] == rows[2]["record_id"]
+    assert rows[0]["content_hash"] == rows[1]["content_hash"] != rows[2]["content_hash"]
+
+
+@pytest.mark.parametrize("identifier", ["table", "ns.table; DROP", "https://host/t", "a.`b`", "a b.c", "", "ns..t", 42, ("ns",)])
+def test_bad_identifiers_never_touch_catalog(identifier):
+    catalog = Catalog()
+    with pytest.raises(ValueError):
+        adapter.load(batch(), catalog, identifier)
+    with pytest.raises(ValueError):
+        adapter.ensure_table(catalog, identifier)
+    assert catalog.calls == []
+
+
+@pytest.mark.parametrize("size", [0, -1, 10001, True, 2.5])
+def test_batch_bounds(size):
+    catalog = Catalog()
+    with pytest.raises(ValueError):
+        adapter.load(batch(), catalog, "ns.t", batch_size=size)
+    assert catalog.calls == []
+
+
+def test_failed_block_is_not_retried_or_exposed():
+    catalog = Catalog(fail_block=2)
+    with pytest.raises(adapter.LoadError) as error:
+        adapter.load(batch(), catalog, "ns.t", batch_size=1)
+    assert "block 2" in str(error.value)
+    assert "private destination diagnostic" not in str(error.value)
+    assert len(catalog.tables[("ns", "t")].appended) == 2
+
+
+def test_table_setup_failure_is_not_exposed():
+    catalog = Catalog(fail_create=True)
+    with pytest.raises(adapter.LoadError) as error:
+        adapter.load(batch(), catalog, "ns.t")
+    assert "private catalog diagnostic" not in str(error.value)
+
+
+def test_preprovisioned_table_skips_creation():
+    catalog = Catalog()
+    catalog.tables[("ns", "t")] = Table(("ns", "t"), adapter.iceberg_schema())
+    adapter.load(batch(1), catalog, "ns.t", create_table=False)
+    assert catalog.calls == [("load_table", ("ns", "t"))]
+    assert len(catalog.tables[("ns", "t")].rows()) == 1
+
+
+def test_operation_forwards_public_reader_options(monkeypatch):
+    calls = []
+    monkeypatch.setattr(adapter, "read", lambda *args, **kw: calls.append((args, kw)) or batch(1))
+    factory = object()
+    adapter.load_operation("indicator_history", {"currency": "usd", "indicator": "policy_rate"},
+                           Catalog(), "ns.t", api_key_env="FXMD_API_KEY", max_pages=12,
+                           client_factory=factory)
+    assert calls[0][1] == dict(api_key_env="FXMD_API_KEY", complete_history=True,
+                                max_pages=12, client_factory=factory)
+
+
+def test_invalid_loader_configuration_never_reads_source(monkeypatch):
+    def unexpected(*args, **kwargs):
+        pytest.fail("source must not be called for invalid loader configuration")
+
+    monkeypatch.setattr(adapter, "read", unexpected)
+    with pytest.raises(ValueError):
+        adapter.load_operation("release_calendar", {"currency": "usd"}, Catalog(), "ns.t", batch_size=0)
+    with pytest.raises(ValueError):
+        adapter.load_operation("release_calendar", {"currency": "usd"}, Catalog(), "bad identifier")