questdb-fxmacrodata 0.1.0__py3-none-any.whl

fxmacrodata_questdb/__init__.py

@@ -0,0 +1,139 @@
+"""Load public FXMacroData responses into a customer-owned QuestDB table."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import hashlib
+import json
+import math
+import re
+from typing import Any
+
+try:  # questdb >= 3 exports the sentinel at top level; older paths keep the ingress module.
+    from questdb import ServerTimestamp
+except ImportError:  # pragma: no cover - exercised only on older client layouts
+    from questdb.ingress import ServerTimestamp
+
+from fxmacrodata_platform_core import read
+
+__version__ = "0.1.0"
+
+COLUMNS = (
+    "operation", "source_url", "record_id", "currency", "indicator", "date",
+    "announcement_datetime", "value", "value_text", "record_json", "response_json",
+    "content_hash",
+)
+SYMBOLS = ("operation", "currency", "indicator")
+_IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,127}\Z")
+
+
+class LoadError(RuntimeError):
+    """A destination operation failed; inspect the destination before retrying."""
+
+
+@dataclass(frozen=True)
+class LoadResult:
+    table: str
+    rows: int
+    batches: int
+
+
+def _table(table: str) -> str:
+    if not isinstance(table, str) or not _IDENTIFIER.fullmatch(table):
+        raise ValueError("table must be one simple identifier")
+    return table
+
+
+def _rows(batch: Any) -> list[dict[str, Any]]:
+    rows = []
+    for source in batch.rows():
+        row = {name: source.get(name) for name in COLUMNS[:-1]}
+        if not isinstance(row["operation"], str) or not isinstance(row["record_id"], str):
+            raise ValueError("every row requires string operation and record_id")
+        for name, value in row.items():
+            if name == "value":
+                if value is not None and (
+                    isinstance(value, bool) or not isinstance(value, (int, float)) or
+                    not math.isfinite(value)
+                ):
+                    raise ValueError("value must be a finite number or null")
+            elif value is not None and not isinstance(value, str):
+                raise ValueError(f"{name} must be a string or null")
+        encoded = json.dumps(row, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+        row["content_hash"] = hashlib.sha256(encoded.encode("utf-8")).hexdigest()
+        rows.append(row)
+    return rows
+
+
+def _blocks(rows: list[dict[str, Any]], batch_size: int, max_batch_bytes: int) -> list[list[dict[str, Any]]]:
+    if isinstance(batch_size, bool) or not isinstance(batch_size, int) or not 1 <= batch_size <= 10000:
+        raise ValueError("batch_size must be an integer between 1 and 10000")
+    if isinstance(max_batch_bytes, bool) or not isinstance(max_batch_bytes, int) or not 1 <= max_batch_bytes <= 128 * 1024 * 1024:
+        raise ValueError("max_batch_bytes must be an integer between 1 and 134217728")
+    output, block, size = [], [], 0
+    for row in rows:
+        length = len(json.dumps(row, ensure_ascii=True).encode("utf-8")) + 2
+        if length > max_batch_bytes:
+            raise ValueError("one envelope exceeds max_batch_bytes; narrow the source date range")
+        if block and (len(block) >= batch_size or size + length > max_batch_bytes):
+            output.append(block)
+            block, size = [], 0
+        block.append(row)
+        size += length
+    if block:
+        output.append(block)
+    return output
+
+
+def row_arguments(row: dict[str, Any]) -> tuple[dict[str, str], dict[str, Any]]:
+    """Split one envelope into ``Sender.row`` symbols and columns, omitting nulls."""
+    symbols = {name: row[name] for name in SYMBOLS if row.get(name) is not None}
+    columns = {name: row[name] for name in COLUMNS if name not in SYMBOLS and row.get(name) is not None}
+    if "value" in columns:
+        columns["value"] = float(columns["value"])
+    return symbols, columns
+
+
+def load(batch: Any, sender: Any, table: str, *, batch_size: int = 1000,
+         max_batch_bytes: int = 16 * 1024 * 1024) -> LoadResult:
+    """Send bounded blocks through an open questdb ``Sender`` and flush after each.
+
+    Rows are stamped with QuestDB's server timestamp at ingestion; the source
+    ``date`` and ``announcement_datetime`` remain string columns. A failed
+    block is never retried automatically.
+    """
+    table = _table(table)
+    if isinstance(batch_size, bool) or not isinstance(batch_size, int) or not 1 <= batch_size <= 10000:
+        raise ValueError("batch_size must be an integer between 1 and 10000")
+    rows = _rows(batch)
+    blocks = _blocks(rows, batch_size, max_batch_bytes)
+    count = 0
+    for block in blocks:
+        try:
+            for row in block:
+                symbols, columns = row_arguments(row)
+                sender.row(table, symbols=symbols, columns=columns, at=ServerTimestamp)
+            sender.flush()
+        except Exception:
+            raise LoadError(
+                f"QuestDB ingestion block {count + 1} failed; earlier blocks were flushed. "
+                "Inspect the destination and deduplicate on content_hash after a retry."
+            ) from None
+        count += 1
+    return LoadResult(table, len(rows), count)
+
+
+def load_operation(operation: str, args: dict[str, Any] | None, sender: Any, table: str,
+                   *, api_key_env: str | None = None, complete_history: bool = True,
+                   max_pages: int = 10000, client_factory: Any = None,
+                   batch_size: int = 1000, max_batch_bytes: int = 16 * 1024 * 1024) -> LoadResult:
+    """Read one public operation and load its complete response envelope."""
+    _table(table)
+    _blocks([], batch_size, max_batch_bytes)
+    batch = read(operation, args, api_key_env=api_key_env,
+                 complete_history=complete_history, max_pages=max_pages,
+                 client_factory=client_factory)
+    return load(batch, sender, table, batch_size=batch_size, max_batch_bytes=max_batch_bytes)
+
+
+__all__ = ["COLUMNS", "SYMBOLS", "LoadError", "LoadResult", "row_arguments", "load", "load_operation"]

questdb_fxmacrodata-0.1.0.dist-info/METADATA

@@ -0,0 +1,142 @@
+Metadata-Version: 2.4
+Name: questdb-fxmacrodata
+Version: 0.1.0
+Summary: Load public FXMacroData responses into QuestDB
+License-Expression: MIT
+Project-URL: Documentation, https://fxmacrodata.com/documentation/reference?utm_source=pypi&utm_medium=referral&utm_campaign=questdb-fxmacrodata&utm_content=package
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: fxmacrodata-platform-core==0.1.0
+Requires-Dist: questdb<6,>=5.0.0
+Provides-Extra: test
+Requires-Dist: pytest<9,>=8; extra == "test"
+Dynamic: license-file
+
+# FXMacroData for QuestDB
+
+Load FXMacroData public responses into QuestDB with the official `questdb`
+Python client's ingestion `Sender`. Keep macro announcements next to your
+time-series market data while retaining the source payload and revision
+metadata.
+
+[FXMacroData documentation](https://fxmacrodata.com/documentation/reference?utm_source=github&utm_medium=referral&utm_campaign=questdb-fxmacrodata&utm_content=readme)
+
+## Install and connect
+
+Requires Python 3.10 or later. Install this package and its adjacent public
+client dependencies from the distribution supplied to you:
+
+```sh
+python -m pip install /path/to/fxmacrodata-public-client /path/to/fxmacrodata-platform-core /path/to/questdb-fxmacrodata
+```
+
+Configure `FXMD_API_KEY` and `QDB_CLIENT_CONF` (the QuestDB client's own
+configuration string, including any token) in your runtime secret store. Then
+load into a customer-owned QuestDB instance:
+
+```python
+from questdb.ingress import Sender
+from fxmacrodata_platform_core import read
+from fxmacrodata_questdb import load
+
+with Sender.from_env() as sender:
+    batch = read("indicator_history", {"currency": "usd", "indicator": "policy_rate"})
+    result = load(batch, sender, "fxmacrodata", batch_size=1000)
+```
+
+USD operations work without a key. For other currencies pass
+`api_key_env="FXMD_API_KEY"` to `read` or `load_operation`.
+
+You open and close the `Sender`; the loader only calls `row` and `flush`.
+QuestDB creates the table on first ingestion. Table names are one simple
+identifier such as `fxmacrodata`; dotted names, SQL fragments and URL-like
+names are rejected.
+
+Every operation exposed by `fxmacrodata_platform_core.operations()` is available,
+including REST and MCP operations. The loader adds a tabular envelope without
+changing the original public response. `record_json` retains the original record;
+`response_json` retains the complete response, including pagination and metadata.
+An empty response produces one envelope with `record_json = 'null'`, so absence of
+data and the server's metadata remain inspectable. It is not an observation.
+
+Columns are `operation`, `source_url`, `record_id`, `currency`, `indicator`,
+`date`, `announcement_datetime`, `value`, `value_text`, `record_json`,
+`response_json`, and the loader's `content_hash`. `operation`, `currency` and
+`indicator` are sent as QuestDB symbols; only `value` is numeric (a double);
+everything else is a string. Null values are omitted from the row, which
+QuestDB stores as `NULL`.
+
+Every row is stamped with QuestDB's server timestamp at ingestion
+(`ServerTimestamp`). The designated timestamp therefore records when you
+loaded the data, never when a release was published. `date` and
+`announcement_datetime` remain source strings in their own columns; no
+publication time is inferred, and no missing value is filled. `value_text` and
+JSON preserve source numbers supplied as strings. JSON numeric tokens follow the
+public Python client parser, so original lexical precision is not guaranteed.
+
+Select operations with the shared catalogue rather than constructing URLs:
+
+```python
+from fxmacrodata_platform_core import operations, schemas
+
+available = operations()
+input_schemas = schemas()
+```
+
+`load_operation` accepts `api_key_env`, `complete_history`, `max_pages`, and
+`client_factory` for the public source reader. `max_pages` bounds history
+pagination; an incomplete read raises an error before a destination load starts.
+`batch_size` bounds destination blocks to 1–10,000 rows; the sender is flushed
+after each block. `max_batch_bytes` also bounds each block using a conservative
+JSON byte estimate (16 MiB by default, 128 MiB maximum). An oversized individual
+envelope is rejected before destination changes; narrow the source date range
+to reduce the response size. The source response is materialized before
+loading; choose a source date range suitable for memory. Read the installed
+operation's schema for required arguments and date filters.
+
+Supply source credentials through your runtime secret store, exposing only the
+environment variable name to the adapter. Destination authentication belongs to
+the `Sender` you construct. The adapter neither saves nor prints credentials
+and never constructs connection configuration for you.
+
+The package licence covers this adapter code. FXMacroData data access and onward
+redistribution remain subject to your subscription and applicable licence. Loading
+data into a database does not grant permission to publish or resell it.
+
+## Retries and querying
+
+Delivery is at-least-once. QuestDB's ingestion protocol has no unique key, and
+because rows carry a server timestamp, a retried batch appends rows again with
+a later timestamp. Every row carries `content_hash` (a digest of the ten source
+columns), so duplicates are exact and easy to collapse:
+
+```sql
+SELECT currency, indicator, date, value, record_json
+FROM fxmacrodata
+WHERE record_json <> 'null'
+LATEST ON timestamp PARTITION BY content_hash;
+```
+
+Changed record data or response metadata produces a distinct content hash and is
+retained alongside the earlier version. Blocks are not retried automatically.
+On a failure, earlier blocks were already flushed; retrying the same source
+batch is safe when you deduplicate on `content_hash` as above.
+`LoadResult.rows` counts envelopes submitted.
+
+See [event_study.sql](examples/event_study.sql) for an announcement/price join.
+The example expects your own licensed market-price table and uses source
+announcement timestamps only. Timestamp presence alone is not evidence that a
+release had already been published; use source publication metadata when
+building a point-in-time backtest.
+
+## Development
+
+Tests are offline and fake the sender:
+
+```sh
+python -m pytest tests -q
+```
+
+Official client reference:
+[QuestDB Python client](https://py-questdb-client.readthedocs.io/?utm_source=pypi&utm_medium=referral&utm_campaign=questdb-fxmacrodata&utm_content=upstream-reference).

questdb_fxmacrodata-0.1.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+fxmacrodata_questdb/__init__.py,sha256=2M0k7qjh2AsZ3BugrtB8hRXcEjz0nHvfPuOa6A7ls1s,5934
+fxmacrodata_questdb/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+questdb_fxmacrodata-0.1.0.dist-info/licenses/LICENSE,sha256=rklK7Fl8aL1eWGubsrozZ7rCNTsN-MATw4M6LQIMv10,1092
+questdb_fxmacrodata-0.1.0.dist-info/METADATA,sha256=wulqXlplww7IL7PJTvrDRqItglAi5mqOUs9YnjTVmuQ,6646
+questdb_fxmacrodata-0.1.0.dist-info/WHEEL,sha256=16fc_9Rp854SLcdhaDEjr0u65p0e3FhZxihNgcnCE98,91
+questdb_fxmacrodata-0.1.0.dist-info/top_level.txt,sha256=Q8oDNb5DYbVh36LSVbMAzbsjzpkQog7iOs3YA9b0SUs,20
+questdb_fxmacrodata-0.1.0.dist-info/RECORD,,

questdb_fxmacrodata-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.1.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

questdb_fxmacrodata-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Robert Tidball
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

questdb_fxmacrodata-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+fxmacrodata_questdb