snowflake-fxmacrodata 0.1.0__py3-none-any.whl

fxmacrodata_snowflake/__init__.py

@@ -0,0 +1,201 @@
+"""Load public FXMacroData responses into a customer-owned Snowflake account."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import hashlib
+import json
+import math
+import re
+from typing import Any
+from uuid import uuid4
+
+from fxmacrodata_platform_core import read
+
+__version__ = "0.1.0"
+COLUMNS = (
+    "operation", "source_url", "record_id", "currency", "indicator", "date",
+    "announcement_datetime", "value", "value_text", "record_json", "response_json",
+    "content_hash",
+)
+_IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,254}\Z")
+
+
+class LoadError(RuntimeError):
+    """A load failed; transaction outcome must be checked after connection loss."""
+
+
+@dataclass(frozen=True)
+class LoadResult:
+    table: str
+    rows_submitted: int
+    batches: int
+
+
+def _table(table: str) -> str:
+    if not isinstance(table, str):
+        raise ValueError("table must contain simple Snowflake identifiers")
+    parts = table.split(".")
+    if not 1 <= len(parts) <= 3 or any(not _IDENTIFIER.fullmatch(p) for p in parts):
+        raise ValueError("table must contain one to three simple SQL identifiers")
+    return ".".join(f'"{p.upper()}"' for p in parts)
+
+
+def _rows(batch: Any) -> list[dict[str, Any]]:
+    rows = []
+    seen = set()
+    for source in batch.rows():
+        row = {name: source.get(name) for name in COLUMNS[:-1]}
+        if not isinstance(row["operation"], str) or not isinstance(row["record_id"], str):
+            raise ValueError("every row requires string operation and record_id")
+        for name, value in row.items():
+            if name == "value":
+                if value is not None and (
+                    isinstance(value, bool) or not isinstance(value, (int, float)) or
+                    not math.isfinite(value)
+                ):
+                    raise ValueError("value must be a finite number or null")
+            elif value is not None and not isinstance(value, str):
+                raise ValueError(f"{name} must be a string or null")
+        encoded = json.dumps(row, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+        row["content_hash"] = hashlib.sha256(encoded.encode("utf-8")).hexdigest()
+        key = (row["operation"], row["record_id"], row["content_hash"])
+        if key not in seen:
+            seen.add(key)
+            rows.append(row)
+    return rows
+
+
+def create_table_sql(table: str) -> str:
+    """Return DDL preserving source dates, publication status and JSON as text."""
+    fields = ",\n    ".join(
+        f'"{name.upper()}" {"DOUBLE" if name == "value" else "VARCHAR"}' +
+        (" NOT NULL" if name in {"operation", "record_id", "content_hash"} else "")
+        for name in COLUMNS
+    )
+    return f"CREATE TABLE IF NOT EXISTS {_table(table)} (\n    {fields}\n)"
+
+
+def _blocks(rows: list[dict[str, Any]], batch_size: int, max_batch_bytes: int) -> list[list[dict[str, Any]]]:
+    if isinstance(batch_size, bool) or not isinstance(batch_size, int) or not 1 <= batch_size <= 10000:
+        raise ValueError("batch_size must be an integer between 1 and 10000")
+    if isinstance(max_batch_bytes, bool) or not isinstance(max_batch_bytes, int) or not 1 <= max_batch_bytes <= 128 * 1024 * 1024:
+        raise ValueError("max_batch_bytes must be an integer between 1 and 134217728")
+    output, block, size = [], [], 0
+    for row in rows:
+        length = len(json.dumps(row, ensure_ascii=True).encode("utf-8")) + 2
+        if length > max_batch_bytes:
+            raise ValueError("one envelope exceeds max_batch_bytes; narrow the source date range")
+        if block and (len(block) >= batch_size or size + length > max_batch_bytes):
+            output.append(block)
+            block, size = [], 0
+        block.append(row)
+        size += length
+    if block:
+        output.append(block)
+    return output
+
+
+def load(batch: Any, connection: Any, table: str, *, batch_size: int = 1000,
+         max_batch_bytes: int = 16 * 1024 * 1024,
+         create_table: bool = True, paramstyle: str = "pyformat") -> LoadResult:
+    """Stage bounded batches before merging envelopes in one DML transaction.
+
+    Use a dedicated connection without an active transaction. Table and temporary
+    stage DDL and all batch binding execute before BEGIN: the native connector
+    can create a binding stage during executemany. Repeated source envelopes
+    match existing rows; changed response metadata or revisions remain distinct.
+    """
+    quoted = _table(table)
+    if isinstance(batch_size, bool) or not isinstance(batch_size, int) or not 1 <= batch_size <= 10000:
+        raise ValueError("batch_size must be an integer between 1 and 10000")
+    if paramstyle not in {"pyformat", "format", "qmark", "numeric"}:
+        raise ValueError("paramstyle must match the Snowflake connection")
+    rows = _rows(batch)
+    blocks = _blocks(rows, batch_size, max_batch_bytes)
+    stage = f'"FXMD_STAGE_{uuid4().hex.upper()}"'
+    fields = ", ".join(f'"{name.upper()}"' for name in COLUMNS)
+    placeholder = ", ".join(
+        "?" if paramstyle == "qmark" else f":{i + 1}" if paramstyle == "numeric" else "%s"
+        for i in range(len(COLUMNS))
+    )
+    cursor = None
+    transaction = False
+    stage_created = False
+    committed = False
+    batches = 0
+    try:
+        cursor = connection.cursor()
+        if create_table:
+            cursor.execute(create_table_sql(table))
+        cursor.execute(f"CREATE TEMPORARY TABLE {stage} LIKE {quoted}")
+        stage_created = True
+        for block in blocks:
+            cursor.executemany(
+                f"INSERT INTO {stage} ({fields}) VALUES ({placeholder})",
+                [tuple(row[name] for name in COLUMNS) for row in block],
+            )
+            batches += 1
+        # qmark/numeric binding can issue CREATE TEMPORARY STAGE internally.
+        # Finish every staging insert before the destination DML transaction.
+        cursor.execute("BEGIN")
+        transaction = True
+        cursor.execute(
+            f"MERGE INTO {quoted} AS target USING {stage} AS source ON "
+            'target."OPERATION" = source."OPERATION" AND '
+            'target."RECORD_ID" = source."RECORD_ID" AND '
+            'target."CONTENT_HASH" = source."CONTENT_HASH" '
+            f"WHEN NOT MATCHED THEN INSERT ({fields}) VALUES (" +
+            ", ".join(f'source."{name.upper()}"' for name in COLUMNS) + ")"
+        )
+        cursor.execute("COMMIT")
+        transaction = False
+        committed = True
+    except Exception:
+        if transaction and cursor is not None:
+            try:
+                cursor.execute("ROLLBACK")
+                transaction = False
+            except Exception:
+                pass
+        raise LoadError(
+            "Snowflake load did not complete successfully. Check transaction outcome "
+            "before retrying; use a single writer per destination table."
+        ) from None
+    finally:
+        # DDL commits transactions, so never run cleanup after a failed rollback.
+        if cursor is not None:
+            if stage_created and not transaction:
+                try:
+                    cursor.execute(f"DROP TABLE IF EXISTS {stage}")
+                except Exception:
+                    pass
+            try:
+                cursor.close()
+            except Exception:
+                pass
+    if not committed:
+        raise LoadError("Snowflake transaction was not committed")
+    return LoadResult(quoted, len(rows), batches)
+
+
+def load_operation(operation: str, args: dict[str, Any] | None, connection: Any, table: str,
+                   *, api_key_env: str | None = None, complete_history: bool = True,
+                   max_pages: int = 10000, client_factory: Any = None,
+                   batch_size: int = 1000, max_batch_bytes: int = 16 * 1024 * 1024,
+                   create_table: bool = True,
+                   paramstyle: str = "pyformat") -> LoadResult:
+    """Read one public operation and load its complete response envelope."""
+    _table(table)
+    _blocks([], batch_size, max_batch_bytes)
+    if paramstyle not in {"pyformat", "format", "qmark", "numeric"}:
+        raise ValueError("paramstyle must match the Snowflake connection")
+    batch = read(operation, args, api_key_env=api_key_env,
+                 complete_history=complete_history, max_pages=max_pages,
+                 client_factory=client_factory)
+    return load(batch, connection, table, batch_size=batch_size,
+                max_batch_bytes=max_batch_bytes, create_table=create_table,
+                paramstyle=paramstyle)
+
+
+__all__ = ["COLUMNS", "LoadError", "LoadResult", "create_table_sql", "load", "load_operation"]

snowflake_fxmacrodata-0.1.0.dist-info/METADATA

@@ -0,0 +1,155 @@
+Metadata-Version: 2.4
+Name: snowflake-fxmacrodata
+Version: 0.1.0
+Summary: Load public FXMacroData responses into Snowflake
+License-Expression: MIT
+Project-URL: Documentation, https://fxmacrodata.com/documentation/reference?utm_source=pypi&utm_medium=referral&utm_campaign=snowflake-fxmacrodata&utm_content=package
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: fxmacrodata-platform-core==0.1.0
+Requires-Dist: snowflake-connector-python<5,>=4.8.1
+Requires-Dist: filelock<5,>=4.2.9
+Provides-Extra: test
+Requires-Dist: pytest<9,>=8; extra == "test"
+Dynamic: license-file
+
+# FXMacroData for Snowflake
+
+Load FXMacroData public responses through the official Snowflake Connector for
+Python. Bounded batch inserts feed a temporary staging table, followed by a
+transactional merge into your analytical table.
+
+[FXMacroData documentation](https://fxmacrodata.com/documentation/reference?utm_source=github&utm_medium=referral&utm_campaign=snowflake-fxmacrodata&utm_content=readme)
+
+## Install and connect
+
+Requires Python 3.10 or later and Snowflake Connector for Python 4.8.1 or later
+within version 4. Install the supplied adapter wheel; its public client and
+platform core dependencies resolve from PyPI:
+
+```sh
+python -m pip install /path/to/snowflake_fxmacrodata-0.1.0-py3-none-any.whl
+```
+
+Use a dedicated Snowflake connection with its current database and schema set.
+This example uses interactive SSO; headless workloads should use your
+organization's approved SDK authentication method and runtime secret store.
+Expose the FXMacroData source key through the `FXMD_API_KEY` environment name.
+
+```python
+import os
+import snowflake.connector
+from fxmacrodata_platform_core import read
+from fxmacrodata_snowflake import load
+
+connection = snowflake.connector.connect(
+    account=os.environ["SNOWFLAKE_ACCOUNT"],
+    user=os.environ["SNOWFLAKE_USER"],
+    authenticator="externalbrowser",
+    warehouse=os.environ["SNOWFLAKE_WAREHOUSE"],
+    database=os.environ["SNOWFLAKE_DATABASE"],
+    schema=os.environ["SNOWFLAKE_SCHEMA"],
+    paramstyle="qmark",
+)
+try:
+    batch = read("release_calendar", {"currency": "usd"}, api_key_env="FXMD_API_KEY")
+    result = load(batch, connection, "FXMACRODATA", paramstyle="qmark")
+finally:
+    connection.close()
+```
+
+One-, two- and three-part simple SQL names are accepted and normalized to
+uppercase, matching Snowflake's unquoted identifier convention. Quoted,
+case-sensitive names and SQL fragments are rejected. The database and schema
+must exist. The caller needs table/stage creation and DML privileges. Use
+`create_table=False` if an administrator has provisioned the envelope table with
+`create_table_sql(table)`.
+
+The loader must own the connection's transaction. Snowflake DDL can commit
+an active transaction, so do not pass a connection with unrelated pending work.
+`paramstyle` must match the connection (`qmark`, `numeric`, `format`, or
+`pyformat`, the connector default). No global connector settings are changed.
+
+Every operation exposed by `fxmacrodata_platform_core.operations()` is available,
+including REST and MCP operations. The loader adds a tabular envelope without
+changing the original public response. `record_json` retains the original record;
+`response_json` retains the complete response, including pagination and metadata.
+An empty response produces one envelope with `record_json = 'null'`, so absence of
+data and the server's metadata remain inspectable. It is not an observation.
+
+Columns are `operation`, `source_url`, `record_id`, `currency`, `indicator`,
+`date`, `announcement_datetime`, `value`, `value_text`, `record_json`,
+`response_json`, and the loader's `content_hash`. Only `value` is numeric.
+Dates and announcement times remain source strings. No publication time is
+inferred, and no missing value is filled. `value_text` and JSON preserve source
+numbers supplied as strings. JSON numeric tokens follow the public Python client
+parser, so original lexical precision is not guaranteed.
+
+Select operations with the shared catalogue rather than constructing URLs:
+
+```python
+from fxmacrodata_platform_core import operations, schemas
+
+available = operations()
+input_schemas = schemas()
+```
+
+`load_operation` accepts `api_key_env`, `complete_history`, `max_pages`, and
+`client_factory` for the public source reader. `max_pages` bounds history
+pagination; an incomplete read raises an error before a destination load starts.
+`batch_size` bounds destination blocks to 1–10,000 rows. `max_batch_bytes` also
+bounds each block using a conservative JSON byte estimate (16 MiB by default,
+128 MiB maximum). An oversized individual envelope is rejected before destination
+changes; narrow the source date range to reduce the response size. The source response is
+materialized before loading; choose a source date range suitable for memory.
+Read the installed operation's schema for required arguments and date filters.
+
+Supply source credentials through your runtime secret store, exposing only the
+environment variable name to the adapter. Destination authentication belongs to
+the official SDK connection you pass to the loader. The adapter neither saves nor
+prints credentials and never constructs destination credentials for you.
+
+The package licence covers this adapter code. FXMacroData data access and onward
+redistribution remain subject to your subscription and applicable licence. Loading
+data into a warehouse does not grant permission to publish or resell it.
+
+## Retries and querying
+
+The loader creates the destination and temporary table, then fills every bounded
+block before beginning the destination DML transaction. This also keeps any
+temporary binding-stage DDL issued by the connector outside the transaction.
+It then begins, merges on operation, record ID and content hash, and commits.
+A staging failure leaves destination rows untouched. A merge or commit failure
+triggers rollback. If rollback itself fails, the loader avoids cleanup DDL that
+could commit unresolved work.
+
+Repeated exact envelopes are skipped; changed source records or metadata remain
+distinct. Use one writer per destination table. Concurrent merges into standard
+Snowflake tables are not an exactly-once guarantee. The loader never retries a
+write automatically. After connection loss, inspect the transaction outcome
+before retrying. `LoadResult.rows_submitted` counts unique envelopes offered to
+the merge, rather than rows newly inserted.
+
+```sql
+SELECT CURRENCY, INDICATOR, DATE, VALUE,
+       PARSE_JSON(RECORD_JSON):publication_time_status::STRING AS PUBLICATION_STATUS
+FROM FXMACRODATA
+WHERE RECORD_JSON != 'null';
+```
+
+[event_study.sql](examples/event_study.sql) joins source announcement times to
+your licensed market-price table. Inspect source publication metadata separately
+for point-in-time backtesting. [sharing.sql](examples/sharing.sql) shows a
+customer-controlled secure share; sharing data requires the applicable rights
+and recipient approval. This package does not create a Marketplace listing.
+
+## Test
+
+```sh
+python -m pytest tests -q
+```
+
+Official SDK references:
+[batch binding and transactions](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-example),
+[connector API](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-api).

snowflake_fxmacrodata-0.1.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+fxmacrodata_snowflake/__init__.py,sha256=NfLEILOeoai0-XR5jybZYm--RsJZnGNZB9XXViKaBYg,8620
+snowflake_fxmacrodata-0.1.0.dist-info/licenses/LICENSE,sha256=mLVV_vNUMbtObjBLP4s10Tc8K4m_5sbNJ96E169Td84,1089
+snowflake_fxmacrodata-0.1.0.dist-info/METADATA,sha256=DkfAzrnGr9Rskwg2DTPNvN4KZEmg-8k0ATrYtf9N3go,7485
+snowflake_fxmacrodata-0.1.0.dist-info/WHEEL,sha256=16fc_9Rp854SLcdhaDEjr0u65p0e3FhZxihNgcnCE98,91
+snowflake_fxmacrodata-0.1.0.dist-info/top_level.txt,sha256=a_RsOylRuAobRH2h1qTUJK9lGB9HEkHS1BxzVUvARj0,22
+snowflake_fxmacrodata-0.1.0.dist-info/RECORD,,

snowflake_fxmacrodata-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.1.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

snowflake_fxmacrodata-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 FXMacroData
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

snowflake_fxmacrodata-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+fxmacrodata_snowflake