snowflake-fxmacrodata 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- snowflake_fxmacrodata-0.1.0/LICENSE +21 -0
- snowflake_fxmacrodata-0.1.0/MANIFEST.in +11 -0
- snowflake_fxmacrodata-0.1.0/PKG-INFO +155 -0
- snowflake_fxmacrodata-0.1.0/PUBLIC_FILES.json +11 -0
- snowflake_fxmacrodata-0.1.0/README.md +139 -0
- snowflake_fxmacrodata-0.1.0/examples/event_study.sql +25 -0
- snowflake_fxmacrodata-0.1.0/examples/sharing.sql +15 -0
- snowflake_fxmacrodata-0.1.0/pyproject.toml +27 -0
- snowflake_fxmacrodata-0.1.0/setup.cfg +4 -0
- snowflake_fxmacrodata-0.1.0/src/fxmacrodata_snowflake/__init__.py +201 -0
- snowflake_fxmacrodata-0.1.0/src/snowflake_fxmacrodata.egg-info/PKG-INFO +155 -0
- snowflake_fxmacrodata-0.1.0/src/snowflake_fxmacrodata.egg-info/SOURCES.txt +14 -0
- snowflake_fxmacrodata-0.1.0/src/snowflake_fxmacrodata.egg-info/dependency_links.txt +1 -0
- snowflake_fxmacrodata-0.1.0/src/snowflake_fxmacrodata.egg-info/requires.txt +6 -0
- snowflake_fxmacrodata-0.1.0/src/snowflake_fxmacrodata.egg-info/top_level.txt +1 -0
- snowflake_fxmacrodata-0.1.0/tests/test_loader.py +215 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 FXMacroData
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
include LICENSE
|
|
2
|
+
include README.md
|
|
3
|
+
include pyproject.toml
|
|
4
|
+
include PUBLIC_FILES.json
|
|
5
|
+
include MANIFEST.in
|
|
6
|
+
include src/fxmacrodata_snowflake/__init__.py
|
|
7
|
+
include tests/test_loader.py
|
|
8
|
+
include examples/event_study.sql
|
|
9
|
+
include examples/sharing.sql
|
|
10
|
+
global-exclude *.pyc *.pyo
|
|
11
|
+
global-exclude __pycache__/*
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: snowflake-fxmacrodata
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Load public FXMacroData responses into Snowflake
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Documentation, https://fxmacrodata.com/documentation/reference?utm_source=pypi&utm_medium=referral&utm_campaign=snowflake-fxmacrodata&utm_content=package
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: fxmacrodata-platform-core==0.1.0
|
|
11
|
+
Requires-Dist: snowflake-connector-python<5,>=4.8.1
|
|
12
|
+
Requires-Dist: filelock<5,>=4.2.9
|
|
13
|
+
Provides-Extra: test
|
|
14
|
+
Requires-Dist: pytest<9,>=8; extra == "test"
|
|
15
|
+
Dynamic: license-file
|
|
16
|
+
|
|
17
|
+
# FXMacroData for Snowflake
|
|
18
|
+
|
|
19
|
+
Load FXMacroData public responses through the official Snowflake Connector for
|
|
20
|
+
Python. Bounded batch inserts feed a temporary staging table, followed by a
|
|
21
|
+
transactional merge into your analytical table.
|
|
22
|
+
|
|
23
|
+
[FXMacroData documentation](https://fxmacrodata.com/documentation/reference?utm_source=github&utm_medium=referral&utm_campaign=snowflake-fxmacrodata&utm_content=readme)
|
|
24
|
+
|
|
25
|
+
## Install and connect
|
|
26
|
+
|
|
27
|
+
Requires Python 3.10 or later and Snowflake Connector for Python 4.8.1 or later
|
|
28
|
+
within version 4. Install the supplied adapter wheel; its public client and
|
|
29
|
+
platform core dependencies resolve from PyPI:
|
|
30
|
+
|
|
31
|
+
```sh
|
|
32
|
+
python -m pip install /path/to/snowflake_fxmacrodata-0.1.0-py3-none-any.whl
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Use a dedicated Snowflake connection with its current database and schema set.
|
|
36
|
+
This example uses interactive SSO; headless workloads should use your
|
|
37
|
+
organization's approved SDK authentication method and runtime secret store.
|
|
38
|
+
Expose the FXMacroData source key through the `FXMD_API_KEY` environment name.
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
import os
|
|
42
|
+
import snowflake.connector
|
|
43
|
+
from fxmacrodata_platform_core import read
|
|
44
|
+
from fxmacrodata_snowflake import load
|
|
45
|
+
|
|
46
|
+
connection = snowflake.connector.connect(
|
|
47
|
+
account=os.environ["SNOWFLAKE_ACCOUNT"],
|
|
48
|
+
user=os.environ["SNOWFLAKE_USER"],
|
|
49
|
+
authenticator="externalbrowser",
|
|
50
|
+
warehouse=os.environ["SNOWFLAKE_WAREHOUSE"],
|
|
51
|
+
database=os.environ["SNOWFLAKE_DATABASE"],
|
|
52
|
+
schema=os.environ["SNOWFLAKE_SCHEMA"],
|
|
53
|
+
paramstyle="qmark",
|
|
54
|
+
)
|
|
55
|
+
try:
|
|
56
|
+
batch = read("release_calendar", {"currency": "usd"}, api_key_env="FXMD_API_KEY")
|
|
57
|
+
result = load(batch, connection, "FXMACRODATA", paramstyle="qmark")
|
|
58
|
+
finally:
|
|
59
|
+
connection.close()
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
One-, two- and three-part simple SQL names are accepted and normalized to
|
|
63
|
+
uppercase, matching Snowflake's unquoted identifier convention. Quoted,
|
|
64
|
+
case-sensitive names and SQL fragments are rejected. The database and schema
|
|
65
|
+
must exist. The caller needs table/stage creation and DML privileges. Use
|
|
66
|
+
`create_table=False` if an administrator has provisioned the envelope table with
|
|
67
|
+
`create_table_sql(table)`.
|
|
68
|
+
|
|
69
|
+
The loader must own the connection's transaction. Snowflake DDL can commit
|
|
70
|
+
an active transaction, so do not pass a connection with unrelated pending work.
|
|
71
|
+
`paramstyle` must match the connection (`qmark`, `numeric`, `format`, or
|
|
72
|
+
`pyformat`, the connector default). No global connector settings are changed.
|
|
73
|
+
|
|
74
|
+
Every operation exposed by `fxmacrodata_platform_core.operations()` is available,
|
|
75
|
+
including REST and MCP operations. The loader adds a tabular envelope without
|
|
76
|
+
changing the original public response. `record_json` retains the original record;
|
|
77
|
+
`response_json` retains the complete response, including pagination and metadata.
|
|
78
|
+
An empty response produces one envelope with `record_json = 'null'`, so absence of
|
|
79
|
+
data and the server's metadata remain inspectable. It is not an observation.
|
|
80
|
+
|
|
81
|
+
Columns are `operation`, `source_url`, `record_id`, `currency`, `indicator`,
|
|
82
|
+
`date`, `announcement_datetime`, `value`, `value_text`, `record_json`,
|
|
83
|
+
`response_json`, and the loader's `content_hash`. Only `value` is numeric.
|
|
84
|
+
Dates and announcement times remain source strings. No publication time is
|
|
85
|
+
inferred, and no missing value is filled. `value_text` and JSON preserve source
|
|
86
|
+
numbers supplied as strings. JSON numeric tokens follow the public Python client
|
|
87
|
+
parser, so original lexical precision is not guaranteed.
|
|
88
|
+
|
|
89
|
+
Select operations with the shared catalogue rather than constructing URLs:
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
from fxmacrodata_platform_core import operations, schemas
|
|
93
|
+
|
|
94
|
+
available = operations()
|
|
95
|
+
input_schemas = schemas()
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
`load_operation` accepts `api_key_env`, `complete_history`, `max_pages`, and
|
|
99
|
+
`client_factory` for the public source reader. `max_pages` bounds history
|
|
100
|
+
pagination; an incomplete read raises an error before a destination load starts.
|
|
101
|
+
`batch_size` bounds destination blocks to 1–10,000 rows. `max_batch_bytes` also
|
|
102
|
+
bounds each block using a conservative JSON byte estimate (16 MiB by default,
|
|
103
|
+
128 MiB maximum). An oversized individual envelope is rejected before destination
|
|
104
|
+
changes; narrow the source date range to reduce the response size. The source response is
|
|
105
|
+
materialized before loading; choose a source date range suitable for memory.
|
|
106
|
+
Read the installed operation's schema for required arguments and date filters.
|
|
107
|
+
|
|
108
|
+
Supply source credentials through your runtime secret store, exposing only the
|
|
109
|
+
environment variable name to the adapter. Destination authentication belongs to
|
|
110
|
+
the official SDK connection you pass to the loader. The adapter neither saves nor
|
|
111
|
+
prints credentials and never constructs destination credentials for you.
|
|
112
|
+
|
|
113
|
+
The package licence covers this adapter code. FXMacroData data access and onward
|
|
114
|
+
redistribution remain subject to your subscription and applicable licence. Loading
|
|
115
|
+
data into a warehouse does not grant permission to publish or resell it.
|
|
116
|
+
|
|
117
|
+
## Retries and querying
|
|
118
|
+
|
|
119
|
+
The loader creates the destination and temporary table, then fills every bounded
|
|
120
|
+
block before beginning the destination DML transaction. This also keeps any
|
|
121
|
+
temporary binding-stage DDL issued by the connector outside the transaction.
|
|
122
|
+
It then begins, merges on operation, record ID and content hash, and commits.
|
|
123
|
+
A staging failure leaves destination rows untouched. A merge or commit failure
|
|
124
|
+
triggers rollback. If rollback itself fails, the loader avoids cleanup DDL that
|
|
125
|
+
could commit unresolved work.
|
|
126
|
+
|
|
127
|
+
Repeated exact envelopes are skipped; changed source records or metadata remain
|
|
128
|
+
distinct. Use one writer per destination table. Concurrent merges into standard
|
|
129
|
+
Snowflake tables are not an exactly-once guarantee. The loader never retries a
|
|
130
|
+
write automatically. After connection loss, inspect the transaction outcome
|
|
131
|
+
before retrying. `LoadResult.rows_submitted` counts unique envelopes offered to
|
|
132
|
+
the merge, rather than rows newly inserted.
|
|
133
|
+
|
|
134
|
+
```sql
|
|
135
|
+
SELECT CURRENCY, INDICATOR, DATE, VALUE,
|
|
136
|
+
PARSE_JSON(RECORD_JSON):publication_time_status::STRING AS PUBLICATION_STATUS
|
|
137
|
+
FROM FXMACRODATA
|
|
138
|
+
WHERE RECORD_JSON != 'null';
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
[event_study.sql](examples/event_study.sql) joins source announcement times to
|
|
142
|
+
your licensed market-price table. Inspect source publication metadata separately
|
|
143
|
+
for point-in-time backtesting. [sharing.sql](examples/sharing.sql) shows a
|
|
144
|
+
customer-controlled secure share; sharing data requires the applicable rights
|
|
145
|
+
and recipient approval. This package does not create a Marketplace listing.
|
|
146
|
+
|
|
147
|
+
## Test
|
|
148
|
+
|
|
149
|
+
```sh
|
|
150
|
+
python -m pytest tests -q
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
Official SDK references:
|
|
154
|
+
[batch binding and transactions](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-example),
|
|
155
|
+
[connector API](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-api).
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
# FXMacroData for Snowflake
|
|
2
|
+
|
|
3
|
+
Load FXMacroData public responses through the official Snowflake Connector for
|
|
4
|
+
Python. Bounded batch inserts feed a temporary staging table, followed by a
|
|
5
|
+
transactional merge into your analytical table.
|
|
6
|
+
|
|
7
|
+
[FXMacroData documentation](https://fxmacrodata.com/documentation/reference?utm_source=github&utm_medium=referral&utm_campaign=snowflake-fxmacrodata&utm_content=readme)
|
|
8
|
+
|
|
9
|
+
## Install and connect
|
|
10
|
+
|
|
11
|
+
Requires Python 3.10 or later and Snowflake Connector for Python 4.8.1 or later
|
|
12
|
+
within version 4. Install the supplied adapter wheel; its public client and
|
|
13
|
+
platform core dependencies resolve from PyPI:
|
|
14
|
+
|
|
15
|
+
```sh
|
|
16
|
+
python -m pip install /path/to/snowflake_fxmacrodata-0.1.0-py3-none-any.whl
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Use a dedicated Snowflake connection with its current database and schema set.
|
|
20
|
+
This example uses interactive SSO; headless workloads should use your
|
|
21
|
+
organization's approved SDK authentication method and runtime secret store.
|
|
22
|
+
Expose the FXMacroData source key through the `FXMD_API_KEY` environment name.
|
|
23
|
+
|
|
24
|
+
```python
|
|
25
|
+
import os
|
|
26
|
+
import snowflake.connector
|
|
27
|
+
from fxmacrodata_platform_core import read
|
|
28
|
+
from fxmacrodata_snowflake import load
|
|
29
|
+
|
|
30
|
+
connection = snowflake.connector.connect(
|
|
31
|
+
account=os.environ["SNOWFLAKE_ACCOUNT"],
|
|
32
|
+
user=os.environ["SNOWFLAKE_USER"],
|
|
33
|
+
authenticator="externalbrowser",
|
|
34
|
+
warehouse=os.environ["SNOWFLAKE_WAREHOUSE"],
|
|
35
|
+
database=os.environ["SNOWFLAKE_DATABASE"],
|
|
36
|
+
schema=os.environ["SNOWFLAKE_SCHEMA"],
|
|
37
|
+
paramstyle="qmark",
|
|
38
|
+
)
|
|
39
|
+
try:
|
|
40
|
+
batch = read("release_calendar", {"currency": "usd"}, api_key_env="FXMD_API_KEY")
|
|
41
|
+
result = load(batch, connection, "FXMACRODATA", paramstyle="qmark")
|
|
42
|
+
finally:
|
|
43
|
+
connection.close()
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
One-, two- and three-part simple SQL names are accepted and normalized to
|
|
47
|
+
uppercase, matching Snowflake's unquoted identifier convention. Quoted,
|
|
48
|
+
case-sensitive names and SQL fragments are rejected. The database and schema
|
|
49
|
+
must exist. The caller needs table/stage creation and DML privileges. Use
|
|
50
|
+
`create_table=False` if an administrator has provisioned the envelope table with
|
|
51
|
+
`create_table_sql(table)`.
|
|
52
|
+
|
|
53
|
+
The loader must own the connection's transaction. Snowflake DDL can commit
|
|
54
|
+
an active transaction, so do not pass a connection with unrelated pending work.
|
|
55
|
+
`paramstyle` must match the connection (`qmark`, `numeric`, `format`, or
|
|
56
|
+
`pyformat`, the connector default). No global connector settings are changed.
|
|
57
|
+
|
|
58
|
+
Every operation exposed by `fxmacrodata_platform_core.operations()` is available,
|
|
59
|
+
including REST and MCP operations. The loader adds a tabular envelope without
|
|
60
|
+
changing the original public response. `record_json` retains the original record;
|
|
61
|
+
`response_json` retains the complete response, including pagination and metadata.
|
|
62
|
+
An empty response produces one envelope with `record_json = 'null'`, so absence of
|
|
63
|
+
data and the server's metadata remain inspectable. It is not an observation.
|
|
64
|
+
|
|
65
|
+
Columns are `operation`, `source_url`, `record_id`, `currency`, `indicator`,
|
|
66
|
+
`date`, `announcement_datetime`, `value`, `value_text`, `record_json`,
|
|
67
|
+
`response_json`, and the loader's `content_hash`. Only `value` is numeric.
|
|
68
|
+
Dates and announcement times remain source strings. No publication time is
|
|
69
|
+
inferred, and no missing value is filled. `value_text` and JSON preserve source
|
|
70
|
+
numbers supplied as strings. JSON numeric tokens follow the public Python client
|
|
71
|
+
parser, so original lexical precision is not guaranteed.
|
|
72
|
+
|
|
73
|
+
Select operations with the shared catalogue rather than constructing URLs:
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
from fxmacrodata_platform_core import operations, schemas
|
|
77
|
+
|
|
78
|
+
available = operations()
|
|
79
|
+
input_schemas = schemas()
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
`load_operation` accepts `api_key_env`, `complete_history`, `max_pages`, and
|
|
83
|
+
`client_factory` for the public source reader. `max_pages` bounds history
|
|
84
|
+
pagination; an incomplete read raises an error before a destination load starts.
|
|
85
|
+
`batch_size` bounds destination blocks to 1–10,000 rows. `max_batch_bytes` also
|
|
86
|
+
bounds each block using a conservative JSON byte estimate (16 MiB by default,
|
|
87
|
+
128 MiB maximum). An oversized individual envelope is rejected before destination
|
|
88
|
+
changes; narrow the source date range to reduce the response size. The source response is
|
|
89
|
+
materialized before loading; choose a source date range suitable for memory.
|
|
90
|
+
Read the installed operation's schema for required arguments and date filters.
|
|
91
|
+
|
|
92
|
+
Supply source credentials through your runtime secret store, exposing only the
|
|
93
|
+
environment variable name to the adapter. Destination authentication belongs to
|
|
94
|
+
the official SDK connection you pass to the loader. The adapter neither saves nor
|
|
95
|
+
prints credentials and never constructs destination credentials for you.
|
|
96
|
+
|
|
97
|
+
The package licence covers this adapter code. FXMacroData data access and onward
|
|
98
|
+
redistribution remain subject to your subscription and applicable licence. Loading
|
|
99
|
+
data into a warehouse does not grant permission to publish or resell it.
|
|
100
|
+
|
|
101
|
+
## Retries and querying
|
|
102
|
+
|
|
103
|
+
The loader creates the destination and temporary table, then fills every bounded
|
|
104
|
+
block before beginning the destination DML transaction. This also keeps any
|
|
105
|
+
temporary binding-stage DDL issued by the connector outside the transaction.
|
|
106
|
+
It then begins, merges on operation, record ID and content hash, and commits.
|
|
107
|
+
A staging failure leaves destination rows untouched. A merge or commit failure
|
|
108
|
+
triggers rollback. If rollback itself fails, the loader avoids cleanup DDL that
|
|
109
|
+
could commit unresolved work.
|
|
110
|
+
|
|
111
|
+
Repeated exact envelopes are skipped; changed source records or metadata remain
|
|
112
|
+
distinct. Use one writer per destination table. Concurrent merges into standard
|
|
113
|
+
Snowflake tables are not an exactly-once guarantee. The loader never retries a
|
|
114
|
+
write automatically. After connection loss, inspect the transaction outcome
|
|
115
|
+
before retrying. `LoadResult.rows_submitted` counts unique envelopes offered to
|
|
116
|
+
the merge, rather than rows newly inserted.
|
|
117
|
+
|
|
118
|
+
```sql
|
|
119
|
+
SELECT CURRENCY, INDICATOR, DATE, VALUE,
|
|
120
|
+
PARSE_JSON(RECORD_JSON):publication_time_status::STRING AS PUBLICATION_STATUS
|
|
121
|
+
FROM FXMACRODATA
|
|
122
|
+
WHERE RECORD_JSON != 'null';
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
[event_study.sql](examples/event_study.sql) joins source announcement times to
|
|
126
|
+
your licensed market-price table. Inspect source publication metadata separately
|
|
127
|
+
for point-in-time backtesting. [sharing.sql](examples/sharing.sql) shows a
|
|
128
|
+
customer-controlled secure share; sharing data requires the applicable rights
|
|
129
|
+
and recipient approval. This package does not create a Marketplace listing.
|
|
130
|
+
|
|
131
|
+
## Test
|
|
132
|
+
|
|
133
|
+
```sh
|
|
134
|
+
python -m pytest tests -q
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
Official SDK references:
|
|
138
|
+
[batch binding and transactions](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-example),
|
|
139
|
+
[connector API](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-api).
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
-- Requires MARKET_PRICES(CURRENCY VARCHAR, TS TIMESTAMP_TZ, CLOSE DOUBLE).
|
|
2
|
+
-- Inspect source publisher/publication metadata for point-in-time research.
|
|
3
|
+
WITH events AS (
|
|
4
|
+
SELECT DISTINCT RECORD_ID, CURRENCY,
|
|
5
|
+
TRY_TO_TIMESTAMP_TZ(ANNOUNCEMENT_DATETIME) AS EVENT_TIME
|
|
6
|
+
FROM FXMACRODATA
|
|
7
|
+
WHERE ANNOUNCEMENT_DATETIME IS NOT NULL AND RECORD_JSON != 'null'
|
|
8
|
+
), prior AS (
|
|
9
|
+
SELECT e.RECORD_ID, e.CURRENCY, e.EVENT_TIME, p.CLOSE AS PRE_PRICE
|
|
10
|
+
FROM events e JOIN MARKET_PRICES p ON p.CURRENCY = e.CURRENCY
|
|
11
|
+
AND p.TS BETWEEN DATEADD(minute, -5, e.EVENT_TIME) AND e.EVENT_TIME
|
|
12
|
+
QUALIFY ROW_NUMBER() OVER (
|
|
13
|
+
PARTITION BY e.RECORD_ID, e.CURRENCY, e.EVENT_TIME ORDER BY p.TS DESC
|
|
14
|
+
) = 1
|
|
15
|
+
), following AS (
|
|
16
|
+
SELECT e.RECORD_ID, e.CURRENCY, e.EVENT_TIME, p.CLOSE AS POST_PRICE
|
|
17
|
+
FROM events e JOIN MARKET_PRICES p ON p.CURRENCY = e.CURRENCY
|
|
18
|
+
AND p.TS BETWEEN DATEADD(minute, 5, e.EVENT_TIME) AND DATEADD(minute, 10, e.EVENT_TIME)
|
|
19
|
+
QUALIFY ROW_NUMBER() OVER (
|
|
20
|
+
PARTITION BY e.RECORD_ID, e.CURRENCY, e.EVENT_TIME ORDER BY p.TS ASC
|
|
21
|
+
) = 1
|
|
22
|
+
)
|
|
23
|
+
SELECT p.RECORD_ID, p.CURRENCY, p.EVENT_TIME, p.PRE_PRICE, f.POST_PRICE,
|
|
24
|
+
(f.POST_PRICE / NULLIF(p.PRE_PRICE, 0) - 1) * 100 AS RETURN_PCT
|
|
25
|
+
FROM prior p JOIN following f USING (RECORD_ID, CURRENCY, EVENT_TIME);
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
-- Run only after licensing and recipient-access review in your own account.
|
|
2
|
+
-- Replace customer-specific names before execution. These commands are not run by the loader.
|
|
3
|
+
-- The source database/schema must already exist and contain your reviewed data.
|
|
4
|
+
CREATE SHARE FXMD_APPROVED_RESEARCH;
|
|
5
|
+
GRANT USAGE ON DATABASE RESEARCH TO SHARE FXMD_APPROVED_RESEARCH;
|
|
6
|
+
GRANT USAGE ON SCHEMA RESEARCH.MACRO TO SHARE FXMD_APPROVED_RESEARCH;
|
|
7
|
+
CREATE SECURE VIEW RESEARCH.MACRO.FXMD_SHARED AS
|
|
8
|
+
SELECT OPERATION, SOURCE_URL, RECORD_ID, CURRENCY, INDICATOR,
|
|
9
|
+
DATE, ANNOUNCEMENT_DATETIME, VALUE, VALUE_TEXT, RECORD_JSON
|
|
10
|
+
FROM RESEARCH.MACRO.FXMACRODATA
|
|
11
|
+
WHERE RECORD_JSON != 'null';
|
|
12
|
+
GRANT SELECT ON VIEW RESEARCH.MACRO.FXMD_SHARED TO SHARE FXMD_APPROVED_RESEARCH;
|
|
13
|
+
-- After verifying the recipient identifier and its data entitlement:
|
|
14
|
+
-- ALTER SHARE FXMD_APPROVED_RESEARCH ADD ACCOUNTS = RECIPIENT_ORG.RECIPIENT_ACCOUNT;
|
|
15
|
+
-- Recipient onboarding and a Marketplace listing are separate operator workflows.
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "snowflake-fxmacrodata"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Load public FXMacroData responses into Snowflake"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
dependencies = ["fxmacrodata-platform-core==0.1.0", "snowflake-connector-python>=4.8.1,<5", "filelock>=4.2.9,<5"]
|
|
13
|
+
|
|
14
|
+
[project.urls]
|
|
15
|
+
Documentation = "https://fxmacrodata.com/documentation/reference?utm_source=pypi&utm_medium=referral&utm_campaign=snowflake-fxmacrodata&utm_content=package"
|
|
16
|
+
|
|
17
|
+
[project.optional-dependencies]
|
|
18
|
+
test = ["pytest>=8,<9"]
|
|
19
|
+
|
|
20
|
+
[tool.setuptools]
|
|
21
|
+
packages = ["fxmacrodata_snowflake"]
|
|
22
|
+
package-dir = {"" = "src"}
|
|
23
|
+
include-package-data = false
|
|
24
|
+
|
|
25
|
+
[tool.pytest.ini_options]
|
|
26
|
+
testpaths = ["tests"]
|
|
27
|
+
addopts = "-q"
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
"""Load public FXMacroData responses into a customer-owned Snowflake account."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import math
|
|
9
|
+
import re
|
|
10
|
+
from typing import Any
|
|
11
|
+
from uuid import uuid4
|
|
12
|
+
|
|
13
|
+
from fxmacrodata_platform_core import read
|
|
14
|
+
|
|
15
|
+
__version__ = "0.1.0"
|
|
16
|
+
COLUMNS = (
|
|
17
|
+
"operation", "source_url", "record_id", "currency", "indicator", "date",
|
|
18
|
+
"announcement_datetime", "value", "value_text", "record_json", "response_json",
|
|
19
|
+
"content_hash",
|
|
20
|
+
)
|
|
21
|
+
_IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]{0,254}\Z")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class LoadError(RuntimeError):
|
|
25
|
+
"""A load failed; transaction outcome must be checked after connection loss."""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class LoadResult:
|
|
30
|
+
table: str
|
|
31
|
+
rows_submitted: int
|
|
32
|
+
batches: int
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _table(table: str) -> str:
|
|
36
|
+
if not isinstance(table, str):
|
|
37
|
+
raise ValueError("table must contain simple Snowflake identifiers")
|
|
38
|
+
parts = table.split(".")
|
|
39
|
+
if not 1 <= len(parts) <= 3 or any(not _IDENTIFIER.fullmatch(p) for p in parts):
|
|
40
|
+
raise ValueError("table must contain one to three simple SQL identifiers")
|
|
41
|
+
return ".".join(f'"{p.upper()}"' for p in parts)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _rows(batch: Any) -> list[dict[str, Any]]:
|
|
45
|
+
rows = []
|
|
46
|
+
seen = set()
|
|
47
|
+
for source in batch.rows():
|
|
48
|
+
row = {name: source.get(name) for name in COLUMNS[:-1]}
|
|
49
|
+
if not isinstance(row["operation"], str) or not isinstance(row["record_id"], str):
|
|
50
|
+
raise ValueError("every row requires string operation and record_id")
|
|
51
|
+
for name, value in row.items():
|
|
52
|
+
if name == "value":
|
|
53
|
+
if value is not None and (
|
|
54
|
+
isinstance(value, bool) or not isinstance(value, (int, float)) or
|
|
55
|
+
not math.isfinite(value)
|
|
56
|
+
):
|
|
57
|
+
raise ValueError("value must be a finite number or null")
|
|
58
|
+
elif value is not None and not isinstance(value, str):
|
|
59
|
+
raise ValueError(f"{name} must be a string or null")
|
|
60
|
+
encoded = json.dumps(row, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
|
61
|
+
row["content_hash"] = hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
|
62
|
+
key = (row["operation"], row["record_id"], row["content_hash"])
|
|
63
|
+
if key not in seen:
|
|
64
|
+
seen.add(key)
|
|
65
|
+
rows.append(row)
|
|
66
|
+
return rows
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def create_table_sql(table: str) -> str:
|
|
70
|
+
"""Return DDL preserving source dates, publication status and JSON as text."""
|
|
71
|
+
fields = ",\n ".join(
|
|
72
|
+
f'"{name.upper()}" {"DOUBLE" if name == "value" else "VARCHAR"}' +
|
|
73
|
+
(" NOT NULL" if name in {"operation", "record_id", "content_hash"} else "")
|
|
74
|
+
for name in COLUMNS
|
|
75
|
+
)
|
|
76
|
+
return f"CREATE TABLE IF NOT EXISTS {_table(table)} (\n {fields}\n)"
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _blocks(rows: list[dict[str, Any]], batch_size: int, max_batch_bytes: int) -> list[list[dict[str, Any]]]:
|
|
80
|
+
if isinstance(batch_size, bool) or not isinstance(batch_size, int) or not 1 <= batch_size <= 10000:
|
|
81
|
+
raise ValueError("batch_size must be an integer between 1 and 10000")
|
|
82
|
+
if isinstance(max_batch_bytes, bool) or not isinstance(max_batch_bytes, int) or not 1 <= max_batch_bytes <= 128 * 1024 * 1024:
|
|
83
|
+
raise ValueError("max_batch_bytes must be an integer between 1 and 134217728")
|
|
84
|
+
output, block, size = [], [], 0
|
|
85
|
+
for row in rows:
|
|
86
|
+
length = len(json.dumps(row, ensure_ascii=True).encode("utf-8")) + 2
|
|
87
|
+
if length > max_batch_bytes:
|
|
88
|
+
raise ValueError("one envelope exceeds max_batch_bytes; narrow the source date range")
|
|
89
|
+
if block and (len(block) >= batch_size or size + length > max_batch_bytes):
|
|
90
|
+
output.append(block)
|
|
91
|
+
block, size = [], 0
|
|
92
|
+
block.append(row)
|
|
93
|
+
size += length
|
|
94
|
+
if block:
|
|
95
|
+
output.append(block)
|
|
96
|
+
return output
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def load(batch: Any, connection: Any, table: str, *, batch_size: int = 1000,
|
|
100
|
+
max_batch_bytes: int = 16 * 1024 * 1024,
|
|
101
|
+
create_table: bool = True, paramstyle: str = "pyformat") -> LoadResult:
|
|
102
|
+
"""Stage bounded batches before merging envelopes in one DML transaction.
|
|
103
|
+
|
|
104
|
+
Use a dedicated connection without an active transaction. Table and temporary
|
|
105
|
+
stage DDL and all batch binding execute before BEGIN: the native connector
|
|
106
|
+
can create a binding stage during executemany. Repeated source envelopes
|
|
107
|
+
match existing rows; changed response metadata or revisions remain distinct.
|
|
108
|
+
"""
|
|
109
|
+
quoted = _table(table)
|
|
110
|
+
if isinstance(batch_size, bool) or not isinstance(batch_size, int) or not 1 <= batch_size <= 10000:
|
|
111
|
+
raise ValueError("batch_size must be an integer between 1 and 10000")
|
|
112
|
+
if paramstyle not in {"pyformat", "format", "qmark", "numeric"}:
|
|
113
|
+
raise ValueError("paramstyle must match the Snowflake connection")
|
|
114
|
+
rows = _rows(batch)
|
|
115
|
+
blocks = _blocks(rows, batch_size, max_batch_bytes)
|
|
116
|
+
stage = f'"FXMD_STAGE_{uuid4().hex.upper()}"'
|
|
117
|
+
fields = ", ".join(f'"{name.upper()}"' for name in COLUMNS)
|
|
118
|
+
placeholder = ", ".join(
|
|
119
|
+
"?" if paramstyle == "qmark" else f":{i + 1}" if paramstyle == "numeric" else "%s"
|
|
120
|
+
for i in range(len(COLUMNS))
|
|
121
|
+
)
|
|
122
|
+
cursor = None
|
|
123
|
+
transaction = False
|
|
124
|
+
stage_created = False
|
|
125
|
+
committed = False
|
|
126
|
+
batches = 0
|
|
127
|
+
try:
|
|
128
|
+
cursor = connection.cursor()
|
|
129
|
+
if create_table:
|
|
130
|
+
cursor.execute(create_table_sql(table))
|
|
131
|
+
cursor.execute(f"CREATE TEMPORARY TABLE {stage} LIKE {quoted}")
|
|
132
|
+
stage_created = True
|
|
133
|
+
for block in blocks:
|
|
134
|
+
cursor.executemany(
|
|
135
|
+
f"INSERT INTO {stage} ({fields}) VALUES ({placeholder})",
|
|
136
|
+
[tuple(row[name] for name in COLUMNS) for row in block],
|
|
137
|
+
)
|
|
138
|
+
batches += 1
|
|
139
|
+
# qmark/numeric binding can issue CREATE TEMPORARY STAGE internally.
|
|
140
|
+
# Finish every staging insert before the destination DML transaction.
|
|
141
|
+
cursor.execute("BEGIN")
|
|
142
|
+
transaction = True
|
|
143
|
+
cursor.execute(
|
|
144
|
+
f"MERGE INTO {quoted} AS target USING {stage} AS source ON "
|
|
145
|
+
'target."OPERATION" = source."OPERATION" AND '
|
|
146
|
+
'target."RECORD_ID" = source."RECORD_ID" AND '
|
|
147
|
+
'target."CONTENT_HASH" = source."CONTENT_HASH" '
|
|
148
|
+
f"WHEN NOT MATCHED THEN INSERT ({fields}) VALUES (" +
|
|
149
|
+
", ".join(f'source."{name.upper()}"' for name in COLUMNS) + ")"
|
|
150
|
+
)
|
|
151
|
+
cursor.execute("COMMIT")
|
|
152
|
+
transaction = False
|
|
153
|
+
committed = True
|
|
154
|
+
except Exception:
|
|
155
|
+
if transaction and cursor is not None:
|
|
156
|
+
try:
|
|
157
|
+
cursor.execute("ROLLBACK")
|
|
158
|
+
transaction = False
|
|
159
|
+
except Exception:
|
|
160
|
+
pass
|
|
161
|
+
raise LoadError(
|
|
162
|
+
"Snowflake load did not complete successfully. Check transaction outcome "
|
|
163
|
+
"before retrying; use a single writer per destination table."
|
|
164
|
+
) from None
|
|
165
|
+
finally:
|
|
166
|
+
# DDL commits transactions, so never run cleanup after a failed rollback.
|
|
167
|
+
if cursor is not None:
|
|
168
|
+
if stage_created and not transaction:
|
|
169
|
+
try:
|
|
170
|
+
cursor.execute(f"DROP TABLE IF EXISTS {stage}")
|
|
171
|
+
except Exception:
|
|
172
|
+
pass
|
|
173
|
+
try:
|
|
174
|
+
cursor.close()
|
|
175
|
+
except Exception:
|
|
176
|
+
pass
|
|
177
|
+
if not committed:
|
|
178
|
+
raise LoadError("Snowflake transaction was not committed")
|
|
179
|
+
return LoadResult(quoted, len(rows), batches)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def load_operation(operation: str, args: dict[str, Any] | None, connection: Any, table: str,
|
|
183
|
+
*, api_key_env: str | None = None, complete_history: bool = True,
|
|
184
|
+
max_pages: int = 10000, client_factory: Any = None,
|
|
185
|
+
batch_size: int = 1000, max_batch_bytes: int = 16 * 1024 * 1024,
|
|
186
|
+
create_table: bool = True,
|
|
187
|
+
paramstyle: str = "pyformat") -> LoadResult:
|
|
188
|
+
"""Read one public operation and load its complete response envelope."""
|
|
189
|
+
_table(table)
|
|
190
|
+
_blocks([], batch_size, max_batch_bytes)
|
|
191
|
+
if paramstyle not in {"pyformat", "format", "qmark", "numeric"}:
|
|
192
|
+
raise ValueError("paramstyle must match the Snowflake connection")
|
|
193
|
+
batch = read(operation, args, api_key_env=api_key_env,
|
|
194
|
+
complete_history=complete_history, max_pages=max_pages,
|
|
195
|
+
client_factory=client_factory)
|
|
196
|
+
return load(batch, connection, table, batch_size=batch_size,
|
|
197
|
+
max_batch_bytes=max_batch_bytes, create_table=create_table,
|
|
198
|
+
paramstyle=paramstyle)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
__all__ = ["COLUMNS", "LoadError", "LoadResult", "create_table_sql", "load", "load_operation"]
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: snowflake-fxmacrodata
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Load public FXMacroData responses into Snowflake
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Documentation, https://fxmacrodata.com/documentation/reference?utm_source=pypi&utm_medium=referral&utm_campaign=snowflake-fxmacrodata&utm_content=package
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: fxmacrodata-platform-core==0.1.0
|
|
11
|
+
Requires-Dist: snowflake-connector-python<5,>=4.8.1
|
|
12
|
+
Requires-Dist: filelock<5,>=4.2.9
|
|
13
|
+
Provides-Extra: test
|
|
14
|
+
Requires-Dist: pytest<9,>=8; extra == "test"
|
|
15
|
+
Dynamic: license-file
|
|
16
|
+
|
|
17
|
+
# FXMacroData for Snowflake
|
|
18
|
+
|
|
19
|
+
Load FXMacroData public responses through the official Snowflake Connector for
|
|
20
|
+
Python. Bounded batch inserts feed a temporary staging table, followed by a
|
|
21
|
+
transactional merge into your analytical table.
|
|
22
|
+
|
|
23
|
+
[FXMacroData documentation](https://fxmacrodata.com/documentation/reference?utm_source=github&utm_medium=referral&utm_campaign=snowflake-fxmacrodata&utm_content=readme)
|
|
24
|
+
|
|
25
|
+
## Install and connect
|
|
26
|
+
|
|
27
|
+
Requires Python 3.10 or later and Snowflake Connector for Python 4.8.1 or later
|
|
28
|
+
within version 4. Install the supplied adapter wheel; its public client and
|
|
29
|
+
platform core dependencies resolve from PyPI:
|
|
30
|
+
|
|
31
|
+
```sh
|
|
32
|
+
python -m pip install /path/to/snowflake_fxmacrodata-0.1.0-py3-none-any.whl
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Use a dedicated Snowflake connection with its current database and schema set.
|
|
36
|
+
This example uses interactive SSO; headless workloads should use your
|
|
37
|
+
organization's approved SDK authentication method and runtime secret store.
|
|
38
|
+
Expose the FXMacroData source key through the `FXMD_API_KEY` environment name.
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
import os
|
|
42
|
+
import snowflake.connector
|
|
43
|
+
from fxmacrodata_platform_core import read
|
|
44
|
+
from fxmacrodata_snowflake import load
|
|
45
|
+
|
|
46
|
+
connection = snowflake.connector.connect(
|
|
47
|
+
account=os.environ["SNOWFLAKE_ACCOUNT"],
|
|
48
|
+
user=os.environ["SNOWFLAKE_USER"],
|
|
49
|
+
authenticator="externalbrowser",
|
|
50
|
+
warehouse=os.environ["SNOWFLAKE_WAREHOUSE"],
|
|
51
|
+
database=os.environ["SNOWFLAKE_DATABASE"],
|
|
52
|
+
schema=os.environ["SNOWFLAKE_SCHEMA"],
|
|
53
|
+
paramstyle="qmark",
|
|
54
|
+
)
|
|
55
|
+
try:
|
|
56
|
+
batch = read("release_calendar", {"currency": "usd"}, api_key_env="FXMD_API_KEY")
|
|
57
|
+
result = load(batch, connection, "FXMACRODATA", paramstyle="qmark")
|
|
58
|
+
finally:
|
|
59
|
+
connection.close()
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
One-, two- and three-part simple SQL names are accepted and normalized to
|
|
63
|
+
uppercase, matching Snowflake's unquoted identifier convention. Quoted,
|
|
64
|
+
case-sensitive names and SQL fragments are rejected. The database and schema
|
|
65
|
+
must exist. The caller needs table/stage creation and DML privileges. Use
|
|
66
|
+
`create_table=False` if an administrator has provisioned the envelope table with
|
|
67
|
+
`create_table_sql(table)`.
|
|
68
|
+
|
|
69
|
+
The loader must own the connection's transaction. Snowflake DDL can commit
|
|
70
|
+
an active transaction, so do not pass a connection with unrelated pending work.
|
|
71
|
+
`paramstyle` must match the connection (`qmark`, `numeric`, `format`, or
|
|
72
|
+
`pyformat`, the connector default). No global connector settings are changed.
|
|
73
|
+
|
|
74
|
+
Every operation exposed by `fxmacrodata_platform_core.operations()` is available,
|
|
75
|
+
including REST and MCP operations. The loader adds a tabular envelope without
|
|
76
|
+
changing the original public response. `record_json` retains the original record;
|
|
77
|
+
`response_json` retains the complete response, including pagination and metadata.
|
|
78
|
+
An empty response produces one envelope with `record_json = 'null'`, so absence of
|
|
79
|
+
data and the server's metadata remain inspectable. It is not an observation.
|
|
80
|
+
|
|
81
|
+
Columns are `operation`, `source_url`, `record_id`, `currency`, `indicator`,
|
|
82
|
+
`date`, `announcement_datetime`, `value`, `value_text`, `record_json`,
|
|
83
|
+
`response_json`, and the loader's `content_hash`. Only `value` is numeric.
|
|
84
|
+
Dates and announcement times remain source strings. No publication time is
|
|
85
|
+
inferred, and no missing value is filled. `value_text` and JSON preserve source
|
|
86
|
+
numbers supplied as strings. JSON numeric tokens follow the public Python client
|
|
87
|
+
parser, so original lexical precision is not guaranteed.
|
|
88
|
+
|
|
89
|
+
Select operations with the shared catalogue rather than constructing URLs:
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
from fxmacrodata_platform_core import operations, schemas
|
|
93
|
+
|
|
94
|
+
available = operations()
|
|
95
|
+
input_schemas = schemas()
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
`load_operation` accepts `api_key_env`, `complete_history`, `max_pages`, and
|
|
99
|
+
`client_factory` for the public source reader. `max_pages` bounds history
|
|
100
|
+
pagination; an incomplete read raises an error before a destination load starts.
|
|
101
|
+
`batch_size` bounds destination blocks to 1–10,000 rows. `max_batch_bytes` also
|
|
102
|
+
bounds each block using a conservative JSON byte estimate (16 MiB by default,
|
|
103
|
+
128 MiB maximum). An oversized individual envelope is rejected before destination
|
|
104
|
+
changes; narrow the source date range to reduce the response size. The source response is
|
|
105
|
+
materialized before loading; choose a source date range suitable for memory.
|
|
106
|
+
Read the installed operation's schema for required arguments and date filters.
|
|
107
|
+
|
|
108
|
+
Supply source credentials through your runtime secret store, exposing only the
|
|
109
|
+
environment variable name to the adapter. Destination authentication belongs to
|
|
110
|
+
the official SDK connection you pass to the loader. The adapter neither saves nor
|
|
111
|
+
prints credentials and never constructs destination credentials for you.
|
|
112
|
+
|
|
113
|
+
The package licence covers this adapter code. FXMacroData data access and onward
|
|
114
|
+
redistribution remain subject to your subscription and applicable licence. Loading
|
|
115
|
+
data into a warehouse does not grant permission to publish or resell it.
|
|
116
|
+
|
|
117
|
+
## Retries and querying
|
|
118
|
+
|
|
119
|
+
The loader creates the destination and temporary table, then fills every bounded
|
|
120
|
+
block before beginning the destination DML transaction. This also keeps any
|
|
121
|
+
temporary binding-stage DDL issued by the connector outside the transaction.
|
|
122
|
+
It then begins, merges on operation, record ID and content hash, and commits.
|
|
123
|
+
A staging failure leaves destination rows untouched. A merge or commit failure
|
|
124
|
+
triggers rollback. If rollback itself fails, the loader avoids cleanup DDL that
|
|
125
|
+
could commit unresolved work.
|
|
126
|
+
|
|
127
|
+
Repeated exact envelopes are skipped; changed source records or metadata remain
|
|
128
|
+
distinct. Use one writer per destination table. Concurrent merges into standard
|
|
129
|
+
Snowflake tables are not an exactly-once guarantee. The loader never retries a
|
|
130
|
+
write automatically. After connection loss, inspect the transaction outcome
|
|
131
|
+
before retrying. `LoadResult.rows_submitted` counts unique envelopes offered to
|
|
132
|
+
the merge, rather than rows newly inserted.
|
|
133
|
+
|
|
134
|
+
```sql
|
|
135
|
+
SELECT CURRENCY, INDICATOR, DATE, VALUE,
|
|
136
|
+
PARSE_JSON(RECORD_JSON):publication_time_status::STRING AS PUBLICATION_STATUS
|
|
137
|
+
FROM FXMACRODATA
|
|
138
|
+
WHERE RECORD_JSON != 'null';
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
[event_study.sql](examples/event_study.sql) joins source announcement times to
|
|
142
|
+
your licensed market-price table. Inspect source publication metadata separately
|
|
143
|
+
for point-in-time backtesting. [sharing.sql](examples/sharing.sql) shows a
|
|
144
|
+
customer-controlled secure share; sharing data requires the applicable rights
|
|
145
|
+
and recipient approval. This package does not create a Marketplace listing.
|
|
146
|
+
|
|
147
|
+
## Test
|
|
148
|
+
|
|
149
|
+
```sh
|
|
150
|
+
python -m pytest tests -q
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
Official SDK references:
|
|
154
|
+
[batch binding and transactions](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-example),
|
|
155
|
+
[connector API](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-api).
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
MANIFEST.in
|
|
3
|
+
PUBLIC_FILES.json
|
|
4
|
+
README.md
|
|
5
|
+
pyproject.toml
|
|
6
|
+
examples/event_study.sql
|
|
7
|
+
examples/sharing.sql
|
|
8
|
+
src/fxmacrodata_snowflake/__init__.py
|
|
9
|
+
src/snowflake_fxmacrodata.egg-info/PKG-INFO
|
|
10
|
+
src/snowflake_fxmacrodata.egg-info/SOURCES.txt
|
|
11
|
+
src/snowflake_fxmacrodata.egg-info/dependency_links.txt
|
|
12
|
+
src/snowflake_fxmacrodata.egg-info/requires.txt
|
|
13
|
+
src/snowflake_fxmacrodata.egg-info/top_level.txt
|
|
14
|
+
tests/test_loader.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
fxmacrodata_snowflake
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from fxmacrodata_platform_core import Batch
|
|
6
|
+
import fxmacrodata_snowflake as adapter
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Cursor:
|
|
10
|
+
def __init__(self, fail=None, rollback_fail=False):
|
|
11
|
+
self.statements, self.blocks, self.events = [], [], []
|
|
12
|
+
self.fail, self.rollback_fail = fail, rollback_fail
|
|
13
|
+
self.closed = False
|
|
14
|
+
|
|
15
|
+
def execute(self, sql):
|
|
16
|
+
self.statements.append(sql)
|
|
17
|
+
self.events.append(("execute", sql))
|
|
18
|
+
if sql == self.fail or (self.fail == "MERGE" and sql.startswith("MERGE")):
|
|
19
|
+
raise RuntimeError("private destination diagnostic")
|
|
20
|
+
if sql == "ROLLBACK" and self.rollback_fail:
|
|
21
|
+
raise RuntimeError("rollback diagnostic")
|
|
22
|
+
return self
|
|
23
|
+
|
|
24
|
+
def executemany(self, sql, rows):
|
|
25
|
+
self.blocks.append((sql, rows))
|
|
26
|
+
self.events.append(("executemany", sql))
|
|
27
|
+
if self.fail == "INSERT":
|
|
28
|
+
raise RuntimeError("private destination diagnostic")
|
|
29
|
+
return self
|
|
30
|
+
|
|
31
|
+
def close(self):
|
|
32
|
+
self.closed = True
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class Connection:
|
|
36
|
+
def __init__(self, **kw):
|
|
37
|
+
self.recording = Cursor(**kw)
|
|
38
|
+
|
|
39
|
+
def cursor(self):
|
|
40
|
+
return self.recording
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def batch(count=3, metadata=None):
|
|
44
|
+
return Batch("indicator_history", {
|
|
45
|
+
"data": [{"currency": "USD", "indicator": "policy_rate", "value": str(i),
|
|
46
|
+
"date": "2026-10-01", "publication_time_status": "unknown"}
|
|
47
|
+
for i in range(count)],
|
|
48
|
+
"metadata": metadata or {"revision": 1},
|
|
49
|
+
})
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def test_bounded_binding_and_transaction_order():
|
|
53
|
+
source, connection = batch(), Connection()
|
|
54
|
+
result = adapter.load(source, connection, "research.macro.fxmacrodata", batch_size=2, paramstyle="qmark")
|
|
55
|
+
cursor = connection.recording
|
|
56
|
+
assert result.rows_submitted == 3 and result.batches == 2
|
|
57
|
+
assert result.table == '"RESEARCH"."MACRO"."FXMACRODATA"'
|
|
58
|
+
assert cursor.statements[0].startswith("CREATE TABLE IF NOT EXISTS")
|
|
59
|
+
assert cursor.statements[1].startswith("CREATE TEMPORARY TABLE")
|
|
60
|
+
assert cursor.statements[2] == "BEGIN"
|
|
61
|
+
assert cursor.statements[3].startswith("MERGE")
|
|
62
|
+
assert cursor.statements[4] == "COMMIT"
|
|
63
|
+
assert cursor.statements[5].startswith("DROP TABLE") and cursor.closed
|
|
64
|
+
assert [len(rows) for _, rows in cursor.blocks] == [2, 1]
|
|
65
|
+
assert [kind for kind, _ in cursor.events] == [
|
|
66
|
+
"execute", "execute", "executemany", "executemany",
|
|
67
|
+
"execute", "execute", "execute", "execute",
|
|
68
|
+
]
|
|
69
|
+
assert cursor.events[4] == ("execute", "BEGIN")
|
|
70
|
+
assert cursor.events[5][1].startswith("MERGE")
|
|
71
|
+
assert all('INSERT INTO "FXMD_STAGE_' in sql
|
|
72
|
+
for kind, sql in cursor.events if kind == "executemany")
|
|
73
|
+
assert "VALUES (?, ?, ?" in cursor.blocks[0][0]
|
|
74
|
+
row = dict(zip(adapter.COLUMNS, cursor.blocks[0][1][0]))
|
|
75
|
+
assert row["date"] == "2026-10-01" and row["announcement_datetime"] is None
|
|
76
|
+
assert row["value"] == 0.0 and row["value_text"] == "0"
|
|
77
|
+
assert json.loads(row["response_json"]) == source.payload
|
|
78
|
+
assert json.loads(row["record_json"])["publication_time_status"] == "unknown"
|
|
79
|
+
assert 'target."CONTENT_HASH" = source."CONTENT_HASH"' in cursor.statements[3]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def test_empty_response_is_retained():
|
|
83
|
+
connection, source = Connection(), batch(0)
|
|
84
|
+
result = adapter.load(source, connection, "FXMACRODATA")
|
|
85
|
+
row = dict(zip(adapter.COLUMNS, connection.recording.blocks[0][1][0]))
|
|
86
|
+
assert result.rows_submitted == 1 and row["record_json"] == "null"
|
|
87
|
+
assert row["date"] is None and row["value"] is None
|
|
88
|
+
assert json.loads(row["response_json"]) == source.payload
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@pytest.mark.parametrize("failure", ["MERGE", "COMMIT"])
|
|
92
|
+
def test_dml_failure_rolls_back_without_diagnostic_leak(failure):
|
|
93
|
+
connection = Connection(fail=failure)
|
|
94
|
+
with pytest.raises(adapter.LoadError) as error:
|
|
95
|
+
adapter.load(batch(), connection, "FXMACRODATA")
|
|
96
|
+
statements = connection.recording.statements
|
|
97
|
+
assert "ROLLBACK" in statements
|
|
98
|
+
assert statements[-1].startswith("DROP TABLE")
|
|
99
|
+
assert "private destination diagnostic" not in str(error.value)
|
|
100
|
+
assert connection.recording.closed
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def test_staging_failure_never_begins_destination_transaction():
|
|
104
|
+
connection = Connection(fail="INSERT")
|
|
105
|
+
with pytest.raises(adapter.LoadError) as error:
|
|
106
|
+
adapter.load(batch(), connection, "FXMACRODATA", paramstyle="qmark")
|
|
107
|
+
cursor = connection.recording
|
|
108
|
+
assert not any(sql in {"BEGIN", "COMMIT", "ROLLBACK"} or sql.startswith("MERGE")
|
|
109
|
+
for sql in cursor.statements)
|
|
110
|
+
assert cursor.statements[-1].startswith("DROP TABLE")
|
|
111
|
+
assert "private destination diagnostic" not in str(error.value)
|
|
112
|
+
assert cursor.closed
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def test_failed_rollback_never_runs_cleanup_ddl():
|
|
116
|
+
connection = Connection(fail="MERGE", rollback_fail=True)
|
|
117
|
+
with pytest.raises(adapter.LoadError):
|
|
118
|
+
adapter.load(batch(), connection, "FXMACRODATA")
|
|
119
|
+
assert connection.recording.statements[-1] == "ROLLBACK"
|
|
120
|
+
assert not any(s.startswith("DROP") for s in connection.recording.statements)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def test_duplicate_input_envelopes_are_deduplicated_before_merge():
|
|
124
|
+
class DuplicateBatch:
|
|
125
|
+
def rows(self):
|
|
126
|
+
row = batch(1).rows()[0]
|
|
127
|
+
return [row, row]
|
|
128
|
+
connection = Connection()
|
|
129
|
+
result = adapter.load(DuplicateBatch(), connection, "FXMACRODATA")
|
|
130
|
+
assert result.rows_submitted == 1
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def test_hashes_are_stable_and_changed_metadata_stays_distinct():
|
|
134
|
+
one, retry, changed = Connection(), Connection(), Connection()
|
|
135
|
+
adapter.load(batch(1), one, "FXMACRODATA")
|
|
136
|
+
adapter.load(batch(1), retry, "FXMACRODATA")
|
|
137
|
+
adapter.load(batch(1, {"revision": 2}), changed, "FXMACRODATA")
|
|
138
|
+
assert one.recording.blocks[0][1] == retry.recording.blocks[0][1]
|
|
139
|
+
row, newer = one.recording.blocks[0][1][0], changed.recording.blocks[0][1][0]
|
|
140
|
+
assert row[adapter.COLUMNS.index("record_id")] == newer[adapter.COLUMNS.index("record_id")]
|
|
141
|
+
assert row[-1] != newer[-1]
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@pytest.mark.parametrize("style,placeholder", [("qmark", "?"), ("numeric", ":1"), ("format", "%s"), ("pyformat", "%s")])
|
|
145
|
+
def test_binding_style_is_connection_local(style, placeholder):
|
|
146
|
+
connection = Connection()
|
|
147
|
+
adapter.load(batch(1), connection, "FXMACRODATA", paramstyle=style, create_table=False)
|
|
148
|
+
assert f"VALUES ({placeholder}, " in connection.recording.blocks[0][0]
|
|
149
|
+
assert not connection.recording.statements[0].startswith("CREATE TABLE IF NOT EXISTS")
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
@pytest.mark.parametrize("table", ["a;b", "a.b.c.d", '"CaseSensitive"', "a b", ""])
|
|
153
|
+
def test_identifiers_rejected_before_connection_use(table):
|
|
154
|
+
connection = Connection()
|
|
155
|
+
with pytest.raises(ValueError):
|
|
156
|
+
adapter.load(batch(), connection, table)
|
|
157
|
+
assert connection.recording.statements == []
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
@pytest.mark.parametrize("size", [0, -1, 10001, True, 2.5])
|
|
161
|
+
def test_batch_size_bounds(size):
|
|
162
|
+
with pytest.raises(ValueError):
|
|
163
|
+
adapter.load(batch(), Connection(), "FXMACRODATA", batch_size=size)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def test_unsupported_binding_style_is_rejected():
|
|
167
|
+
with pytest.raises(ValueError):
|
|
168
|
+
adapter.load(batch(), Connection(), "FXMACRODATA", paramstyle="raw_sql")
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def test_operation_forwards_only_reader_configuration(monkeypatch):
|
|
172
|
+
calls = []
|
|
173
|
+
monkeypatch.setattr(adapter, "read", lambda *a, **kw: calls.append((a, kw)) or batch(1))
|
|
174
|
+
adapter.load_operation("release_calendar", {"currency": "usd"}, Connection(), "FXMACRODATA",
|
|
175
|
+
api_key_env="FXMD_API_KEY", max_pages=7, paramstyle="numeric")
|
|
176
|
+
assert calls[0][0] == ("release_calendar", {"currency": "usd"})
|
|
177
|
+
assert calls[0][1]["api_key_env"] == "FXMD_API_KEY" and calls[0][1]["max_pages"] == 7
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def test_byte_limit_splits_staging_blocks_and_rejects_oversized_envelopes_before_ddl():
|
|
181
|
+
source, connection = batch(), Connection()
|
|
182
|
+
budget = max(len(json.dumps(r, ensure_ascii=True).encode()) + 2 for r in adapter._rows(source))
|
|
183
|
+
result = adapter.load(source, connection, "FXMACRODATA", batch_size=100, max_batch_bytes=budget)
|
|
184
|
+
assert result.batches == 3
|
|
185
|
+
untouched = Connection()
|
|
186
|
+
with pytest.raises(ValueError, match="narrow the source date range"):
|
|
187
|
+
adapter.load(source, untouched, "FXMACRODATA", max_batch_bytes=1)
|
|
188
|
+
assert untouched.recording.statements == [] and untouched.recording.blocks == []
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def test_invalid_loader_configuration_never_reads_source(monkeypatch):
|
|
192
|
+
def unexpected(*args, **kwargs):
|
|
193
|
+
pytest.fail("source must not be called for invalid loader configuration")
|
|
194
|
+
monkeypatch.setattr(adapter, "read", unexpected)
|
|
195
|
+
for options in ({"batch_size": 0}, {"max_batch_bytes": 0}, {"paramstyle": "unsafe"}):
|
|
196
|
+
with pytest.raises(ValueError):
|
|
197
|
+
adapter.load_operation("release_calendar", {"currency": "usd"}, Connection(), "FXMACRODATA", **options)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def test_envelopes_bind_with_official_sdk_without_network():
|
|
201
|
+
from snowflake.connector.converter import SnowflakeConverter
|
|
202
|
+
|
|
203
|
+
converter = SnowflakeConverter()
|
|
204
|
+
connection = Connection()
|
|
205
|
+
adapter.load(batch(), connection, "FXMACRODATA", paramstyle="qmark")
|
|
206
|
+
for _, rows in connection.recording.blocks:
|
|
207
|
+
for row in rows:
|
|
208
|
+
for value in row:
|
|
209
|
+
# Both server-side (qmark/numeric) and client-side bindings
|
|
210
|
+
# must accept the actual values submitted by the loader.
|
|
211
|
+
kind = converter.snowflake_type(value)
|
|
212
|
+
assert kind is not None
|
|
213
|
+
converted = converter.to_snowflake_bindings(kind, value)
|
|
214
|
+
assert converted is None if value is None else isinstance(converted, str)
|
|
215
|
+
converter.to_snowflake(value)
|