heimdall-mimird 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- heimdall_mimird-0.2.0/.github/workflows/ci.yml +30 -0
- heimdall_mimird-0.2.0/.gitignore +16 -0
- heimdall_mimird-0.2.0/.python-version +1 -0
- heimdall_mimird-0.2.0/CONTRIBUTING.md +42 -0
- heimdall_mimird-0.2.0/PKG-INFO +72 -0
- heimdall_mimird-0.2.0/README.md +52 -0
- heimdall_mimird-0.2.0/ROADMAP.md +61 -0
- heimdall_mimird-0.2.0/docs/writing-a-provider.md +125 -0
- heimdall_mimird-0.2.0/pyproject.toml +59 -0
- heimdall_mimird-0.2.0/sandbox/README.md +46 -0
- heimdall_mimird-0.2.0/sandbox/__init__.py +1 -0
- heimdall_mimird-0.2.0/sandbox/__main__.py +7 -0
- heimdall_mimird-0.2.0/sandbox/_helpers.py +93 -0
- heimdall_mimird-0.2.0/sandbox/examples/01_fred_to_terminal.py +19 -0
- heimdall_mimird-0.2.0/sandbox/examples/02_yfinance_to_terminal.py +18 -0
- heimdall_mimird-0.2.0/sandbox/examples/03_custom_provider.py +57 -0
- heimdall_mimird-0.2.0/sandbox/examples/04_export_to_files.py +28 -0
- heimdall_mimird-0.2.0/sandbox/examples/05_provider_with_config.py +56 -0
- heimdall_mimird-0.2.0/sandbox/play.py +66 -0
- heimdall_mimird-0.2.0/sandbox/scratch/.gitkeep +1 -0
- heimdall_mimird-0.2.0/src/heimdall/__init__.py +104 -0
- heimdall_mimird-0.2.0/src/heimdall/_logging.py +23 -0
- heimdall_mimird-0.2.0/src/heimdall/_retry.py +61 -0
- heimdall_mimird-0.2.0/src/heimdall/_time.py +10 -0
- heimdall_mimird-0.2.0/src/heimdall/contracts.py +145 -0
- heimdall_mimird-0.2.0/src/heimdall/errors.py +35 -0
- heimdall_mimird-0.2.0/src/heimdall/provider.py +81 -0
- heimdall_mimird-0.2.0/src/heimdall/providers/__init__.py +9 -0
- heimdall_mimird-0.2.0/src/heimdall/providers/fred/__init__.py +16 -0
- heimdall_mimird-0.2.0/src/heimdall/providers/fred/provider.py +103 -0
- heimdall_mimird-0.2.0/src/heimdall/providers/yfinance/__init__.py +16 -0
- heimdall_mimird-0.2.0/src/heimdall/providers/yfinance/provider.py +105 -0
- heimdall_mimird-0.2.0/src/heimdall/registry.py +112 -0
- heimdall_mimird-0.2.0/src/heimdall/schemas.py +50 -0
- heimdall_mimird-0.2.0/src/heimdall/testing.py +107 -0
- heimdall_mimird-0.2.0/tests/conftest.py +47 -0
- heimdall_mimird-0.2.0/tests/providers/test_fred.py +80 -0
- heimdall_mimird-0.2.0/tests/providers/test_yfinance.py +90 -0
- heimdall_mimird-0.2.0/tests/test_contracts.py +97 -0
- heimdall_mimird-0.2.0/tests/test_registry.py +55 -0
- heimdall_mimird-0.2.0/uv.lock +1735 -0
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
check:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
strategy:
|
|
12
|
+
fail-fast: false
|
|
13
|
+
matrix:
|
|
14
|
+
python-version: ["3.11", "3.12"]
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
- name: Install uv
|
|
18
|
+
uses: astral-sh/setup-uv@v5
|
|
19
|
+
with:
|
|
20
|
+
python-version: ${{ matrix.python-version }}
|
|
21
|
+
- name: Sync
|
|
22
|
+
run: uv sync --extra all --group dev
|
|
23
|
+
- name: Lint
|
|
24
|
+
run: uv run ruff check .
|
|
25
|
+
- name: Format
|
|
26
|
+
run: uv run ruff format --check .
|
|
27
|
+
- name: Types
|
|
28
|
+
run: uv run mypy src/heimdall
|
|
29
|
+
- name: Test
|
|
30
|
+
run: uv run pytest -m "not network" -q
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
*.egg-info
|
|
2
|
+
dist
|
|
3
|
+
.venv
|
|
4
|
+
__pycache__/
|
|
5
|
+
*.pyc
|
|
6
|
+
.pytest_cache/
|
|
7
|
+
.mypy_cache/
|
|
8
|
+
.ruff_cache/
|
|
9
|
+
.coverage
|
|
10
|
+
htmlcov/
|
|
11
|
+
|
|
12
|
+
# dev sandbox - keep the folder, ignore experiments and local data
|
|
13
|
+
sandbox/scratch/*
|
|
14
|
+
!sandbox/scratch/.gitkeep
|
|
15
|
+
*.db
|
|
16
|
+
*.duckdb
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.11
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
## Dev setup
|
|
4
|
+
|
|
5
|
+
```bash
|
|
6
|
+
uv sync --extra all --group dev
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
## Checks (what CI runs)
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
uv run ruff check .
|
|
13
|
+
uv run ruff format --check .
|
|
14
|
+
uv run mypy src/heimdall
|
|
15
|
+
uv run pytest -m "not network" # offline suite
|
|
16
|
+
uv run pytest -m network # hits real FRED / Yahoo; run before releases
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## Adding a provider
|
|
20
|
+
|
|
21
|
+
First-party providers live under `src/heimdall/providers/<name>/` and are
|
|
22
|
+
registered by a module-level `_register(registry)` hook (see the `fred` and
|
|
23
|
+
`yfinance` packages). Their client library goes in a matching
|
|
24
|
+
`[project.optional-dependencies]` extra so `import heimdall` still works without
|
|
25
|
+
it.
|
|
26
|
+
|
|
27
|
+
Every provider must:
|
|
28
|
+
|
|
29
|
+
1. Subclass `heimdall.Provider`, set a stable `id` and a `Capabilities`.
|
|
30
|
+
2. Return a `FetchResult` whose `frame` passes `schema.validate()` - reuse
|
|
31
|
+
`heimdall.schemas.OHLCV_BARS` / `OBSERVATIONS` or define your own `SchemaSpec`.
|
|
32
|
+
3. Raise `heimdall.errors.*` (`RequestError`, `UpstreamError`, `ConfigError`),
|
|
33
|
+
never bare `ValueError` / `KeyError`.
|
|
34
|
+
4. Pass the conformance suite. Add a test module that subclasses
|
|
35
|
+
`heimdall.testing.ProviderContractTests` plus a `@pytest.mark.network`
|
|
36
|
+
smoke test.
|
|
37
|
+
|
|
38
|
+
`heimdall.testing` needs pytest. It is not part of the base install - an
|
|
39
|
+
out-of-tree provider package gets it with `pip install heimdall-mimird[testing]`.
|
|
40
|
+
Heimdall's own `dev` group already includes pytest.
|
|
41
|
+
|
|
42
|
+
See `docs/writing-a-provider.md` for a full example.
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: heimdall-mimird
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: A data-provider SDK
|
|
5
|
+
Requires-Python: >=3.11
|
|
6
|
+
Requires-Dist: polars>=1.44.1
|
|
7
|
+
Requires-Dist: ramonavocado-logger>=0.1.3
|
|
8
|
+
Provides-Extra: all
|
|
9
|
+
Requires-Dist: httpx>=0.27; extra == 'all'
|
|
10
|
+
Requires-Dist: pyarrow>=15; extra == 'all'
|
|
11
|
+
Requires-Dist: yfinance>=0.2.40; extra == 'all'
|
|
12
|
+
Provides-Extra: fred
|
|
13
|
+
Requires-Dist: httpx>=0.27; extra == 'fred'
|
|
14
|
+
Provides-Extra: testing
|
|
15
|
+
Requires-Dist: pytest>=8.2; extra == 'testing'
|
|
16
|
+
Provides-Extra: yfinance
|
|
17
|
+
Requires-Dist: pyarrow>=15; extra == 'yfinance'
|
|
18
|
+
Requires-Dist: yfinance>=0.2.40; extra == 'yfinance'
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
# Heimdall
|
|
22
|
+
|
|
23
|
+
A host-agnostic **data-provider SDK**. Finance data first (Yahoo Finance OHLCV,
|
|
24
|
+
FRED economic series), built so that any data source - yours or the community's -
|
|
25
|
+
can plug in behind one small contract.
|
|
26
|
+
|
|
27
|
+
A provider takes a `FetchRequest` and returns a `FetchResult`: a
|
|
28
|
+
[Polars](https://pola.rs) frame, a `SchemaSpec` that describes and validates it,
|
|
29
|
+
and some metadata. That's the whole surface. Nothing in the core is
|
|
30
|
+
finance-specific; OHLCV bars and economic observations are just two predefined
|
|
31
|
+
schemas.
|
|
32
|
+
|
|
33
|
+
## Install
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
pip install heimdall-mimird[fred,yfinance] # or: heimdall-mimird[all]
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
The core depends only on `polars` (plus a small logging shim). Each provider
|
|
40
|
+
pulls its own client library through an extra, so you install only what you use.
|
|
41
|
+
|
|
42
|
+
## Quick start
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
import heimdall
|
|
46
|
+
|
|
47
|
+
# string shorthand -> FetchRequest(resource="DGS10")
|
|
48
|
+
result = heimdall.fetch("fred", "DGS10")
|
|
49
|
+
result.schema.validate(result.frame) # raises SchemaError if it doesn't conform
|
|
50
|
+
print(result.frame.head())
|
|
51
|
+
|
|
52
|
+
bars = heimdall.fetch("yfinance", "AAPL", interval="1d")
|
|
53
|
+
print(bars.frame.tail())
|
|
54
|
+
print(bars.metadata) # {"ticker": "AAPL", "interval": "1d", "rows": ...}
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
`heimdall.list_providers()` shows what's registered. Providers are registered
|
|
58
|
+
automatically when their extra is installed; register your own with
|
|
59
|
+
`heimdall.register(MyProvider)`.
|
|
60
|
+
|
|
61
|
+
## Writing a provider
|
|
62
|
+
|
|
63
|
+
Subclass `heimdall.Provider`, set `id` + `capabilities`, implement `fetch`. See
|
|
64
|
+
[docs/writing-a-provider.md](docs/writing-a-provider.md) for a full worked
|
|
65
|
+
example, and run `heimdall.testing.assert_provider_conformance(...)` to check it
|
|
66
|
+
against the contract.
|
|
67
|
+
|
|
68
|
+
## Status
|
|
69
|
+
|
|
70
|
+
Early. The provider contract and the two bundled providers are usable today. The
|
|
71
|
+
ingestion/storage layer, entry-point plugin discovery, and retry/rate-limit
|
|
72
|
+
middleware are on the [roadmap](ROADMAP.md).
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# Heimdall
|
|
2
|
+
|
|
3
|
+
A host-agnostic **data-provider SDK**. Finance data first (Yahoo Finance OHLCV,
|
|
4
|
+
FRED economic series), built so that any data source - yours or the community's -
|
|
5
|
+
can plug in behind one small contract.
|
|
6
|
+
|
|
7
|
+
A provider takes a `FetchRequest` and returns a `FetchResult`: a
|
|
8
|
+
[Polars](https://pola.rs) frame, a `SchemaSpec` that describes and validates it,
|
|
9
|
+
and some metadata. That's the whole surface. Nothing in the core is
|
|
10
|
+
finance-specific; OHLCV bars and economic observations are just two predefined
|
|
11
|
+
schemas.
|
|
12
|
+
|
|
13
|
+
## Install
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
pip install heimdall-mimird[fred,yfinance] # or: heimdall-mimird[all]
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
The core depends only on `polars` (plus a small logging shim). Each provider
|
|
20
|
+
pulls its own client library through an extra, so you install only what you use.
|
|
21
|
+
|
|
22
|
+
## Quick start
|
|
23
|
+
|
|
24
|
+
```python
|
|
25
|
+
import heimdall
|
|
26
|
+
|
|
27
|
+
# string shorthand -> FetchRequest(resource="DGS10")
|
|
28
|
+
result = heimdall.fetch("fred", "DGS10")
|
|
29
|
+
result.schema.validate(result.frame) # raises SchemaError if it doesn't conform
|
|
30
|
+
print(result.frame.head())
|
|
31
|
+
|
|
32
|
+
bars = heimdall.fetch("yfinance", "AAPL", interval="1d")
|
|
33
|
+
print(bars.frame.tail())
|
|
34
|
+
print(bars.metadata) # {"ticker": "AAPL", "interval": "1d", "rows": ...}
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
`heimdall.list_providers()` shows what's registered. Providers are registered
|
|
38
|
+
automatically when their extra is installed; register your own with
|
|
39
|
+
`heimdall.register(MyProvider)`.
|
|
40
|
+
|
|
41
|
+
## Writing a provider
|
|
42
|
+
|
|
43
|
+
Subclass `heimdall.Provider`, set `id` + `capabilities`, implement `fetch`. See
|
|
44
|
+
[docs/writing-a-provider.md](docs/writing-a-provider.md) for a full worked
|
|
45
|
+
example, and run `heimdall.testing.assert_provider_conformance(...)` to check it
|
|
46
|
+
against the contract.
|
|
47
|
+
|
|
48
|
+
## Status
|
|
49
|
+
|
|
50
|
+
Early. The provider contract and the two bundled providers are usable today. The
|
|
51
|
+
ingestion/storage layer, entry-point plugin discovery, and retry/rate-limit
|
|
52
|
+
middleware are on the [roadmap](ROADMAP.md).
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# Roadmap
|
|
2
|
+
|
|
3
|
+
Legend: `[x]` shipped · `[~]` partially built · `[ ]` planned
|
|
4
|
+
|
|
5
|
+
## Shipped
|
|
6
|
+
|
|
7
|
+
- [x] **Core SDK** - `FetchRequest` / `FetchResult` / `SchemaSpec` contract,
|
|
8
|
+
`Provider` base + `Capabilities`, `ProviderRegistry` with explicit
|
|
9
|
+
registration, error hierarchy, `heimdall.fetch(...)`.
|
|
10
|
+
- [x] **Bundled providers** - `fred` (public `fredgraph.csv` endpoint, no key)
|
|
11
|
+
and `yfinance` (OHLCV via the `yfinance` library), each behind an extra.
|
|
12
|
+
- [x] **Provider conformance kit** - `heimdall.testing.assert_provider_conformance`
|
|
13
|
+
+ `ProviderContractTests`, behind the `testing` extra. This is the "testing
|
|
14
|
+
harness for provider authors" - third parties prove their provider fits the
|
|
15
|
+
spec by running it.
|
|
16
|
+
- [x] **Tooling** - ruff, mypy, pytest (with a `network` marker), GitHub Actions CI.
|
|
17
|
+
- [x] **Dev sandbox** - `sandbox/` with a `python -m sandbox` runner and example
|
|
18
|
+
scripts; results print to the terminal and can be dumped to parquet / CSV.
|
|
19
|
+
|
|
20
|
+
## Planned (next)
|
|
21
|
+
|
|
22
|
+
- [ ] **Conformance kit polish** - recorded-fixture helpers so third parties can
|
|
23
|
+
write offline contract tests without hand-rolling mocks.
|
|
24
|
+
- [ ] **Docs** - expand `docs/writing-a-provider.md` with a schema-authoring
|
|
25
|
+
section.
|
|
26
|
+
|
|
27
|
+
## Candidate ideas (evaluated, not yet scheduled)
|
|
28
|
+
|
|
29
|
+
- [~] **Entry-point plugin discovery** - `ProviderRegistry.load_entry_points()`
|
|
30
|
+
already works; wire a documented `heimdall.providers` entry-point group and add
|
|
31
|
+
a `@heimdall.provider` decorator so `pip install heimdall-stripe` auto-registers
|
|
32
|
+
with the core package. Single biggest driver of ecosystem growth (cf. dlt,
|
|
33
|
+
pytest, SQLAlchemy dialects).
|
|
34
|
+
- [~] **Config & credentials layer** - already have `Provider(config=...)`,
|
|
35
|
+
`Capabilities.required_config`, and `config_from_env()`. Add `.env` loading and
|
|
36
|
+
a pluggable secret-source hook (env / file / secret manager) so provider
|
|
37
|
+
authors don't reinvent it.
|
|
38
|
+
- [~] **Finer error taxonomy** - split `UpstreamError` into `ProviderAuthError` /
|
|
39
|
+
`ProviderRateLimitError` / `ProviderTransientError` so orchestration code can
|
|
40
|
+
react generically: retry vs fail vs backoff.
|
|
41
|
+
- [ ] **Rate limiting / backpressure** - a shared limiter providers opt into;
|
|
42
|
+
make `Capabilities.rate_limit_per_min` enforced rather than advisory. Part of a
|
|
43
|
+
middleware stack that wraps `fetch()` host-side (retry + rate-limit + cache).
|
|
44
|
+
- [ ] **State / checkpoint store** - a pluggable interface (file / DB / redis) for
|
|
45
|
+
"last synced cursor", so incremental sync is usable in production. Needs a
|
|
46
|
+
cursor concept in the contract (`FetchRequest.cursor` + a next-cursor on
|
|
47
|
+
`FetchResult`). Larger design item.
|
|
48
|
+
- [ ] **Observability hooks** - a logging/metrics callback protocol (records
|
|
49
|
+
fetched, latency, errors) that plugs into whatever monitoring the host already
|
|
50
|
+
runs. Overlaps the middleware work.
|
|
51
|
+
- [ ] **`create-provider` scaffolder** - a `heimdall new-provider <name>` command
|
|
52
|
+
or a cookiecutter template that emits a provider package skeleton plus its
|
|
53
|
+
conformance test. Directly lowers the bar for third-party contributors.
|
|
54
|
+
- [ ] **`heimdall.ingest`** - persist `FetchResult`s: parquet tiers + a JSON
|
|
55
|
+
manifest, plus the transform set (`diff_1`, `pct_change_1`, `sma_20`, `sma_50`).
|
|
56
|
+
Starting point is parked at `docs/legacy/essential.py`.
|
|
57
|
+
- [ ] **FRED official API** - swap/augment the CSV endpoint with
|
|
58
|
+
`api.stlouisfed.org` (needs `FRED_API_KEY` -> `Capabilities.required_config`),
|
|
59
|
+
giving vintages, metadata, and pagination.
|
|
60
|
+
- [ ] **Packaging split** - break `heimdall-core` and each provider into separate
|
|
61
|
+
distributions, if the boundaries hold up.
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
# Writing a provider
|
|
2
|
+
|
|
3
|
+
A provider is one class: it turns a `FetchRequest` into a `FetchResult`. You can
|
|
4
|
+
keep it inside your own project or publish it - Heimdall imposes nothing beyond
|
|
5
|
+
the contract.
|
|
6
|
+
|
|
7
|
+
## The contract
|
|
8
|
+
|
|
9
|
+
```python
|
|
10
|
+
FetchRequest(resource, interval=None, start=None, end=None, params={})
|
|
11
|
+
FetchResult(frame, schema, provider_id, request, retrieved_at, metadata={})
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
- **`resource`** is provider-specific: a ticker, a series id, an endpoint key.
|
|
15
|
+
- **`frame`** is a `polars.DataFrame`.
|
|
16
|
+
- **`schema`** is a `SchemaSpec` that must actually validate `frame`. Reuse
|
|
17
|
+
`heimdall.schemas.OHLCV_BARS` / `OBSERVATIONS`, or define your own.
|
|
18
|
+
|
|
19
|
+
## A minimal example
|
|
20
|
+
|
|
21
|
+
A provider that pulls a CSV with `date,value` columns from a URL:
|
|
22
|
+
|
|
23
|
+
```python
|
|
24
|
+
from datetime import datetime
|
|
25
|
+
from io import StringIO
|
|
26
|
+
|
|
27
|
+
import httpx
|
|
28
|
+
import polars as pl
|
|
29
|
+
|
|
30
|
+
import heimdall
|
|
31
|
+
from heimdall import Capabilities, FetchRequest, FetchResult, Provider
|
|
32
|
+
from heimdall.contracts import ColumnSpec, SchemaSpec
|
|
33
|
+
from heimdall.errors import RequestError, UpstreamError
|
|
34
|
+
|
|
35
|
+
CSV_SERIES = SchemaSpec(
|
|
36
|
+
name="csv.series",
|
|
37
|
+
columns=(
|
|
38
|
+
ColumnSpec("timestamp", pl.Datetime, nullable=False),
|
|
39
|
+
ColumnSpec("value", pl.Float64, nullable=False),
|
|
40
|
+
),
|
|
41
|
+
time_column="timestamp",
|
|
42
|
+
primary_key=("timestamp",),
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class CsvUrlProvider(Provider):
|
|
47
|
+
id = "csv-url"
|
|
48
|
+
capabilities = Capabilities(data_kinds=(CSV_SERIES.name,))
|
|
49
|
+
|
|
50
|
+
def fetch(self, request: FetchRequest) -> FetchResult:
|
|
51
|
+
url = request.params.get("url") or request.resource
|
|
52
|
+
try:
|
|
53
|
+
resp = httpx.get(url, timeout=30, follow_redirects=True)
|
|
54
|
+
resp.raise_for_status()
|
|
55
|
+
except httpx.HTTPError as exc:
|
|
56
|
+
raise UpstreamError(f"could not fetch {url}: {exc}") from exc
|
|
57
|
+
|
|
58
|
+
frame = (
|
|
59
|
+
pl.read_csv(StringIO(resp.text))
|
|
60
|
+
.rename({"date": "timestamp", "value": "value"})
|
|
61
|
+
.with_columns(
|
|
62
|
+
pl.col("timestamp").str.strptime(pl.Datetime, strict=False),
|
|
63
|
+
pl.col("value").cast(pl.Float64, strict=False),
|
|
64
|
+
)
|
|
65
|
+
.drop_nulls()
|
|
66
|
+
.sort("timestamp")
|
|
67
|
+
)
|
|
68
|
+
if frame.is_empty():
|
|
69
|
+
raise RequestError(f"no rows returned from {url}")
|
|
70
|
+
|
|
71
|
+
CSV_SERIES.validate(frame)
|
|
72
|
+
return FetchResult(
|
|
73
|
+
frame=frame,
|
|
74
|
+
schema=CSV_SERIES,
|
|
75
|
+
provider_id=self.id,
|
|
76
|
+
request=request,
|
|
77
|
+
retrieved_at=heimdall.utcnow(),
|
|
78
|
+
)
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## Register and use it
|
|
82
|
+
|
|
83
|
+
```python
|
|
84
|
+
import heimdall
|
|
85
|
+
|
|
86
|
+
heimdall.register(CsvUrlProvider)
|
|
87
|
+
result = heimdall.fetch("csv-url", "https://example.com/rates.csv")
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
For config/secrets, declare `required_config` in `Capabilities` and pass values
|
|
91
|
+
at registration - never read the environment inside the provider:
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
heimdall.register(MyProvider, config=heimdall.config_from_env("HEIMDALL_MYPROVIDER_"))
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## Prove it conforms
|
|
98
|
+
|
|
99
|
+
`heimdall.testing` needs pytest, which the base install does not pull in:
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
pip install heimdall-mimird[testing]
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
from heimdall.testing import assert_provider_conformance
|
|
107
|
+
from heimdall import FetchRequest
|
|
108
|
+
|
|
109
|
+
assert_provider_conformance(
|
|
110
|
+
CsvUrlProvider(), FetchRequest(resource="https://example.com/rates.csv")
|
|
111
|
+
)
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Or in a pytest suite, subclass `heimdall.testing.ProviderContractTests` and
|
|
115
|
+
implement `make_provider()` / `sample_request()`.
|
|
116
|
+
|
|
117
|
+
## Rules the conformance kit enforces
|
|
118
|
+
|
|
119
|
+
- `id` is a non-empty string; `capabilities.data_kinds` is non-empty.
|
|
120
|
+
- `fetch()` returns a `FetchResult` with a non-empty frame that passes
|
|
121
|
+
`schema.validate()`.
|
|
122
|
+
- `result.provider_id == self.id`; `retrieved_at` is timezone-aware.
|
|
123
|
+
- Two identical requests return the same schema and columns.
|
|
124
|
+
- An unsupported `interval` raises `RequestError` (not a bare exception).
|
|
125
|
+
- Missing `required_config` raises `ConfigError` at construction.
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "heimdall-mimird"
|
|
3
|
+
version = "0.2.0"
|
|
4
|
+
description = "A data-provider SDK"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.11"
|
|
7
|
+
dependencies = [
|
|
8
|
+
"polars>=1.44.1",
|
|
9
|
+
"ramonavocado-logger>=0.1.3",
|
|
10
|
+
]
|
|
11
|
+
|
|
12
|
+
[project.optional-dependencies]
|
|
13
|
+
fred = ["httpx>=0.27"]
|
|
14
|
+
yfinance = ["yfinance>=0.2.40", "pyarrow>=15"]
|
|
15
|
+
testing = ["pytest>=8.2"]
|
|
16
|
+
all = ["httpx>=0.27", "yfinance>=0.2.40", "pyarrow>=15"]
|
|
17
|
+
|
|
18
|
+
[dependency-groups]
|
|
19
|
+
# sandbox = ["sqlalchemy>=2"] # add when the dev sandbox needs a real database
|
|
20
|
+
dev = [
|
|
21
|
+
"pytest>=8.2",
|
|
22
|
+
"pytest-cov>=5.0",
|
|
23
|
+
"respx>=0.21",
|
|
24
|
+
"ruff>=0.6",
|
|
25
|
+
"mypy>=1.11",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[build-system]
|
|
29
|
+
requires = ["hatchling"]
|
|
30
|
+
build-backend = "hatchling.build"
|
|
31
|
+
|
|
32
|
+
[tool.hatch.build.targets.wheel]
|
|
33
|
+
packages = ["src/heimdall"]
|
|
34
|
+
|
|
35
|
+
[tool.ruff]
|
|
36
|
+
line-length = 100
|
|
37
|
+
src = ["src", "tests", "sandbox"]
|
|
38
|
+
extend-exclude = ["sandbox/scratch"]
|
|
39
|
+
|
|
40
|
+
[tool.ruff.lint]
|
|
41
|
+
select = ["E", "F", "I", "UP", "B", "SIM"]
|
|
42
|
+
|
|
43
|
+
[tool.mypy]
|
|
44
|
+
python_version = "3.11"
|
|
45
|
+
files = ["src/heimdall"]
|
|
46
|
+
disallow_untyped_defs = true
|
|
47
|
+
warn_unused_ignores = true
|
|
48
|
+
warn_redundant_casts = true
|
|
49
|
+
no_implicit_optional = true
|
|
50
|
+
|
|
51
|
+
[[tool.mypy.overrides]]
|
|
52
|
+
module = ["ramonavocado_logger.*", "yfinance.*"]
|
|
53
|
+
ignore_missing_imports = true
|
|
54
|
+
|
|
55
|
+
[tool.pytest.ini_options]
|
|
56
|
+
testpaths = ["tests"]
|
|
57
|
+
markers = [
|
|
58
|
+
"network: test hits a real external service (deselect with '-m \"not network\"')",
|
|
59
|
+
]
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# sandbox/
|
|
2
|
+
|
|
3
|
+
A dev-only playground for exercising Heimdall providers by hand. **Not shipped** -
|
|
4
|
+
the package build only includes `src/heimdall`.
|
|
5
|
+
|
|
6
|
+
- **Committed** (`play.py`, `_helpers.py`, `examples/`) - reference material, kept
|
|
7
|
+
tidy and lint-clean.
|
|
8
|
+
- **`scratch/`** - git-ignored. Put your own throwaway scripts and any generated
|
|
9
|
+
`.parquet` / `.csv` / `.db` files here.
|
|
10
|
+
|
|
11
|
+
Run everything from the repo root, after `uv sync --extra all --group dev`.
|
|
12
|
+
|
|
13
|
+
## The runner
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
uv run python -m sandbox --list
|
|
17
|
+
uv run python -m sandbox fred DGS10
|
|
18
|
+
uv run python -m sandbox yfinance AAPL --interval 1d --rows 5
|
|
19
|
+
uv run python -m sandbox fred DGS10 --start 2020-01-01 --to-parquet --to-csv
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
It calls `heimdall.fetch(provider, resource, ...)`, prints a header + schema
|
|
23
|
+
check + the frame via `_helpers.show()`, and optionally writes the frame to
|
|
24
|
+
`sandbox/scratch/`. Provider errors print as one line, no traceback.
|
|
25
|
+
|
|
26
|
+
## The examples
|
|
27
|
+
|
|
28
|
+
```bash
|
|
29
|
+
uv run python sandbox/examples/01_fred_to_terminal.py
|
|
30
|
+
uv run python sandbox/examples/03_custom_provider.py # no network
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
| file | shows |
|
|
34
|
+
|---|---|
|
|
35
|
+
| `01_fred_to_terminal.py` | fetch a FRED series, print it |
|
|
36
|
+
| `02_yfinance_to_terminal.py` | fetch OHLCV bars, print them |
|
|
37
|
+
| `03_custom_provider.py` | define + register a provider inline, then conformance-check it |
|
|
38
|
+
| `04_export_to_files.py` | fetch, write parquet + CSV to `scratch/`, read back |
|
|
39
|
+
| `05_provider_with_config.py` | `required_config`, `config_from_env`, and the `ConfigError` when a key is missing |
|
|
40
|
+
|
|
41
|
+
## Exporting to a real database
|
|
42
|
+
|
|
43
|
+
The file sinks (`to_parquet`, `to_csv`) work out of the box. `_helpers.py` has a
|
|
44
|
+
commented `to_database()` using `polars.DataFrame.write_database`; enable it by
|
|
45
|
+
adding a `sandbox` dependency group (`sqlalchemy>=2` + a driver) and setting
|
|
46
|
+
`HEIMDALL_SANDBOX_DB_URL`. Details are in the comment block.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Dev sandbox for Heimdall"""
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""Shared display + file-sink helpers for the sandbox scripts."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from datetime import UTC, datetime
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import polars as pl
|
|
10
|
+
|
|
11
|
+
from heimdall.contracts import FetchResult
|
|
12
|
+
|
|
13
|
+
SCRATCH = Path(__file__).resolve().parent / "scratch"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _slug(text: str) -> str:
|
|
17
|
+
cleaned = re.sub(r"[^0-9A-Za-z._-]+", "-", text).strip("-")
|
|
18
|
+
return cleaned or "resource"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _stamp() -> str:
|
|
22
|
+
return datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def show(result: FetchResult, *, rows: int = 10) -> None:
|
|
26
|
+
"""Print a FetchResult to the terminal: header, schema check, then the frame."""
|
|
27
|
+
frame = result.frame
|
|
28
|
+
print(f"provider : {result.provider_id}")
|
|
29
|
+
print(f"resource : {result.request.resource}")
|
|
30
|
+
if result.request.interval:
|
|
31
|
+
print(f"interval : {result.request.interval}")
|
|
32
|
+
print(f"schema : {result.schema.name}")
|
|
33
|
+
print(f"shape : {frame.height} rows x {frame.width} cols")
|
|
34
|
+
print(f"retrieved : {result.retrieved_at:%Y-%m-%d %H:%M:%SZ}")
|
|
35
|
+
print(f"metadata : {dict(result.metadata)}")
|
|
36
|
+
try:
|
|
37
|
+
result.schema.validate(frame)
|
|
38
|
+
print("validate : OK")
|
|
39
|
+
except Exception as exc: # noqa: BLE001 - sandbox: surface whatever went wrong
|
|
40
|
+
print(f"validate : FAIL - {exc}")
|
|
41
|
+
print()
|
|
42
|
+
with pl.Config(tbl_rows=rows, tbl_width_chars=120):
|
|
43
|
+
print(frame.head(rows))
|
|
44
|
+
if frame.height > rows:
|
|
45
|
+
print(f"... {frame.height - rows} more rows ...")
|
|
46
|
+
print(frame.tail(3))
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _target(result: FetchResult, ext: str) -> Path:
|
|
50
|
+
SCRATCH.mkdir(exist_ok=True)
|
|
51
|
+
name = f"{result.provider_id}_{_slug(result.request.resource)}_{_stamp()}.{ext}"
|
|
52
|
+
return SCRATCH / name
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def to_parquet(result: FetchResult) -> Path:
|
|
56
|
+
"""Write ``result.frame`` to ``sandbox/scratch/`` as parquet; return the path."""
|
|
57
|
+
path = _target(result, "parquet")
|
|
58
|
+
result.frame.write_parquet(path)
|
|
59
|
+
print(f"wrote {result.frame.height} rows -> {path}")
|
|
60
|
+
return path
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def to_csv(result: FetchResult) -> Path:
|
|
64
|
+
"""Write ``result.frame`` to ``sandbox/scratch/`` as CSV; return the path."""
|
|
65
|
+
path = _target(result, "csv")
|
|
66
|
+
result.frame.write_csv(path)
|
|
67
|
+
print(f"wrote {result.frame.height} rows -> {path}")
|
|
68
|
+
return path
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
# --- Database sink (not wired yet) -----------------------------------------------
|
|
72
|
+
# Heimdall keeps `fetch()` pure and leaves persistence to the host. To push a
|
|
73
|
+
# FetchResult into a real database from the sandbox:
|
|
74
|
+
#
|
|
75
|
+
# 1. add a dependency group in pyproject.toml, e.g.
|
|
76
|
+
# [dependency-groups]
|
|
77
|
+
# sandbox = ["sqlalchemy>=2"] # SQLite (driver is stdlib)
|
|
78
|
+
# # sandbox = ["sqlalchemy>=2", "psycopg[binary]>=3"] # PostgreSQL
|
|
79
|
+
# then: uv sync --group sandbox
|
|
80
|
+
# 2. set HEIMDALL_SANDBOX_DB_URL, e.g. sqlite:///sandbox/scratch/heimdall.db
|
|
81
|
+
# 3. uncomment:
|
|
82
|
+
#
|
|
83
|
+
# import os
|
|
84
|
+
#
|
|
85
|
+
# def to_database(
|
|
86
|
+
# result: FetchResult, *, url: str | None = None, table: str | None = None
|
|
87
|
+
# ) -> None:
|
|
88
|
+
# url = url or os.environ.get(
|
|
89
|
+
# "HEIMDALL_SANDBOX_DB_URL", "sqlite:///sandbox/scratch/heimdall.db"
|
|
90
|
+
# )
|
|
91
|
+
# table = table or f"{result.provider_id}_{_slug(result.request.resource)}"
|
|
92
|
+
# result.frame.write_database(table, url, if_table_exists="replace")
|
|
93
|
+
# print(f"wrote {result.frame.height} rows -> {table} @ {url}")
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""Fetch a FRED series and print it. Needs `heimdall-mimird[fred]`.
|
|
2
|
+
|
|
3
|
+
uv run python sandbox/examples/01_fred_to_terminal.py
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import pathlib
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
# Run-by-path bootstrap: put the repo root on sys.path so `import sandbox` works.
|
|
12
|
+
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2]))
|
|
13
|
+
|
|
14
|
+
from sandbox._helpers import show
|
|
15
|
+
|
|
16
|
+
import heimdall
|
|
17
|
+
|
|
18
|
+
result = heimdall.fetch("fred", "DGS10")
|
|
19
|
+
show(result)
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""Fetch OHLCV bars from Yahoo Finance and print them. Needs `heimdall-mimird[yfinance]`.
|
|
2
|
+
|
|
3
|
+
uv run python sandbox/examples/02_yfinance_to_terminal.py
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import pathlib
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2]))
|
|
12
|
+
|
|
13
|
+
from sandbox._helpers import show # noqa: E402
|
|
14
|
+
|
|
15
|
+
import heimdall # noqa: E402
|
|
16
|
+
|
|
17
|
+
result = heimdall.fetch("yfinance", "AAPL", interval="1d")
|
|
18
|
+
show(result, rows=8)
|