heimdall-mimird 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. heimdall_mimird-0.2.0/.github/workflows/ci.yml +30 -0
  2. heimdall_mimird-0.2.0/.gitignore +16 -0
  3. heimdall_mimird-0.2.0/.python-version +1 -0
  4. heimdall_mimird-0.2.0/CONTRIBUTING.md +42 -0
  5. heimdall_mimird-0.2.0/PKG-INFO +72 -0
  6. heimdall_mimird-0.2.0/README.md +52 -0
  7. heimdall_mimird-0.2.0/ROADMAP.md +61 -0
  8. heimdall_mimird-0.2.0/docs/writing-a-provider.md +125 -0
  9. heimdall_mimird-0.2.0/pyproject.toml +59 -0
  10. heimdall_mimird-0.2.0/sandbox/README.md +46 -0
  11. heimdall_mimird-0.2.0/sandbox/__init__.py +1 -0
  12. heimdall_mimird-0.2.0/sandbox/__main__.py +7 -0
  13. heimdall_mimird-0.2.0/sandbox/_helpers.py +93 -0
  14. heimdall_mimird-0.2.0/sandbox/examples/01_fred_to_terminal.py +19 -0
  15. heimdall_mimird-0.2.0/sandbox/examples/02_yfinance_to_terminal.py +18 -0
  16. heimdall_mimird-0.2.0/sandbox/examples/03_custom_provider.py +57 -0
  17. heimdall_mimird-0.2.0/sandbox/examples/04_export_to_files.py +28 -0
  18. heimdall_mimird-0.2.0/sandbox/examples/05_provider_with_config.py +56 -0
  19. heimdall_mimird-0.2.0/sandbox/play.py +66 -0
  20. heimdall_mimird-0.2.0/sandbox/scratch/.gitkeep +1 -0
  21. heimdall_mimird-0.2.0/src/heimdall/__init__.py +104 -0
  22. heimdall_mimird-0.2.0/src/heimdall/_logging.py +23 -0
  23. heimdall_mimird-0.2.0/src/heimdall/_retry.py +61 -0
  24. heimdall_mimird-0.2.0/src/heimdall/_time.py +10 -0
  25. heimdall_mimird-0.2.0/src/heimdall/contracts.py +145 -0
  26. heimdall_mimird-0.2.0/src/heimdall/errors.py +35 -0
  27. heimdall_mimird-0.2.0/src/heimdall/provider.py +81 -0
  28. heimdall_mimird-0.2.0/src/heimdall/providers/__init__.py +9 -0
  29. heimdall_mimird-0.2.0/src/heimdall/providers/fred/__init__.py +16 -0
  30. heimdall_mimird-0.2.0/src/heimdall/providers/fred/provider.py +103 -0
  31. heimdall_mimird-0.2.0/src/heimdall/providers/yfinance/__init__.py +16 -0
  32. heimdall_mimird-0.2.0/src/heimdall/providers/yfinance/provider.py +105 -0
  33. heimdall_mimird-0.2.0/src/heimdall/registry.py +112 -0
  34. heimdall_mimird-0.2.0/src/heimdall/schemas.py +50 -0
  35. heimdall_mimird-0.2.0/src/heimdall/testing.py +107 -0
  36. heimdall_mimird-0.2.0/tests/conftest.py +47 -0
  37. heimdall_mimird-0.2.0/tests/providers/test_fred.py +80 -0
  38. heimdall_mimird-0.2.0/tests/providers/test_yfinance.py +90 -0
  39. heimdall_mimird-0.2.0/tests/test_contracts.py +97 -0
  40. heimdall_mimird-0.2.0/tests/test_registry.py +55 -0
  41. heimdall_mimird-0.2.0/uv.lock +1735 -0
@@ -0,0 +1,30 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [main]
6
+ pull_request:
7
+
8
+ jobs:
9
+ check:
10
+ runs-on: ubuntu-latest
11
+ strategy:
12
+ fail-fast: false
13
+ matrix:
14
+ python-version: ["3.11", "3.12"]
15
+ steps:
16
+ - uses: actions/checkout@v4
17
+ - name: Install uv
18
+ uses: astral-sh/setup-uv@v5
19
+ with:
20
+ python-version: ${{ matrix.python-version }}
21
+ - name: Sync
22
+ run: uv sync --extra all --group dev
23
+ - name: Lint
24
+ run: uv run ruff check .
25
+ - name: Format
26
+ run: uv run ruff format --check .
27
+ - name: Types
28
+ run: uv run mypy src/heimdall
29
+ - name: Test
30
+ run: uv run pytest -m "not network" -q
@@ -0,0 +1,16 @@
1
+ *.egg-info
2
+ dist
3
+ .venv
4
+ __pycache__/
5
+ *.pyc
6
+ .pytest_cache/
7
+ .mypy_cache/
8
+ .ruff_cache/
9
+ .coverage
10
+ htmlcov/
11
+
12
+ # dev sandbox - keep the folder, ignore experiments and local data
13
+ sandbox/scratch/*
14
+ !sandbox/scratch/.gitkeep
15
+ *.db
16
+ *.duckdb
@@ -0,0 +1 @@
1
+ 3.11
@@ -0,0 +1,42 @@
1
+ # Contributing
2
+
3
+ ## Dev setup
4
+
5
+ ```bash
6
+ uv sync --extra all --group dev
7
+ ```
8
+
9
+ ## Checks (what CI runs)
10
+
11
+ ```bash
12
+ uv run ruff check .
13
+ uv run ruff format --check .
14
+ uv run mypy src/heimdall
15
+ uv run pytest -m "not network" # offline suite
16
+ uv run pytest -m network # hits real FRED / Yahoo; run before releases
17
+ ```
18
+
19
+ ## Adding a provider
20
+
21
+ First-party providers live under `src/heimdall/providers/<name>/` and are
22
+ registered by a module-level `_register(registry)` hook (see the `fred` and
23
+ `yfinance` packages). Their client library goes in a matching
24
+ `[project.optional-dependencies]` extra so `import heimdall` still works without
25
+ it.
26
+
27
+ Every provider must:
28
+
29
+ 1. Subclass `heimdall.Provider`, set a stable `id` and a `Capabilities`.
30
+ 2. Return a `FetchResult` whose `frame` passes `schema.validate()` - reuse
31
+ `heimdall.schemas.OHLCV_BARS` / `OBSERVATIONS` or define your own `SchemaSpec`.
32
+ 3. Raise `heimdall.errors.*` (`RequestError`, `UpstreamError`, `ConfigError`),
33
+ never bare `ValueError` / `KeyError`.
34
+ 4. Pass the conformance suite. Add a test module that subclasses
35
+ `heimdall.testing.ProviderContractTests` plus a `@pytest.mark.network`
36
+ smoke test.
37
+
38
+ `heimdall.testing` needs pytest. It is not part of the base install - an
39
+ out-of-tree provider package gets it with `pip install heimdall-mimird[testing]`.
40
+ Heimdall's own `dev` group already includes pytest.
41
+
42
+ See `docs/writing-a-provider.md` for a full example.
@@ -0,0 +1,72 @@
1
+ Metadata-Version: 2.5
2
+ Name: heimdall-mimird
3
+ Version: 0.2.0
4
+ Summary: A data-provider SDK
5
+ Requires-Python: >=3.11
6
+ Requires-Dist: polars>=1.44.1
7
+ Requires-Dist: ramonavocado-logger>=0.1.3
8
+ Provides-Extra: all
9
+ Requires-Dist: httpx>=0.27; extra == 'all'
10
+ Requires-Dist: pyarrow>=15; extra == 'all'
11
+ Requires-Dist: yfinance>=0.2.40; extra == 'all'
12
+ Provides-Extra: fred
13
+ Requires-Dist: httpx>=0.27; extra == 'fred'
14
+ Provides-Extra: testing
15
+ Requires-Dist: pytest>=8.2; extra == 'testing'
16
+ Provides-Extra: yfinance
17
+ Requires-Dist: pyarrow>=15; extra == 'yfinance'
18
+ Requires-Dist: yfinance>=0.2.40; extra == 'yfinance'
19
+ Description-Content-Type: text/markdown
20
+
21
+ # Heimdall
22
+
23
+ A host-agnostic **data-provider SDK**. Finance data first (Yahoo Finance OHLCV,
24
+ FRED economic series), built so that any data source - yours or the community's -
25
+ can plug in behind one small contract.
26
+
27
+ A provider takes a `FetchRequest` and returns a `FetchResult`: a
28
+ [Polars](https://pola.rs) frame, a `SchemaSpec` that describes and validates it,
29
+ and some metadata. That's the whole surface. Nothing in the core is
30
+ finance-specific; OHLCV bars and economic observations are just two predefined
31
+ schemas.
32
+
33
+ ## Install
34
+
35
+ ```bash
36
+ pip install heimdall-mimird[fred,yfinance] # or: heimdall-mimird[all]
37
+ ```
38
+
39
+ The core depends only on `polars` (plus a small logging shim). Each provider
40
+ pulls its own client library through an extra, so you install only what you use.
41
+
42
+ ## Quick start
43
+
44
+ ```python
45
+ import heimdall
46
+
47
+ # string shorthand -> FetchRequest(resource="DGS10")
48
+ result = heimdall.fetch("fred", "DGS10")
49
+ result.schema.validate(result.frame) # raises SchemaError if it doesn't conform
50
+ print(result.frame.head())
51
+
52
+ bars = heimdall.fetch("yfinance", "AAPL", interval="1d")
53
+ print(bars.frame.tail())
54
+ print(bars.metadata) # {"ticker": "AAPL", "interval": "1d", "rows": ...}
55
+ ```
56
+
57
+ `heimdall.list_providers()` shows what's registered. Providers are registered
58
+ automatically when their extra is installed; register your own with
59
+ `heimdall.register(MyProvider)`.
60
+
61
+ ## Writing a provider
62
+
63
+ Subclass `heimdall.Provider`, set `id` + `capabilities`, implement `fetch`. See
64
+ [docs/writing-a-provider.md](docs/writing-a-provider.md) for a full worked
65
+ example, and run `heimdall.testing.assert_provider_conformance(...)` to check it
66
+ against the contract.
67
+
68
+ ## Status
69
+
70
+ Early. The provider contract and the two bundled providers are usable today. The
71
+ ingestion/storage layer, entry-point plugin discovery, and retry/rate-limit
72
+ middleware are on the [roadmap](ROADMAP.md).
@@ -0,0 +1,52 @@
1
+ # Heimdall
2
+
3
+ A host-agnostic **data-provider SDK**. Finance data first (Yahoo Finance OHLCV,
4
+ FRED economic series), built so that any data source - yours or the community's -
5
+ can plug in behind one small contract.
6
+
7
+ A provider takes a `FetchRequest` and returns a `FetchResult`: a
8
+ [Polars](https://pola.rs) frame, a `SchemaSpec` that describes and validates it,
9
+ and some metadata. That's the whole surface. Nothing in the core is
10
+ finance-specific; OHLCV bars and economic observations are just two predefined
11
+ schemas.
12
+
13
+ ## Install
14
+
15
+ ```bash
16
+ pip install heimdall-mimird[fred,yfinance] # or: heimdall-mimird[all]
17
+ ```
18
+
19
+ The core depends only on `polars` (plus a small logging shim). Each provider
20
+ pulls its own client library through an extra, so you install only what you use.
21
+
22
+ ## Quick start
23
+
24
+ ```python
25
+ import heimdall
26
+
27
+ # string shorthand -> FetchRequest(resource="DGS10")
28
+ result = heimdall.fetch("fred", "DGS10")
29
+ result.schema.validate(result.frame) # raises SchemaError if it doesn't conform
30
+ print(result.frame.head())
31
+
32
+ bars = heimdall.fetch("yfinance", "AAPL", interval="1d")
33
+ print(bars.frame.tail())
34
+ print(bars.metadata) # {"ticker": "AAPL", "interval": "1d", "rows": ...}
35
+ ```
36
+
37
+ `heimdall.list_providers()` shows what's registered. Providers are registered
38
+ automatically when their extra is installed; register your own with
39
+ `heimdall.register(MyProvider)`.
40
+
41
+ ## Writing a provider
42
+
43
+ Subclass `heimdall.Provider`, set `id` + `capabilities`, implement `fetch`. See
44
+ [docs/writing-a-provider.md](docs/writing-a-provider.md) for a full worked
45
+ example, and run `heimdall.testing.assert_provider_conformance(...)` to check it
46
+ against the contract.
47
+
48
+ ## Status
49
+
50
+ Early. The provider contract and the two bundled providers are usable today. The
51
+ ingestion/storage layer, entry-point plugin discovery, and retry/rate-limit
52
+ middleware are on the [roadmap](ROADMAP.md).
@@ -0,0 +1,61 @@
1
+ # Roadmap
2
+
3
+ Legend: `[x]` shipped · `[~]` partially built · `[ ]` planned
4
+
5
+ ## Shipped
6
+
7
+ - [x] **Core SDK** - `FetchRequest` / `FetchResult` / `SchemaSpec` contract,
8
+ `Provider` base + `Capabilities`, `ProviderRegistry` with explicit
9
+ registration, error hierarchy, `heimdall.fetch(...)`.
10
+ - [x] **Bundled providers** - `fred` (public `fredgraph.csv` endpoint, no key)
11
+ and `yfinance` (OHLCV via the `yfinance` library), each behind an extra.
12
+ - [x] **Provider conformance kit** - `heimdall.testing.assert_provider_conformance`
13
+ + `ProviderContractTests`, behind the `testing` extra. This is the "testing
14
+ harness for provider authors" - third parties prove their provider fits the
15
+ spec by running it.
16
+ - [x] **Tooling** - ruff, mypy, pytest (with a `network` marker), GitHub Actions CI.
17
+ - [x] **Dev sandbox** - `sandbox/` with a `python -m sandbox` runner and example
18
+ scripts; results print to the terminal and can be dumped to parquet / CSV.
19
+
20
+ ## Planned (next)
21
+
22
+ - [ ] **Conformance kit polish** - recorded-fixture helpers so third parties can
23
+ write offline contract tests without hand-rolling mocks.
24
+ - [ ] **Docs** - expand `docs/writing-a-provider.md` with a schema-authoring
25
+ section.
26
+
27
+ ## Candidate ideas (evaluated, not yet scheduled)
28
+
29
+ - [~] **Entry-point plugin discovery** - `ProviderRegistry.load_entry_points()`
30
+ already works; wire a documented `heimdall.providers` entry-point group and add
31
+ a `@heimdall.provider` decorator so `pip install heimdall-stripe` auto-registers
32
+ with the core package. Single biggest driver of ecosystem growth (cf. dlt,
33
+ pytest, SQLAlchemy dialects).
34
+ - [~] **Config & credentials layer** - already have `Provider(config=...)`,
35
+ `Capabilities.required_config`, and `config_from_env()`. Add `.env` loading and
36
+ a pluggable secret-source hook (env / file / secret manager) so provider
37
+ authors don't reinvent it.
38
+ - [~] **Finer error taxonomy** - split `UpstreamError` into `ProviderAuthError` /
39
+ `ProviderRateLimitError` / `ProviderTransientError` so orchestration code can
40
+ react generically: retry vs fail vs backoff.
41
+ - [ ] **Rate limiting / backpressure** - a shared limiter providers opt into;
42
+ make `Capabilities.rate_limit_per_min` enforced rather than advisory. Part of a
43
+ middleware stack that wraps `fetch()` host-side (retry + rate-limit + cache).
44
+ - [ ] **State / checkpoint store** - a pluggable interface (file / DB / redis) for
45
+ "last synced cursor", so incremental sync is usable in production. Needs a
46
+ cursor concept in the contract (`FetchRequest.cursor` + a next-cursor on
47
+ `FetchResult`). Larger design item.
48
+ - [ ] **Observability hooks** - a logging/metrics callback protocol (records
49
+ fetched, latency, errors) that plugs into whatever monitoring the host already
50
+ runs. Overlaps the middleware work.
51
+ - [ ] **`create-provider` scaffolder** - a `heimdall new-provider <name>` command
52
+ or a cookiecutter template that emits a provider package skeleton plus its
53
+ conformance test. Directly lowers the bar for third-party contributors.
54
+ - [ ] **`heimdall.ingest`** - persist `FetchResult`s: parquet tiers + a JSON
55
+ manifest, plus the transform set (`diff_1`, `pct_change_1`, `sma_20`, `sma_50`).
56
+ Starting point is parked at `docs/legacy/essential.py`.
57
+ - [ ] **FRED official API** - swap/augment the CSV endpoint with
58
+ `api.stlouisfed.org` (needs `FRED_API_KEY` -> `Capabilities.required_config`),
59
+ giving vintages, metadata, and pagination.
60
+ - [ ] **Packaging split** - break `heimdall-core` and each provider into separate
61
+ distributions, if the boundaries hold up.
@@ -0,0 +1,125 @@
1
+ # Writing a provider
2
+
3
+ A provider is one class: it turns a `FetchRequest` into a `FetchResult`. You can
4
+ keep it inside your own project or publish it - Heimdall imposes nothing beyond
5
+ the contract.
6
+
7
+ ## The contract
8
+
9
+ ```python
10
+ FetchRequest(resource, interval=None, start=None, end=None, params={})
11
+ FetchResult(frame, schema, provider_id, request, retrieved_at, metadata={})
12
+ ```
13
+
14
+ - **`resource`** is provider-specific: a ticker, a series id, an endpoint key.
15
+ - **`frame`** is a `polars.DataFrame`.
16
+ - **`schema`** is a `SchemaSpec` that must actually validate `frame`. Reuse
17
+ `heimdall.schemas.OHLCV_BARS` / `OBSERVATIONS`, or define your own.
18
+
19
+ ## A minimal example
20
+
21
+ A provider that pulls a CSV with `date,value` columns from a URL:
22
+
23
+ ```python
24
+ from datetime import datetime
25
+ from io import StringIO
26
+
27
+ import httpx
28
+ import polars as pl
29
+
30
+ import heimdall
31
+ from heimdall import Capabilities, FetchRequest, FetchResult, Provider
32
+ from heimdall.contracts import ColumnSpec, SchemaSpec
33
+ from heimdall.errors import RequestError, UpstreamError
34
+
35
+ CSV_SERIES = SchemaSpec(
36
+ name="csv.series",
37
+ columns=(
38
+ ColumnSpec("timestamp", pl.Datetime, nullable=False),
39
+ ColumnSpec("value", pl.Float64, nullable=False),
40
+ ),
41
+ time_column="timestamp",
42
+ primary_key=("timestamp",),
43
+ )
44
+
45
+
46
+ class CsvUrlProvider(Provider):
47
+ id = "csv-url"
48
+ capabilities = Capabilities(data_kinds=(CSV_SERIES.name,))
49
+
50
+ def fetch(self, request: FetchRequest) -> FetchResult:
51
+ url = request.params.get("url") or request.resource
52
+ try:
53
+ resp = httpx.get(url, timeout=30, follow_redirects=True)
54
+ resp.raise_for_status()
55
+ except httpx.HTTPError as exc:
56
+ raise UpstreamError(f"could not fetch {url}: {exc}") from exc
57
+
58
+ frame = (
59
+ pl.read_csv(StringIO(resp.text))
60
+ .rename({"date": "timestamp", "value": "value"})
61
+ .with_columns(
62
+ pl.col("timestamp").str.strptime(pl.Datetime, strict=False),
63
+ pl.col("value").cast(pl.Float64, strict=False),
64
+ )
65
+ .drop_nulls()
66
+ .sort("timestamp")
67
+ )
68
+ if frame.is_empty():
69
+ raise RequestError(f"no rows returned from {url}")
70
+
71
+ CSV_SERIES.validate(frame)
72
+ return FetchResult(
73
+ frame=frame,
74
+ schema=CSV_SERIES,
75
+ provider_id=self.id,
76
+ request=request,
77
+ retrieved_at=heimdall.utcnow(),
78
+ )
79
+ ```
80
+
81
+ ## Register and use it
82
+
83
+ ```python
84
+ import heimdall
85
+
86
+ heimdall.register(CsvUrlProvider)
87
+ result = heimdall.fetch("csv-url", "https://example.com/rates.csv")
88
+ ```
89
+
90
+ For config/secrets, declare `required_config` in `Capabilities` and pass values
91
+ at registration - never read the environment inside the provider:
92
+
93
+ ```python
94
+ heimdall.register(MyProvider, config=heimdall.config_from_env("HEIMDALL_MYPROVIDER_"))
95
+ ```
96
+
97
+ ## Prove it conforms
98
+
99
+ `heimdall.testing` needs pytest, which the base install does not pull in:
100
+
101
+ ```bash
102
+ pip install heimdall-mimird[testing]
103
+ ```
104
+
105
+ ```python
106
+ from heimdall.testing import assert_provider_conformance
107
+ from heimdall import FetchRequest
108
+
109
+ assert_provider_conformance(
110
+ CsvUrlProvider(), FetchRequest(resource="https://example.com/rates.csv")
111
+ )
112
+ ```
113
+
114
+ Or in a pytest suite, subclass `heimdall.testing.ProviderContractTests` and
115
+ implement `make_provider()` / `sample_request()`.
116
+
117
+ ## Rules the conformance kit enforces
118
+
119
+ - `id` is a non-empty string; `capabilities.data_kinds` is non-empty.
120
+ - `fetch()` returns a `FetchResult` with a non-empty frame that passes
121
+ `schema.validate()`.
122
+ - `result.provider_id == self.id`; `retrieved_at` is timezone-aware.
123
+ - Two identical requests return the same schema and columns.
124
+ - An unsupported `interval` raises `RequestError` (not a bare exception).
125
+ - Missing `required_config` raises `ConfigError` at construction.
@@ -0,0 +1,59 @@
1
+ [project]
2
+ name = "heimdall-mimird"
3
+ version = "0.2.0"
4
+ description = "A data-provider SDK"
5
+ readme = "README.md"
6
+ requires-python = ">=3.11"
7
+ dependencies = [
8
+ "polars>=1.44.1",
9
+ "ramonavocado-logger>=0.1.3",
10
+ ]
11
+
12
+ [project.optional-dependencies]
13
+ fred = ["httpx>=0.27"]
14
+ yfinance = ["yfinance>=0.2.40", "pyarrow>=15"]
15
+ testing = ["pytest>=8.2"]
16
+ all = ["httpx>=0.27", "yfinance>=0.2.40", "pyarrow>=15"]
17
+
18
+ [dependency-groups]
19
+ # sandbox = ["sqlalchemy>=2"] # add when the dev sandbox needs a real database
20
+ dev = [
21
+ "pytest>=8.2",
22
+ "pytest-cov>=5.0",
23
+ "respx>=0.21",
24
+ "ruff>=0.6",
25
+ "mypy>=1.11",
26
+ ]
27
+
28
+ [build-system]
29
+ requires = ["hatchling"]
30
+ build-backend = "hatchling.build"
31
+
32
+ [tool.hatch.build.targets.wheel]
33
+ packages = ["src/heimdall"]
34
+
35
+ [tool.ruff]
36
+ line-length = 100
37
+ src = ["src", "tests", "sandbox"]
38
+ extend-exclude = ["sandbox/scratch"]
39
+
40
+ [tool.ruff.lint]
41
+ select = ["E", "F", "I", "UP", "B", "SIM"]
42
+
43
+ [tool.mypy]
44
+ python_version = "3.11"
45
+ files = ["src/heimdall"]
46
+ disallow_untyped_defs = true
47
+ warn_unused_ignores = true
48
+ warn_redundant_casts = true
49
+ no_implicit_optional = true
50
+
51
+ [[tool.mypy.overrides]]
52
+ module = ["ramonavocado_logger.*", "yfinance.*"]
53
+ ignore_missing_imports = true
54
+
55
+ [tool.pytest.ini_options]
56
+ testpaths = ["tests"]
57
+ markers = [
58
+ "network: test hits a real external service (deselect with '-m \"not network\"')",
59
+ ]
@@ -0,0 +1,46 @@
1
+ # sandbox/
2
+
3
+ A dev-only playground for exercising Heimdall providers by hand. **Not shipped** -
4
+ the package build only includes `src/heimdall`.
5
+
6
+ - **Committed** (`play.py`, `_helpers.py`, `examples/`) - reference material, kept
7
+ tidy and lint-clean.
8
+ - **`scratch/`** - git-ignored. Put your own throwaway scripts and any generated
9
+ `.parquet` / `.csv` / `.db` files here.
10
+
11
+ Run everything from the repo root, after `uv sync --extra all --group dev`.
12
+
13
+ ## The runner
14
+
15
+ ```bash
16
+ uv run python -m sandbox --list
17
+ uv run python -m sandbox fred DGS10
18
+ uv run python -m sandbox yfinance AAPL --interval 1d --rows 5
19
+ uv run python -m sandbox fred DGS10 --start 2020-01-01 --to-parquet --to-csv
20
+ ```
21
+
22
+ It calls `heimdall.fetch(provider, resource, ...)`, prints a header + schema
23
+ check + the frame via `_helpers.show()`, and optionally writes the frame to
24
+ `sandbox/scratch/`. Provider errors print as one line, no traceback.
25
+
26
+ ## The examples
27
+
28
+ ```bash
29
+ uv run python sandbox/examples/01_fred_to_terminal.py
30
+ uv run python sandbox/examples/03_custom_provider.py # no network
31
+ ```
32
+
33
+ | file | shows |
34
+ |---|---|
35
+ | `01_fred_to_terminal.py` | fetch a FRED series, print it |
36
+ | `02_yfinance_to_terminal.py` | fetch OHLCV bars, print them |
37
+ | `03_custom_provider.py` | define + register a provider inline, then conformance-check it |
38
+ | `04_export_to_files.py` | fetch, write parquet + CSV to `scratch/`, read back |
39
+ | `05_provider_with_config.py` | `required_config`, `config_from_env`, and the `ConfigError` when a key is missing |
40
+
41
+ ## Exporting to a real database
42
+
43
+ The file sinks (`to_parquet`, `to_csv`) work out of the box. `_helpers.py` has a
44
+ commented `to_database()` using `polars.DataFrame.write_database`; enable it by
45
+ adding a `sandbox` dependency group (`sqlalchemy>=2` + a driver) and setting
46
+ `HEIMDALL_SANDBOX_DB_URL`. Details are in the comment block.
@@ -0,0 +1 @@
1
+ """Dev sandbox for Heimdall"""
@@ -0,0 +1,7 @@
1
+ """Entry point for ``python -m sandbox``."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from sandbox.play import main
6
+
7
+ raise SystemExit(main())
@@ -0,0 +1,93 @@
1
+ """Shared display + file-sink helpers for the sandbox scripts."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from datetime import UTC, datetime
7
+ from pathlib import Path
8
+
9
+ import polars as pl
10
+
11
+ from heimdall.contracts import FetchResult
12
+
13
+ SCRATCH = Path(__file__).resolve().parent / "scratch"
14
+
15
+
16
+ def _slug(text: str) -> str:
17
+ cleaned = re.sub(r"[^0-9A-Za-z._-]+", "-", text).strip("-")
18
+ return cleaned or "resource"
19
+
20
+
21
+ def _stamp() -> str:
22
+ return datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
23
+
24
+
25
+ def show(result: FetchResult, *, rows: int = 10) -> None:
26
+ """Print a FetchResult to the terminal: header, schema check, then the frame."""
27
+ frame = result.frame
28
+ print(f"provider : {result.provider_id}")
29
+ print(f"resource : {result.request.resource}")
30
+ if result.request.interval:
31
+ print(f"interval : {result.request.interval}")
32
+ print(f"schema : {result.schema.name}")
33
+ print(f"shape : {frame.height} rows x {frame.width} cols")
34
+ print(f"retrieved : {result.retrieved_at:%Y-%m-%d %H:%M:%SZ}")
35
+ print(f"metadata : {dict(result.metadata)}")
36
+ try:
37
+ result.schema.validate(frame)
38
+ print("validate : OK")
39
+ except Exception as exc: # noqa: BLE001 - sandbox: surface whatever went wrong
40
+ print(f"validate : FAIL - {exc}")
41
+ print()
42
+ with pl.Config(tbl_rows=rows, tbl_width_chars=120):
43
+ print(frame.head(rows))
44
+ if frame.height > rows:
45
+ print(f"... {frame.height - rows} more rows ...")
46
+ print(frame.tail(3))
47
+
48
+
49
+ def _target(result: FetchResult, ext: str) -> Path:
50
+ SCRATCH.mkdir(exist_ok=True)
51
+ name = f"{result.provider_id}_{_slug(result.request.resource)}_{_stamp()}.{ext}"
52
+ return SCRATCH / name
53
+
54
+
55
+ def to_parquet(result: FetchResult) -> Path:
56
+ """Write ``result.frame`` to ``sandbox/scratch/`` as parquet; return the path."""
57
+ path = _target(result, "parquet")
58
+ result.frame.write_parquet(path)
59
+ print(f"wrote {result.frame.height} rows -> {path}")
60
+ return path
61
+
62
+
63
+ def to_csv(result: FetchResult) -> Path:
64
+ """Write ``result.frame`` to ``sandbox/scratch/`` as CSV; return the path."""
65
+ path = _target(result, "csv")
66
+ result.frame.write_csv(path)
67
+ print(f"wrote {result.frame.height} rows -> {path}")
68
+ return path
69
+
70
+
71
+ # --- Database sink (not wired yet) -----------------------------------------------
72
+ # Heimdall keeps `fetch()` pure and leaves persistence to the host. To push a
73
+ # FetchResult into a real database from the sandbox:
74
+ #
75
+ # 1. add a dependency group in pyproject.toml, e.g.
76
+ # [dependency-groups]
77
+ # sandbox = ["sqlalchemy>=2"] # SQLite (driver is stdlib)
78
+ # # sandbox = ["sqlalchemy>=2", "psycopg[binary]>=3"] # PostgreSQL
79
+ # then: uv sync --group sandbox
80
+ # 2. set HEIMDALL_SANDBOX_DB_URL, e.g. sqlite:///sandbox/scratch/heimdall.db
81
+ # 3. uncomment:
82
+ #
83
+ # import os
84
+ #
85
+ # def to_database(
86
+ # result: FetchResult, *, url: str | None = None, table: str | None = None
87
+ # ) -> None:
88
+ # url = url or os.environ.get(
89
+ # "HEIMDALL_SANDBOX_DB_URL", "sqlite:///sandbox/scratch/heimdall.db"
90
+ # )
91
+ # table = table or f"{result.provider_id}_{_slug(result.request.resource)}"
92
+ # result.frame.write_database(table, url, if_table_exists="replace")
93
+ # print(f"wrote {result.frame.height} rows -> {table} @ {url}")
@@ -0,0 +1,19 @@
1
+ """Fetch a FRED series and print it. Needs `heimdall-mimird[fred]`.
2
+
3
+ uv run python sandbox/examples/01_fred_to_terminal.py
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import pathlib
9
+ import sys
10
+
11
+ # Run-by-path bootstrap: put the repo root on sys.path so `import sandbox` works.
12
+ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2]))
13
+
14
+ from sandbox._helpers import show
15
+
16
+ import heimdall
17
+
18
+ result = heimdall.fetch("fred", "DGS10")
19
+ show(result)
@@ -0,0 +1,18 @@
1
+ """Fetch OHLCV bars from Yahoo Finance and print them. Needs `heimdall-mimird[yfinance]`.
2
+
3
+ uv run python sandbox/examples/02_yfinance_to_terminal.py
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import pathlib
9
+ import sys
10
+
11
+ sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[2]))
12
+
13
+ from sandbox._helpers import show # noqa: E402
14
+
15
+ import heimdall # noqa: E402
16
+
17
+ result = heimdall.fetch("yfinance", "AAPL", interval="1d")
18
+ show(result, rows=8)