brasa-core 0.5.0__tar.gz

brasa_core-0.5.0/.gitignore

@@ -0,0 +1,143 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Git worktrees
+.worktrees/
+
+data/big-files/BDIN.zip
+data/big-files/OpcoesAcoesEmAberto.zip
+legacy/
+.brasa-cache/
+.vscode/launch.json
+
+# Generated skill lockfile and script outputs
+skills-lock.json
+scripts/out/
+tmp/

brasa_core-0.5.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Wilson Freitas
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

brasa_core-0.5.0/PKG-INFO

@@ -0,0 +1,170 @@
+Metadata-Version: 2.5
+Name: brasa-core
+Version: 0.5.0
+Summary: Template-driven ETL engine: download, parse, transform and query datasets described by YAML templates (official templates for Brazilian market and economic data: brasa-templates).
+Author-email: wilsonfreitas <wilson.freitas@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.11
+Requires-Dist: bizdays>=1.0.15
+Requires-Dist: duckdb>=1.2.0
+Requires-Dist: lxml>=4.9.2
+Requires-Dist: numpy>=2.0.0
+Requires-Dist: openpyxl>=3.1.5
+Requires-Dist: packaging>=23.0
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: platformdirs>=4.0.0
+Requires-Dist: pyarrow>=19.0.0
+Requires-Dist: python-bcb>=0.4.1
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: regexparser>=0.1.0
+Requires-Dist: requests>=2.28.0
+Requires-Dist: rich>=13.0.0
+Description-Content-Type: text/markdown
+
+# brasa
+
+A template-driven ETL engine: download, parse, transform and query datasets
+described by YAML templates. The official templates —
+[brasa-templates](https://github.com/wilsonfreitas/brasa-templates) — cover
+Brazilian market and economic data (B3, ANBIMA, CVM, BCB).
+
+## Installation
+
+Install from PyPI as `brasa-core` (the import name and the command are `brasa`),
+then run `brasa init` to choose a data directory and get the templates:
+
+    pip install brasa-core
+    brasa init
+
+Until 0.4.0 the distribution was called `brasa-marketdata`; since 0.5.0 it is
+`brasa-core`, and the templates live in brasa-templates instead of the package.
+
+Or directly from GitHub (latest `main`):
+
+    pip install git+https://github.com/wilsonfreitas/brasa.git
+
+With `uv`:
+
+    uv add brasa-core
+    uv add git+https://github.com/wilsonfreitas/brasa.git
+
+## Data directory (`brasa init` / `BRASA_DATA_PATH`)
+
+brasa stores everything — raw downloads, parsed parquet, and the metadata DB —
+under a single *brasa home*. On a fresh install, configure it once:
+
+    brasa init
+
+This suggests a platform default (e.g. `~/.local/share/brasa` on Linux),
+lets you confirm or type another path, and persists the choice in
+`~/.config/brasa/config.toml`. It also sets up your templates: it clones the
+official [brasa-templates](https://github.com/wilsonfreitas/brasa-templates)
+repo into `<data path>/templates` (use `--templates <url|path>` for a fork or
+a folder you already have). Get template updates later with
+`brasa update-templates`. Non-interactive variants:
+
+    brasa init --data-path /data/brasa   # explicit path, no prompt
+    brasa init --yes                     # accept the default, no prompt
+
+The `BRASA_DATA_PATH` environment variable, when set, always takes
+precedence over the config file — useful for CI, containers, tests, and
+one-off overrides. Commands that read or write data fail with a clear
+error until one of the two is configured; template introspection commands
+(`list-templates`, `deps`, `graph`) work without any configuration.
+
+Templates can live in your own template folders too (`BRASA_TEMPLATE_PATH`
+or `template_path` in the config file). Templates are code — they can run
+Python — so only add folders you trust; see
+[Template folders](docs/TEMPLATES.md#template-folders).
+
+## Changelog
+
+### Deterministic Download Status Codes
+
+Every download attempt is now classified with a single, unambiguous status code
+persisted in the `download_trials` table:
+
+| Symbol | Name       | Trigger                                           |
+|--------|------------|---------------------------------------------------|
+| `.`    | PASSED     | Successful download                               |
+| `F`    | FAILED     | Expected failure (`DownloadException`)             |
+| `E`    | ERROR      | Unexpected exception                              |
+| `S`    | SKIPPED    | Skipped (cache hit / invalid / duplicated)         |
+| `D`    | DUPLICATED | Raw folder already exists                         |
+| `I`    | INVALID    | Content validation failure                        |
+| `W`    | WARNING    | Success with warnings                             |
+
+**DB migration**: Existing caches are upgraded automatically on startup.
+Legacy `downloaded=1` rows become `PASSED`; `downloaded=0` become `FAILED`.
+(The standalone migration script has been removed after being applied; it is
+recoverable from git history if ever needed.)
+
+See [docs/USER_GUIDE.md](docs/USER_GUIDE.md#download-status-codes) for full details.
+
+### Import Local Files (`import_marketdata` / `brasa import`)
+
+Files with no download URL — a one-off vendor file, a manually-provided upload,
+or a corrected file for backfill — can now be imported using the same
+validate → gzip → checksum-dedup → parse → store engine as `download`. Only
+the acquisition step changes: bytes are read from disk instead of HTTP.
+
+```bash
+# Backfill a single date into a template that normally downloads via HTTP
+brasa import b3-cotahist-daily --path /data/backfill/COTAHIST_D02012024.TXT --arg refdate=2024-01-02
+
+# Bulk import one file per business day using a date pattern
+brasa import my-daily-template --path '/data/prices/%Y-%m-%d.csv' --arg refdate=@2026-06-01:2026-06-30
+```
+
+```python
+from brasa import import_marketdata
+
+import_marketdata("b3-cotahist-daily", path="/data/backfill/COTAHIST_D02012024.TXT", refdate="2024-01-02")
+```
+
+See [docs/CLI.md](docs/CLI.md#import), [docs/API_REFERENCE.md](docs/API_REFERENCE.md#data-import),
+and [docs/TEMPLATES.md](docs/TEMPLATES.md#importing-local-files-importer) for full details.
+
+## Publishing to PyPI
+
+brasa is built with [hatchling](https://hatch.pypa.io/) and published manually
+under the distribution name `brasa-core` (the PyPI name `brasa` was already
+taken by an unrelated project; `import brasa` is unaffected; until 0.4.0 the
+name was `brasa-marketdata`). The SQL DDL ships inside the package
+(`brasa/files/sql/`); templates do not — they live in brasa-templates.
+
+1. Bump `version` in `pyproject.toml`.
+2. Build the wheel and sdist:
+
+   ```bash
+   uv build
+   ```
+
+   Distributions are written to `dist/`.
+
+3. (Optional) Verify the wheel ships the SQL files and no templates:
+
+   ```bash
+   BRASA_BUILD_TEST=1 uv run pytest tests/test_packaging_wheel.py -v
+   ```
+
+4. (Optional) Smoke-test in a clean environment:
+
+   ```bash
+   python -m venv /tmp/brasa-smoke
+   /tmp/brasa-smoke/bin/pip install dist/brasa_core-*.whl
+   /tmp/brasa-smoke/bin/brasa --version
+   ```
+
+5. Publish:
+
+   ```bash
+   # TestPyPI dry-run first (recommended)
+   uv publish --publish-url https://test.pypi.org/legacy/ --token "$TEST_PYPI_TOKEN"
+   # Production
+   uv publish --token "$UV_PUBLISH_TOKEN"
+   ```
+
+   A PyPI API token can be supplied via `--token`, the `UV_PUBLISH_TOKEN`
+   environment variable, or `~/.pypirc`.

brasa_core-0.5.0/README.md

@@ -0,0 +1,146 @@
+# brasa
+
+A template-driven ETL engine: download, parse, transform and query datasets
+described by YAML templates. The official templates —
+[brasa-templates](https://github.com/wilsonfreitas/brasa-templates) — cover
+Brazilian market and economic data (B3, ANBIMA, CVM, BCB).
+
+## Installation
+
+Install from PyPI as `brasa-core` (the import name and the command are `brasa`),
+then run `brasa init` to choose a data directory and get the templates:
+
+    pip install brasa-core
+    brasa init
+
+Until 0.4.0 the distribution was called `brasa-marketdata`; since 0.5.0 it is
+`brasa-core`, and the templates live in brasa-templates instead of the package.
+
+Or directly from GitHub (latest `main`):
+
+    pip install git+https://github.com/wilsonfreitas/brasa.git
+
+With `uv`:
+
+    uv add brasa-core
+    uv add git+https://github.com/wilsonfreitas/brasa.git
+
+## Data directory (`brasa init` / `BRASA_DATA_PATH`)
+
+brasa stores everything — raw downloads, parsed parquet, and the metadata DB —
+under a single *brasa home*. On a fresh install, configure it once:
+
+    brasa init
+
+This suggests a platform default (e.g. `~/.local/share/brasa` on Linux),
+lets you confirm or type another path, and persists the choice in
+`~/.config/brasa/config.toml`. It also sets up your templates: it clones the
+official [brasa-templates](https://github.com/wilsonfreitas/brasa-templates)
+repo into `<data path>/templates` (use `--templates <url|path>` for a fork or
+a folder you already have). Get template updates later with
+`brasa update-templates`. Non-interactive variants:
+
+    brasa init --data-path /data/brasa   # explicit path, no prompt
+    brasa init --yes                     # accept the default, no prompt
+
+The `BRASA_DATA_PATH` environment variable, when set, always takes
+precedence over the config file — useful for CI, containers, tests, and
+one-off overrides. Commands that read or write data fail with a clear
+error until one of the two is configured; template introspection commands
+(`list-templates`, `deps`, `graph`) work without any configuration.
+
+Templates can live in your own template folders too (`BRASA_TEMPLATE_PATH`
+or `template_path` in the config file). Templates are code — they can run
+Python — so only add folders you trust; see
+[Template folders](docs/TEMPLATES.md#template-folders).
+
+## Changelog
+
+### Deterministic Download Status Codes
+
+Every download attempt is now classified with a single, unambiguous status code
+persisted in the `download_trials` table:
+
+| Symbol | Name       | Trigger                                           |
+|--------|------------|---------------------------------------------------|
+| `.`    | PASSED     | Successful download                               |
+| `F`    | FAILED     | Expected failure (`DownloadException`)             |
+| `E`    | ERROR      | Unexpected exception                              |
+| `S`    | SKIPPED    | Skipped (cache hit / invalid / duplicated)         |
+| `D`    | DUPLICATED | Raw folder already exists                         |
+| `I`    | INVALID    | Content validation failure                        |
+| `W`    | WARNING    | Success with warnings                             |
+
+**DB migration**: Existing caches are upgraded automatically on startup.
+Legacy `downloaded=1` rows become `PASSED`; `downloaded=0` become `FAILED`.
+(The standalone migration script has been removed after being applied; it is
+recoverable from git history if ever needed.)
+
+See [docs/USER_GUIDE.md](docs/USER_GUIDE.md#download-status-codes) for full details.
+
+### Import Local Files (`import_marketdata` / `brasa import`)
+
+Files with no download URL — a one-off vendor file, a manually-provided upload,
+or a corrected file for backfill — can now be imported using the same
+validate → gzip → checksum-dedup → parse → store engine as `download`. Only
+the acquisition step changes: bytes are read from disk instead of HTTP.
+
+```bash
+# Backfill a single date into a template that normally downloads via HTTP
+brasa import b3-cotahist-daily --path /data/backfill/COTAHIST_D02012024.TXT --arg refdate=2024-01-02
+
+# Bulk import one file per business day using a date pattern
+brasa import my-daily-template --path '/data/prices/%Y-%m-%d.csv' --arg refdate=@2026-06-01:2026-06-30
+```
+
+```python
+from brasa import import_marketdata
+
+import_marketdata("b3-cotahist-daily", path="/data/backfill/COTAHIST_D02012024.TXT", refdate="2024-01-02")
+```
+
+See [docs/CLI.md](docs/CLI.md#import), [docs/API_REFERENCE.md](docs/API_REFERENCE.md#data-import),
+and [docs/TEMPLATES.md](docs/TEMPLATES.md#importing-local-files-importer) for full details.
+
+## Publishing to PyPI
+
+brasa is built with [hatchling](https://hatch.pypa.io/) and published manually
+under the distribution name `brasa-core` (the PyPI name `brasa` was already
+taken by an unrelated project; `import brasa` is unaffected; until 0.4.0 the
+name was `brasa-marketdata`). The SQL DDL ships inside the package
+(`brasa/files/sql/`); templates do not — they live in brasa-templates.
+
+1. Bump `version` in `pyproject.toml`.
+2. Build the wheel and sdist:
+
+   ```bash
+   uv build
+   ```
+
+   Distributions are written to `dist/`.
+
+3. (Optional) Verify the wheel ships the SQL files and no templates:
+
+   ```bash
+   BRASA_BUILD_TEST=1 uv run pytest tests/test_packaging_wheel.py -v
+   ```
+
+4. (Optional) Smoke-test in a clean environment:
+
+   ```bash
+   python -m venv /tmp/brasa-smoke
+   /tmp/brasa-smoke/bin/pip install dist/brasa_core-*.whl
+   /tmp/brasa-smoke/bin/brasa --version
+   ```
+
+5. Publish:
+
+   ```bash
+   # TestPyPI dry-run first (recommended)
+   uv publish --publish-url https://test.pypi.org/legacy/ --token "$TEST_PYPI_TOKEN"
+   # Production
+   uv publish --token "$UV_PUBLISH_TOKEN"
+   ```
+
+   A PyPI API token can be supplied via `--token`, the `UV_PUBLISH_TOKEN`
+   environment variable, or `~/.pypirc`.

brasa_core-0.5.0/brasa/__init__.py

@@ -0,0 +1,53 @@
+from .engine import (
+    CacheManager,
+    DownloadPlan,
+    Verbosity,
+    download_marketdata,
+    execute_download_plan,
+    get_marketdata,
+    import_marketdata,
+    process_etl,
+    process_marketdata,
+    retrieve_template,
+)
+from .queries import (
+    BrasaDB,
+    create_all_views,
+    describe,
+    describe_dataset,
+    get_dataset,
+    get_industry_sectors,
+    get_prices,
+    get_returns,
+    get_symbols,
+    list_datasets,
+    show,
+    sql,
+    write_dataset,
+)
+
+__all__ = [
+    "BrasaDB",
+    "CacheManager",
+    "DownloadPlan",
+    "Verbosity",
+    "create_all_views",
+    "describe",
+    "describe_dataset",
+    "download_marketdata",
+    "execute_download_plan",
+    "get_dataset",
+    "get_industry_sectors",
+    "get_marketdata",
+    "get_prices",
+    "get_returns",
+    "get_symbols",
+    "import_marketdata",
+    "list_datasets",
+    "process_etl",
+    "process_marketdata",
+    "retrieve_template",
+    "show",
+    "sql",
+    "write_dataset",
+]

brasa_core-0.5.0/brasa/__main__.py

@@ -0,0 +1,6 @@
+"""Allow ``python -m brasa`` — used by the orchestrator's subprocess isolation."""
+
+from brasa.cli import main
+
+if __name__ == "__main__":
+    main()