brasa-marketdata 0.1.0__tar.gz

brasa_marketdata-0.1.0/.gitignore

@@ -0,0 +1,143 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Git worktrees
+.worktrees/
+
+data/big-files/BDIN.zip
+data/big-files/OpcoesAcoesEmAberto.zip
+legacy/
+.brasa-cache/
+.vscode/launch.json
+
+# Generated skill lockfile and script outputs
+skills-lock.json
+scripts/out/
+tmp/

brasa_marketdata-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Wilson Freitas
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

brasa_marketdata-0.1.0/PKG-INFO

@@ -0,0 +1,158 @@
+Metadata-Version: 2.4
+Name: brasa-marketdata
+Version: 0.1.0
+Summary: Python library to extract finance market data from brazillian financial institutions: B3, ANBIMA, Tesouro Direto, CVM.
+Author-email: wilsonfreitas <wilson.freitas@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: beautifulsoup4>=4.12.2
+Requires-Dist: bizdays>=1.0.15
+Requires-Dist: duckdb>=1.2.0
+Requires-Dist: html5lib>=1.1
+Requires-Dist: lxml>=4.9.2
+Requires-Dist: numpy>=2.0.0
+Requires-Dist: openpyxl>=3.1.5
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: progressbar2>=4.3.2
+Requires-Dist: pyarrow>=19.0.0
+Requires-Dist: python-bcb>=0.3.2
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: regexparser>=0.1.0
+Requires-Dist: requests>=2.28.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: xlrd>=2.0.1
+Description-Content-Type: text/markdown
+
+# brasa
+
+Extract finance market data from brazillian financial institutions: B3, ANBIMA, Tesouro Direto, CVM.
+
+## Installation
+
+Install from PyPI:
+
+    pip install brasa
+
+Or directly from GitHub (latest `main`):
+
+    pip install git+https://github.com/wilsonfreitas/brasa.git
+
+With `uv`:
+
+    uv add brasa
+    uv add git+https://github.com/wilsonfreitas/brasa.git
+
+## Cache location (`BRASA_DATA_PATH`)
+
+brasa stores everything — raw downloads, parsed parquet, and the metadata DB —
+under a single *brasa home*, resolved from the `BRASA_DATA_PATH` environment
+variable (falling back to `./.brasa-cache`).
+
+A common setup keeps one central home exported globally (e.g. in `~/.env.local`).
+For a one-off, project-local dataset, create a separate home and point at it for
+the current shell only — a per-shell `export` shadows the global value:
+
+    BRASA_DATA_PATH=./brasa-home uv run python -m brasa.cli setup
+
+`setup` then prints the exact line to activate that home:
+
+    Brasa home ready at /abs/path/to/brasa-home
+
+    To use this home in your shell:
+      export BRASA_DATA_PATH="/abs/path/to/brasa-home"
+
+Run that `export` and every `brasa` command in the session uses the project-local
+home; open a new shell to return to the central one.
+
+## Changelog
+
+### Deterministic Download Status Codes
+
+Every download attempt is now classified with a single, unambiguous status code
+persisted in the `download_trials` table:
+
+| Symbol | Name       | Trigger                                           |
+|--------|------------|---------------------------------------------------|
+| `.`    | PASSED     | Successful download                               |
+| `F`    | FAILED     | Expected failure (`DownloadException`)             |
+| `E`    | ERROR      | Unexpected exception                              |
+| `S`    | SKIPPED    | Skipped (cache hit / invalid / duplicated)         |
+| `D`    | DUPLICATED | Raw folder already exists                         |
+| `I`    | INVALID    | Content validation failure                        |
+| `W`    | WARNING    | Success with warnings                             |
+
+**DB migration**: Existing caches are upgraded automatically on startup.
+Legacy `downloaded=1` rows become `PASSED`; `downloaded=0` become `FAILED`.
+A standalone migration script is also available:
+
+```bash
+uv run python scripts/migrate_download_trials_status.py
+```
+
+See [docs/USER_GUIDE.md](docs/USER_GUIDE.md#download-status-codes) for full details.
+
+### Import Local Files (`import_marketdata` / `brasa import`)
+
+Files with no download URL — a one-off vendor file, a manually-provided upload,
+or a corrected file for backfill — can now be imported using the same
+validate → gzip → checksum-dedup → parse → store engine as `download`. Only
+the acquisition step changes: bytes are read from disk instead of HTTP.
+
+```bash
+# Backfill a single date into a template that normally downloads via HTTP
+brasa import b3-cotahist-daily --path /data/backfill/COTAHIST_D02012024.TXT --arg refdate=2024-01-02
+
+# Bulk import one file per business day using a date pattern
+brasa import my-daily-template --path '/data/prices/%Y-%m-%d.csv' --arg refdate=@2026-06-01:2026-06-30
+```
+
+```python
+from brasa import import_marketdata
+
+import_marketdata("b3-cotahist-daily", path="/data/backfill/COTAHIST_D02012024.TXT", refdate="2024-01-02")
+```
+
+See [docs/CLI.md](docs/CLI.md#import), [docs/API_REFERENCE.md](docs/API_REFERENCE.md#data-import),
+and [docs/TEMPLATES.md](docs/TEMPLATES.md#importing-local-files-importer) for full details.
+
+## Publishing to PyPI
+
+brasa is built with [hatchling](https://hatch.pypa.io/) and published manually.
+Templates and SQL DDL are bundled inside the package (`brasa/files/`), so the
+built wheel is self-contained.
+
+1. Bump `version` in `pyproject.toml` (the repo ships a `0.0.1` placeholder).
+2. Build the wheel and sdist:
+
+   ```bash
+   uv build
+   ```
+
+   Distributions are written to `dist/`.
+
+3. (Optional) Verify the wheel bundles the data files:
+
+   ```bash
+   BRASA_BUILD_TEST=1 uv run pytest tests/test_packaging_wheel.py -v
+   ```
+
+4. (Optional) Smoke-test in a clean environment:
+
+   ```bash
+   python -m venv /tmp/brasa-smoke
+   /tmp/brasa-smoke/bin/pip install dist/brasa-*.whl
+   /tmp/brasa-smoke/bin/python -c "import brasa; from brasa.engine.template import list_templates; print(len(list_templates()))"
+   ```
+
+5. Publish:
+
+   ```bash
+   # TestPyPI dry-run first (recommended)
+   uv publish --publish-url https://test.pypi.org/legacy/ --token "$TEST_PYPI_TOKEN"
+   # Production
+   uv publish --token "$UV_PUBLISH_TOKEN"
+   ```
+
+   A PyPI API token can be supplied via `--token`, the `UV_PUBLISH_TOKEN`
+   environment variable, or `~/.pypirc`.

brasa_marketdata-0.1.0/README.md

@@ -0,0 +1,132 @@
+# brasa
+
+Extract finance market data from brazillian financial institutions: B3, ANBIMA, Tesouro Direto, CVM.
+
+## Installation
+
+Install from PyPI:
+
+    pip install brasa
+
+Or directly from GitHub (latest `main`):
+
+    pip install git+https://github.com/wilsonfreitas/brasa.git
+
+With `uv`:
+
+    uv add brasa
+    uv add git+https://github.com/wilsonfreitas/brasa.git
+
+## Cache location (`BRASA_DATA_PATH`)
+
+brasa stores everything — raw downloads, parsed parquet, and the metadata DB —
+under a single *brasa home*, resolved from the `BRASA_DATA_PATH` environment
+variable (falling back to `./.brasa-cache`).
+
+A common setup keeps one central home exported globally (e.g. in `~/.env.local`).
+For a one-off, project-local dataset, create a separate home and point at it for
+the current shell only — a per-shell `export` shadows the global value:
+
+    BRASA_DATA_PATH=./brasa-home uv run python -m brasa.cli setup
+
+`setup` then prints the exact line to activate that home:
+
+    Brasa home ready at /abs/path/to/brasa-home
+
+    To use this home in your shell:
+      export BRASA_DATA_PATH="/abs/path/to/brasa-home"
+
+Run that `export` and every `brasa` command in the session uses the project-local
+home; open a new shell to return to the central one.
+
+## Changelog
+
+### Deterministic Download Status Codes
+
+Every download attempt is now classified with a single, unambiguous status code
+persisted in the `download_trials` table:
+
+| Symbol | Name       | Trigger                                           |
+|--------|------------|---------------------------------------------------|
+| `.`    | PASSED     | Successful download                               |
+| `F`    | FAILED     | Expected failure (`DownloadException`)             |
+| `E`    | ERROR      | Unexpected exception                              |
+| `S`    | SKIPPED    | Skipped (cache hit / invalid / duplicated)         |
+| `D`    | DUPLICATED | Raw folder already exists                         |
+| `I`    | INVALID    | Content validation failure                        |
+| `W`    | WARNING    | Success with warnings                             |
+
+**DB migration**: Existing caches are upgraded automatically on startup.
+Legacy `downloaded=1` rows become `PASSED`; `downloaded=0` become `FAILED`.
+A standalone migration script is also available:
+
+```bash
+uv run python scripts/migrate_download_trials_status.py
+```
+
+See [docs/USER_GUIDE.md](docs/USER_GUIDE.md#download-status-codes) for full details.
+
+### Import Local Files (`import_marketdata` / `brasa import`)
+
+Files with no download URL — a one-off vendor file, a manually-provided upload,
+or a corrected file for backfill — can now be imported using the same
+validate → gzip → checksum-dedup → parse → store engine as `download`. Only
+the acquisition step changes: bytes are read from disk instead of HTTP.
+
+```bash
+# Backfill a single date into a template that normally downloads via HTTP
+brasa import b3-cotahist-daily --path /data/backfill/COTAHIST_D02012024.TXT --arg refdate=2024-01-02
+
+# Bulk import one file per business day using a date pattern
+brasa import my-daily-template --path '/data/prices/%Y-%m-%d.csv' --arg refdate=@2026-06-01:2026-06-30
+```
+
+```python
+from brasa import import_marketdata
+
+import_marketdata("b3-cotahist-daily", path="/data/backfill/COTAHIST_D02012024.TXT", refdate="2024-01-02")
+```
+
+See [docs/CLI.md](docs/CLI.md#import), [docs/API_REFERENCE.md](docs/API_REFERENCE.md#data-import),
+and [docs/TEMPLATES.md](docs/TEMPLATES.md#importing-local-files-importer) for full details.
+
+## Publishing to PyPI
+
+brasa is built with [hatchling](https://hatch.pypa.io/) and published manually.
+Templates and SQL DDL are bundled inside the package (`brasa/files/`), so the
+built wheel is self-contained.
+
+1. Bump `version` in `pyproject.toml` (the repo ships a `0.0.1` placeholder).
+2. Build the wheel and sdist:
+
+   ```bash
+   uv build
+   ```
+
+   Distributions are written to `dist/`.
+
+3. (Optional) Verify the wheel bundles the data files:
+
+   ```bash
+   BRASA_BUILD_TEST=1 uv run pytest tests/test_packaging_wheel.py -v
+   ```
+
+4. (Optional) Smoke-test in a clean environment:
+
+   ```bash
+   python -m venv /tmp/brasa-smoke
+   /tmp/brasa-smoke/bin/pip install dist/brasa-*.whl
+   /tmp/brasa-smoke/bin/python -c "import brasa; from brasa.engine.template import list_templates; print(len(list_templates()))"
+   ```
+
+5. Publish:
+
+   ```bash
+   # TestPyPI dry-run first (recommended)
+   uv publish --publish-url https://test.pypi.org/legacy/ --token "$TEST_PYPI_TOKEN"
+   # Production
+   uv publish --token "$UV_PUBLISH_TOKEN"
+   ```
+
+   A PyPI API token can be supplied via `--token`, the `UV_PUBLISH_TOKEN`
+   environment variable, or `~/.pypirc`.

brasa_marketdata-0.1.0/brasa/__init__.py

@@ -0,0 +1,83 @@
+from .engine import (
+    CacheManager,
+    DatasetCatalog,
+    DatasetInfo,
+    DownloadPlan,
+    DownloadPlanReport,
+    ExecutionPlan,
+    ExecutionStep,
+    MigrationReport,
+    OrchestratorReport,
+    PipelineOrchestrator,
+    RunAllReport,
+    TaskReport,
+    TemplateDependencyGraph,
+    Verbosity,
+    download_marketdata,
+    execute_download_plan,
+    get_dependency_graph,
+    get_execution_plan,
+    get_marketdata,
+    import_marketdata,
+    process_etl,
+    process_marketdata,
+    retrieve_template,
+    sync_catalog_from_disk,
+)
+from .queries import (
+    BrasaDB,
+    create_all_views,
+    describe,
+    describe_dataset,
+    get_dataset,
+    get_industry_sectors,
+    get_prices,
+    get_returns,
+    get_symbols,
+    list_datasets,
+    list_sql_tables,
+    show,
+    sql,
+    write_dataset,
+)
+
+__all__ = [
+    "BrasaDB",
+    "CacheManager",
+    "DatasetCatalog",
+    "DatasetInfo",
+    "DownloadPlan",
+    "DownloadPlanReport",
+    "ExecutionPlan",
+    "ExecutionStep",
+    "MigrationReport",
+    "OrchestratorReport",
+    "PipelineOrchestrator",
+    "RunAllReport",
+    "TaskReport",
+    "TemplateDependencyGraph",
+    "Verbosity",
+    "create_all_views",
+    "describe",
+    "describe_dataset",
+    "download_marketdata",
+    "execute_download_plan",
+    "get_dataset",
+    "get_dependency_graph",
+    "get_execution_plan",
+    "get_industry_sectors",
+    "get_marketdata",
+    "get_prices",
+    "get_returns",
+    "get_symbols",
+    "import_marketdata",
+    "list_datasets",
+    "list_sql_tables",
+    "process_etl",
+    "process_marketdata",
+    "retrieve_template",
+    "show",
+    "sql",
+    "sync_catalog_from_disk",
+    "write_dataset",
+]