finlab-sentinel 0.1.5__tar.gz

finlab_sentinel-0.1.5/.github/workflows/ci.yml

@@ -0,0 +1,77 @@
+name: CI
+
+on:
+  push:
+    branches: [main, develop]
+  pull_request:
+    branches: [main]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "latest"
+
+      - name: Set up Python
+        run: uv python install 3.12
+
+      - name: Install dependencies
+        run: uv sync --extra dev
+
+      - name: Run ruff linter
+        run: uv run ruff check src/ tests/
+
+      - name: Run ruff formatter check
+        run: uv run ruff format --check src/ tests/
+
+      - name: Run mypy type checker
+        run: uv run mypy src/
+
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "latest"
+
+      - name: Set up Python ${{ matrix.python-version }}
+        run: uv python install ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: uv sync --extra dev
+
+      - name: Run tests with coverage
+        run: uv run pytest tests/ -v --cov=src/finlab_sentinel --cov-report=xml --cov-report=html --cov-fail-under=80 --junitxml=junit/test-results-${{ matrix.python-version }}.xml
+
+      - name: Upload coverage reports to Codecov
+        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12'
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+      - name: Upload test results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: test-results-${{ matrix.os }}-${{ matrix.python-version }}
+          path: |
+            junit/
+            htmlcov/

finlab_sentinel-0.1.5/.github/workflows/release.yml

@@ -0,0 +1,47 @@
+name: Publish to PyPI
+
+on:
+  workflow_dispatch:  # Manual trigger only
+
+jobs:
+  publish:
+    name: Build and Publish to PyPI
+    runs-on: ubuntu-latest
+    environment: pypi-publish  # Requires environment approval
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v4
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
+      with:
+        version: "latest"
+
+    - name: Set up Python
+      run: uv python install 3.12
+
+    - name: Install dependencies
+      run: uv sync --extra dev
+
+    - name: Verify project configuration
+      run: |
+        uv run python -c "import sys; sys.path.insert(0, 'src'); import finlab_sentinel; print(f'Package version: {finlab_sentinel.__version__}')"
+
+    - name: Build package
+      run: uv build
+
+    - name: Check package contents
+      run: uv run twine check dist/*
+
+    - name: Publish to PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        password: ${{ secrets.PYPI_API_TOKEN }}
+        verbose: true
+
+    - name: Upload build artifacts
+      uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: dist
+        path: dist/

finlab_sentinel-0.1.5/.gitignore

@@ -0,0 +1,146 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+junit/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+Pipfile.lock
+
+# PEP 582
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+
+# Local config
+sentinel.toml
+
+# uv
+uv.lock
+
+# finlab-sentinel specific
+.finlab-sentinel/

finlab_sentinel-0.1.5/PKG-INFO

@@ -0,0 +1,272 @@
+Metadata-Version: 2.4
+Name: finlab-sentinel
+Version: 0.1.5
+Summary: Defensive monitoring layer for finlab data.get API - detect unexpected data changes
+Project-URL: Homepage, https://github.com/iapcal/finlab-sentinel
+Project-URL: Repository, https://github.com/iapcal/finlab-sentinel
+Project-URL: Issues, https://github.com/iapcal/finlab-sentinel/issues
+Author-email: iapcal <chiyimin2018@gmail.com>
+License: MIT
+Keywords: backtesting,data-validation,finlab,monitoring,taiwan-stock
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Financial and Insurance Industry
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Office/Business :: Financial :: Investment
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: pandas>=2.0.0
+Requires-Dist: pyarrow>=14.0.0
+Requires-Dist: pydantic-settings>=2.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: typer>=0.12.0
+Requires-Dist: xxhash>=3.0.0
+Provides-Extra: dev
+Requires-Dist: mypy>=1.10.0; extra == 'dev'
+Requires-Dist: pandas-stubs>=2.0.0; extra == 'dev'
+Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
+Requires-Dist: pytest-mock>=3.0.0; extra == 'dev'
+Requires-Dist: pytest>=8.0.0; extra == 'dev'
+Requires-Dist: ruff>=0.4.0; extra == 'dev'
+Requires-Dist: twine>=5.0.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# finlab-sentinel
+
+![Python versions](https://img.shields.io/badge/python-3.10%20%7C%203.11%20%7C%203.12%20%7C%203.13-blue)
+![Windows](https://img.shields.io/badge/OS-Windows-0078D6?logo=windows&logoColor=white)
+![Linux](https://img.shields.io/badge/OS-Linux-FCC624?logo=linux&logoColor=black)
+![macOS](https://img.shields.io/badge/OS-macOS-000000?logo=apple&logoColor=white)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+[![CI](https://github.com/iapcal/finlab-sentinel/actions/workflows/ci.yml/badge.svg)](https://github.com/iapcal/finlab-sentinel/actions/workflows/ci.yml)
+[![coverage](https://img.shields.io/codecov/c/github/iapcal/finlab-sentinel)](https://codecov.io/gh/iapcal/finlab-sentinel)
+
+**finlab-sentinel** 是 [finlab](https://github.com/finlab-python/finlab) 套件的防禦層，用於監控 `data.get` API 的資料變化，防止未預期的資料異動影響回測或選股結果。
+
+## 功能特色
+
+- **自動比對**: 每次 `data.get` 時自動比對歷史資料
+- **滾動備份**: 保留 7 天（可配置）的備份資料
+- **智慧檢測**:
+  - 數值容差比對（可配置 rtol/atol）
+  - dtype 變更檢測
+  - NA 類型差異檢測（pd.NA vs np.nan vs None）
+- **彈性政策**:
+  - `append_only`: 只允許新增，不允許刪除或修改歷史
+  - `threshold`: 允許小幅度變更（如 10% 以內）
+  - 黑名單配置：指定可修改歷史的資料集
+- **可配置行為**:
+  - 拋出例外（預設）
+  - 警告並使用快取
+  - 警告並使用新資料
+- **Preprocess Hook**: 比對前預處理（如四捨五入），支援萬用字元模式
+- **通知機制**: 支援自訂 callback（如 LINE、email 通知）
+- **CLI 工具**: 管理備份、查看差異、接受新資料
+
+## 安裝
+
+```bash
+pip install finlab-sentinel
+```
+
+或使用 uv：
+
+```bash
+uv add finlab-sentinel
+```
+
+## 快速開始
+
+```python
+import finlab_sentinel
+
+# 啟用 sentinel
+finlab_sentinel.enable()
+
+# 正常使用 finlab
+from finlab import data
+close = data.get('price:收盤價')  # 自動備份並比對
+
+# 如果資料異常，會根據配置拋出例外或警告
+```
+
+## 配置
+
+建立 `sentinel.toml` 檔案：
+
+```toml
+[storage]
+path = "~/.finlab-sentinel/"
+retention_days = 7
+
+[comparison]
+rtol = 1e-5
+change_threshold = 0.10
+
+[comparison.policies]
+default_mode = "append_only"
+history_modifiable = ["fundamental_features:某些財報資料"]
+# 允許 NA→有值 的轉換（例如：預估資料後來補上）
+allow_na_to_value = ["price:收盤價"]
+
+[anomaly]
+behavior = "raise"  # raise | warn_return_cached | warn_return_new
+save_reports = true
+
+# 可選：設定通知 callback
+# callback = "myproject.notifications:send_line"
+```
+
+## CLI 使用
+
+```bash
+# 列出所有備份
+sentinel list
+
+# 清理過期備份
+sentinel cleanup --days 14
+
+# 查看資料差異
+sentinel diff "price:收盤價"
+
+# 接受新資料作為基準
+sentinel accept "price:收盤價" --reason "確認資料修正"
+
+# 匯出備份
+sentinel export "price:收盤價" -o ./backup.parquet
+```
+
+## 處理資料異常
+
+當檢測到資料異常時：
+
+```python
+from finlab_sentinel import DataAnomalyError
+
+try:
+    close = data.get('price:收盤價')
+except DataAnomalyError as e:
+    print(f"資料異常: {e.report.summary}")
+    # 檢查報告詳情
+    print(f"變動比例: {e.report.comparison_result.change_ratio:.1%}")
+
+    # 如果確認要接受新資料
+    from finlab_sentinel.core.interceptor import accept_current_data
+    accept_current_data('price:收盤價', reason="確認資料修正")
+```
+
+## Preprocess Hook
+
+Preprocess hook 讓你可以在比對前先對資料做預處理，例如四捨五入、排序欄位等。這在處理預期的浮點數精度差異時特別有用。
+
+**注意**: 預處理只用於比對，回傳給使用者的永遠是原始資料。
+
+```python
+import finlab_sentinel
+
+# 註冊特定 dataset 的 preprocess hook
+finlab_sentinel.register_preprocess_hook(
+    "price:收盤價",
+    lambda df: df.round(2)  # 四捨五入到小數第二位
+)
+
+# 支援萬用字元模式
+finlab_sentinel.register_preprocess_hook(
+    "price:*",  # 符合所有 price: 開頭的 dataset
+    lambda df: df.round(2)
+)
+
+# 也支援 ? 萬用字元（符合單一字元）
+finlab_sentinel.register_preprocess_hook(
+    "price:?",
+    lambda df: df.round(2)
+)
+
+finlab_sentinel.enable()
+
+# 使用 finlab
+from finlab import data
+close = data.get('price:收盤價')  # 比對時會先 round(2)，但回傳原始資料
+```
+
+### 進階用法
+
+```python
+import finlab_sentinel
+
+# 自訂預處理函式
+def normalize_for_comparison(df):
+    """標準化 DataFrame 以忽略預期的差異"""
+    df = df.copy()
+    # 四捨五入數值欄位
+    numeric_cols = df.select_dtypes(include=['float64', 'float32']).columns
+    df[numeric_cols] = df[numeric_cols].round(4)
+    # 排序欄位（忽略欄位順序差異）
+    df = df[sorted(df.columns)]
+    return df
+
+finlab_sentinel.register_preprocess_hook("fundamental_features:*", normalize_for_comparison)
+
+# 取消註冊
+finlab_sentinel.unregister_preprocess_hook("price:收盤價")
+
+# 清除所有 hooks
+finlab_sentinel.clear_preprocess_hooks()
+```
+
+### 優先順序
+
+當多個 pattern 都符合時，精確匹配優先於萬用字元匹配：
+
+```python
+finlab_sentinel.register_preprocess_hook("price:*", lambda df: df.round(1))
+finlab_sentinel.register_preprocess_hook("price:收盤價", lambda df: df.round(2))
+
+# "price:收盤價" 會使用 round(2)（精確匹配）
+# "price:開盤價" 會使用 round(1)（萬用字元匹配）
+```
+
+## 自訂通知
+
+```python
+def send_line_notification(report):
+    """當檢測到異常時發送 LINE 通知"""
+    import requests
+    requests.post(
+        "https://notify-api.line.me/api/notify",
+        headers={"Authorization": f"Bearer {LINE_TOKEN}"},
+        data={"message": f"finlab 資料異常: {report.summary}"}
+    )
+
+# 在 sentinel.toml 中設定
+# [anomaly]
+# callback = "myproject.notifications:send_line_notification"
+```
+
+## 開發
+
+```bash
+# Clone 專案
+git clone https://github.com/yourusername/finlab-sentinel
+cd finlab-sentinel
+
+# 使用 uv 安裝開發依賴
+uv sync --dev
+
+# 執行測試
+uv run pytest
+
+# 執行 lint
+uv run ruff check src/ tests/
+uv run mypy src/
+```
+
+## License
+
+MIT License