regulated-eval 1.0.0__tar.gz

regulated_eval-1.0.0/.gigacode/.gitignore

@@ -0,0 +1,7 @@
+node_modules
+package.json
+package-lock.json
+pnpm-lock.yaml
+bun.lock
+yarn.lock
+.gitignore

regulated_eval-1.0.0/.gigacode/rules/issue-sync.md

@@ -0,0 +1,8 @@
+# Синхронизация статуса Issue
+
+После завершения работы над задачей (ID вида T-xxx), когда код написан и проверен, необходимо:
+1. Найти номер соответствующего Issue на GitHub с помощью команды в терминале: 
+   `gh issue list --search "T-xxx" --json number -q ".[0].number"`
+2. Закрыть найденный Issue командой:
+   `gh issue close <номер> --comment "Реализовано в рамках задачи"`
+3. Если Issue не найден, сообщить об этом пользователю.

regulated_eval-1.0.0/.github/workflows/docs.yml

@@ -0,0 +1,20 @@
+name: docs-check
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  docs-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check docs exist
+        run: |
+          test -f docs/PRD.md || { echo "Missing docs/PRD.md"; exit 1; }
+          test -f docs/methodology.md || { echo "Missing docs/methodology.md"; exit 1; }
+          test -f docs/case_study.md || { echo "Missing docs/case_study.md"; exit 1; }
+          echo "All required docs present"

regulated_eval-1.0.0/.github/workflows/lint.yml

@@ -0,0 +1,32 @@
+name: lint
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: ruff check
+        run: ruff check src/ tests/
+
+      - name: ruff format --check
+        run: ruff format --check src/ tests/
+
+      - name: mypy
+        run: mypy src/

regulated_eval-1.0.0/.github/workflows/matrix.yml

@@ -0,0 +1,31 @@
+name: matrix
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  matrix:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.11", "3.12"]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Run pytest
+        run: pytest -q

regulated_eval-1.0.0/.github/workflows/test.yml

@@ -0,0 +1,26 @@
+name: test
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Run pytest with coverage
+        run: pytest --cov=src/regulated_eval --cov-report=term-missing --cov-fail-under=80 -q

regulated_eval-1.0.0/.gitignore

@@ -0,0 +1,67 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+*.egg
+MANIFEST
+
+# Virtual environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+
+# Testing
+.coverage
+htmlcov/
+.pytest_cache/
+.tox/
+.coverage.*
+*.cover
+*.log
+
+# PyInstaller
+*.manifest
+*.spec
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Regulatory data (contains PII)
+datasets/*.jsonl
+datasets/*.json
+
+# Secrets
+.env
+.env.*
+!/.env.example

regulated_eval-1.0.0/CHANGELOG.md

@@ -0,0 +1,51 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [1.0.0] - 2026-11-08
+
+### Added
+- **Core Framework**
+  - Golden set management (JSONL, Pydantic validation, semver, content hash)
+  - LLM-as-a-judge with calibration (Cohen's kappa + bootstrap CI)
+  - Metrics: hallucination_rate, citation_accuracy, refusal_correctness, factual_consistency
+  - Human-in-the-loop (Streamlit UI, hash-chain annotations)
+  - A/B testing (Wilcoxon, bootstrap CI, Cohen's d)
+  - Adapters: YandexGPT, GigaChat, OpenAI-compatible (opt-in)
+  - PII redaction (Presidio + regex for RU: ИНН, СНИЛС, passport, phone, address)
+  - Prompt-injection filter (allowlist + heuristics)
+  - OTel instrumentation (spans, metrics, audit-logs) with redaction middleware
+  - Markdown/HTML reports with delta comparison
+  - CLI + Library API (Typer + Python API)
+
+- **Observability**
+  - Spans: `eval.run`, `judge.evaluate`, `adapter.call`
+  - Metrics: `eval.run.duration`, `eval.run.failed`, `llm.request.duration`, `judge.calibration.kappa`, `hallucination.rate`, `citation_accuracy`
+  - Audit logs: `pii_redacted`, `injection_suspected`, `conflict_resolved`, `ab_test_completed`
+  - SLO: eval.run p95 ≤ 300s, judge p95 ≤ 5000ms, availability 99.0%
+
+- **CI/CD**
+  - GitHub Actions: lint, test, matrix (ubuntu/macos/windows × py3.11/3.12), docs-check
+  - Coverage target: ≥ 80%
+
+- **Documentation**
+  - PRD, methodology, case_study, 152fz compliance guide
+  - OpenSpec artifacts: proposal, specs, design, threat-model, ADR, API contract, test-specs, tasks, apply
+  - Traceability matrix, 15 threat models, 5 ADRs
+
+- **Tests**
+  - 211 tests: unit, integration, E2E, performance, observability
+  - Security tests: secrets, hash-chain, PII, injection, provider allowlist
+
+### Security
+- Default provider allowlist: Russian providers only (YandexGPT, GigaChat)
+- Foreign provider double opt-in (`--allow-foreign-provider` + `--i-understand-152fz`)
+- PII redaction before LLM submission (recall ≥ 99%)
+- Audit logging without PII values
+- OTel attribute redaction via `RedactingSpanProcessor`
+- Hash-chain annotations for tamper detection

regulated_eval-1.0.0/CONTRIBUTING.md

@@ -0,0 +1,86 @@
+# Contributing Guide
+
+## Getting Started
+
+1. Fork the repository
+2. Clone your fork: `git clone https://github.com/YOUR_USERNAME/regulated-llm-eval-framework.git`
+3. Create a feature branch: `git checkout -b feature/my-feature`
+4. Make changes and commit
+5. Push to your fork: `git push origin feature/my-feature`
+6. Open a Pull Request
+
+## Development Setup
+
+```bash
+git clone https://github.com/realrvs/regulated-llm-eval-framework.git
+cd regulated-llm-eval-framework
+
+python -m venv .venv
+source .venv/bin/activate  # Windows: .venv\Scripts\activate
+
+pip install -e ".[dev]"
+```
+
+## Running Tests
+
+```bash
+# All tests
+pytest -q
+
+# With coverage
+pytest --cov=src/regulated_eval --cov-report=term-missing
+
+# Target coverage: ≥ 80% for judges, metrics, ab, datasets
+```
+
+## Code Style
+
+```bash
+# Lint
+ruff check .
+
+# Format
+ruff format .
+
+# Type checking
+mypy src/regulated_eval
+```
+
+All checks must pass before PR merge.
+
+## Commit Messages
+
+Follow [Conventional Commits](https://www.conventionalcommits.org/):
+
+```
+feat(cli): add calibrate command
+fix(pipeline): handle empty results
+docs: update PRD
+test(security): add hash-chain tests
+ci: add GitHub Actions workflows
+```
+
+Types: `feat`, `fix`, `docs`, `test`, `ci`, `refactor`, `chore`
+
+## Pull Request Checklist
+
+- [ ] Tests pass (`pytest -q`)
+- [ ] Coverage ≥ 80% for changed modules
+- [ ] `ruff check` and `ruff format` clean
+- [ ] `mypy` passes
+- [ ] Commit message follows Conventional Commits
+- [ ] Updated documentation if API changed
+- [ ] For security changes: updated `threat-model.md` and added `TM Ref` in tests
+
+## Security Contributions
+
+For security-related changes:
+
+1. Update `threat-model/threat-model.md` with new threat analysis
+2. Add test with `TM Ref` in docstring
+3. Update `docs/152fz.md` if 152-FZ compliance affected
+4. No secrets in code or tests
+
+## Questions?
+
+Open an issue or use GitHub Discussions.

regulated_eval-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Regulated LLM Eval Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

regulated_eval-1.0.0/PKG-INFO

@@ -0,0 +1,272 @@
+Metadata-Version: 2.5
+Name: regulated-eval
+Version: 1.0.0
+Summary: Regulated LLM Eval Framework — evaluation for LLMs in regulated domains (tax, accounting, legal) with 152-FZ compliance
+Project-URL: Homepage, https://github.com/regulated-llm-eval/regulated-eval
+Project-URL: Repository, https://github.com/regulated-llm-eval/regulated-eval
+Project-URL: Documentation, https://github.com/regulated-llm-eval/regulated-eval#readme
+Project-URL: Bug Tracker, https://github.com/regulated-llm-eval/regulated-eval/issues
+Author: Regulated LLM Eval Contributors
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: Pydantic :: 2
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.11
+Requires-Dist: httpx>=0.27.0
+Requires-Dist: jinja2>=3.1
+Requires-Dist: numpy>=1.26
+Requires-Dist: opentelemetry-api>=1.25.0
+Requires-Dist: opentelemetry-exporter-otlp-proto-grpc>=1.25.0
+Requires-Dist: opentelemetry-instrumentation-httpx>=0.46b0
+Requires-Dist: opentelemetry-sdk>=1.25.0
+Requires-Dist: pandas>=2.0
+Requires-Dist: pydantic-settings>=2.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: scikit-learn>=1.3
+Requires-Dist: scipy>=1.11
+Requires-Dist: tenacity>=8.2.0
+Requires-Dist: typer[all]>=0.12.0
+Provides-Extra: dev
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest-cov>=5.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.4; extra == 'dev'
+Requires-Dist: types-jinja2; extra == 'dev'
+Provides-Extra: presidio
+Requires-Dist: presidio-analyzer>=2.2; extra == 'presidio'
+Requires-Dist: presidio-anonymizer>=2.2; extra == 'presidio'
+Provides-Extra: streamlit
+Requires-Dist: streamlit>=1.30; extra == 'streamlit'
+Description-Content-Type: text/markdown
+
+# regulated-llm-eval
+
+> Eval-фреймворк для LLM в regulated-доменах (налоги, бухгалтерия, право).
+> Калиброванный LLM-as-a-judge, метрики галлюцинаций и citation accuracy, A/B-тесты промптов, PII-redaction под 152-ФЗ, поддержка YandexGPT и GigaChat.
+
+[![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+[![OpenSpec v1.x](https://img.shields.io/badge/OpenSpec-v1.x-green.svg)](docs/PRD.md)
+
+---
+
+## Quickstart (≤ 5 минут)
+
+### 1. Установка
+
+```bash
+pip install regulated-eval
+```
+
+### 2. API-ключ провайдера
+
+```bash
+# YandexGPT (российский, соответствует 152-ФЗ)
+export YANDEXGPT_API_KEY="your-key"
+
+# GigaChat (российский, соответствует 152-ФЗ)
+export GIGACHAT_API_KEY="your-key"
+```
+
+### 3. Первый прогон
+
+```bash
+# Загрузка sample golden set (20 вопросов по УСН/НДС/взносам)
+regulated-eval dataset load --path datasets/tax_qa_v1/sample.jsonl --version 0.1.0
+
+# Прогон через YandexGPT + judge
+regulated-eval run \
+  --dataset tax_qa_v1@0.1.0 \
+  --model yandexgpt \
+  --judge-model yandexgpt \
+  --report md
+
+# Отчёт: reports/<run_id>.md
+```
+
+### 4. Калибровка (обязательно перед публикацией отчёта)
+
+```bash
+regulated-eval calibrate \
+  --dataset calibration_v1 \
+  --judge-model yandexgpt \
+  --threshold 0.6
+# → PASS (kappa ≥ 0.6) или FAIL + рекомендации
+```
+
+### 5. A/B-сравнение моделей
+
+```bash
+regulated-eval ab \
+  --model-a yandexgpt \
+  --model-b gigachat \
+  --dataset tax_qa_v1@0.1.0 \
+  --judge-model yandexgpt
+```
+
+## Что внутри
+
+| Возможность | Что даёт |
+|---|---|
+| **Golden set management** | JSONL + Pydantic-схема, semver + content hash, воспроизводимость прогонов |
+| **LLM-as-a-judge с калибровкой** | Score 1–5 + бинарные флаги (hallucination, wrong citation, incorrect refusal), Cohen's kappa с CI, repair-retry на невалидный JSON |
+| **Метрики** | `hallucination_rate`, `citation_accuracy`, `refusal_correctness`, `factual_consistency`, агрегация по category/difficulty/model |
+| **Human-in-the-loop** | Streamlit UI для экспертов, hash-цепочка аннотаций (защита от подмены), CLI для разрешения конфликтов |
+| **A/B-тесты** | Bootstrap CI, Wilcoxon, Cohen's d, интерпретация `not_significant` |
+| **Адаптеры LLM** | YandexGPT, GigaChat, OpenAI-compatible (с явным opt-in под 152-ФЗ) |
+| **PII-redaction** | Presidio + кастомные regex для РФ (ИНН, СНИЛС, паспорт, телефон, адрес), audit-лог без значений |
+| **Prompt-injection фильтр** | Allowlist доменов + эвристики для retrieved-документов (RAG) |
+| **Observability** | OTel-инструментация (spans, metrics, audit-logs) с redaction-middleware, SLO: eval-run p95 ≤ 300s, judge p95 ≤ 5000ms |
+| **Отчёты** | Markdown / HTML с метриками, top-N ошибок, delta между прогонами |
+| **CLI + Library API** | Typer CLI и Python API — одинаковый функционал |
+
+## Архитектура
+
+```mermaid
+flowchart LR
+    CLI[CLI typer] --> Orch[EvalOrchestrator]
+    Lib[Library API] --> Orch
+    Orch --> DS[datasets<br/>loader + versioning]
+    Orch --> Sec[security<br/>pii_redactor + injection_filter]
+    Orch --> Ad[adapters<br/>yandexgpt / gigachat / openai]
+    Orch --> J[judges<br/>judge + calibration]
+    Orch --> M[metrics<br/>hallucination + citation]
+    Orch --> R[reports<br/>markdown + html]
+    J --> Sec
+    Ad --> Sec
+    Orch --> Obs[observability<br/>otel + redaction]
+    Human[Streamlit UI] --> J
+    Human --> Ann[annotation_store<br/>hash-chain]
+    Ann --> Orch
+```
+
+## 152-ФЗ и foreign providers
+
+По умолчанию разрешены **только российские провайдеры** (YandexGPT, GigaChat) — они соответствуют 152-ФЗ. Для OpenAI-compatible требуется **двойной opt-in**:
+
+```bash
+regulated-eval run \
+  --dataset tax_qa_v1@0.1.0 \
+  --model openai_compat \
+  --allow-foreign-provider \
+  --i-understand-152fz
+```
+
+Или через env (для CI, с предупреждением в stderr):
+
+```bash
+export REGULATED_EVAL_ALLOW_FOREIGN_PROVIDER=true
+export REGULATED_EVAL_ACK_152FZ=true
+```
+
+Каждое разрешение фиксируется в audit-логе `provider.foreign.allowed`. Подробнее — в [`docs/152fz.md`](docs/152fz.md).
+
+## Метрики качества
+
+| Метрика | Как считается | Целевое значение |
+|---|---|---|
+| `judge.calibration.kappa` | Cohen's kappa (judge vs эксперт) | ≥ 0.6 |
+| `hallucination_rate` | LLM-judge + экспертный сэмпл | ниже baseline на ≥ 50% |
+| `citation_accuracy` | Точное/частичное совпадение + judge | ≥ 0.9 для regulated-домена |
+| `refusal_correctness` | Бинарная классификация | ≥ 0.95 |
+| `judge.parse_error.rate` | Доля невалидных JSON от judge | ≤ 5% |
+| `eval.run.duration` p95 | Histogram | ≤ 300s (150 вопросов) |
+| `llm.request.duration{role=judge}` p95 | Histogram | ≤ 5000ms |
+
+Полная методология — в [`docs/methodology.md`](docs/methodology.md).
+
+## Структура репозитория
+
+```
+regulated-llm-eval/
+├── README.md                    # этот файл
+├── proposal.md                  # OpenSpec: why / what / impact
+├── context.md                   # Traceability Matrix, принятые ADR
+├── specs/eval-pipeline/spec.md  # FR/NFR + Observability Contract
+├── design/design.md             # архитектура, потоки, точки инструментирования
+├── threat-model/threat-model.md # STRIDE + OWASP LLM, 15 угроз, 9 Trust Boundaries
+├── adr/                         # 5 ADR (PII, allowlist, judge, OTel, hash-chain)
+├── api/api-contract.yaml        # CLI + Library API + JSON Schema
+├── test-specs/eval-pipeline/tests.md  # 64 теста, coverage matrix
+├── tasks/tasks.md               # 96 задач, ~191 SP
+├── apply/apply.md               # миграции, feature flags, observability verification
+├── docs/
+│   ├── PRD.md
+│   ├── methodology.md
+│   ├── case_study.md
+│   └── 152fz.md
+├── scripts/                     # PowerShell-скрипты для генерации артефактов
+├── datasets/
+│   ├── tax_qa_v1/
+│   └── calibration_v1/
+└── src/regulated_eval/          # (реализация — по tasks.md)
+```
+
+## Разработка
+
+### Требования
+
+- Python 3.11+
+- Git
+
+### Установка для разработки
+
+```bash
+git clone https://github.com/realrvs/regulated-llm-eval-framework.git
+cd regulated-llm-eval-framework
+
+python -m venv .venv
+source .venv/bin/activate  # Windows: .venv\Scripts\activate
+
+pip install -e ".[dev]"
+```
+
+### Тесты
+
+```bash
+pytest --cov=regulated_eval --cov-report=term-missing
+# Целевое покрытие: ≥ 80% для judges, metrics, ab, datasets
+```
+
+### Линтеры
+
+```bash
+ruff check .
+mypy src/regulated_eval
+```
+
+## Roadmap
+
+- [x] OpenSpec pipeline (proposal → specs → design → threat-model → adr → api-contract → test-specs → tasks → apply)
+- [x] MVP implementation (Foundation, Datasets, Adapters, Security, Judges, Metrics, Pipeline, Reports)
+- [x] Phase 6-10: Human-loop, A/B, Reports, Observability, Pipeline
+- [ ] Phase 11: Tests (T-1101..T-1114)
+- [ ] Phase 12: CI (T-1201..T-1205)
+- [ ] Phase 13: Docs (T-1301..T-1305)
+- [ ] Phase 14: Apply (T-1401..T-1403)
+- [ ] v1.0.0 release (PyPI)
+
+Полный план — в [`tasks/tasks.md`](tasks/tasks.md).
+
+## Contributing
+
+1. Fork → branch → PR.
+2. Убедитесь, что `pytest --cov` ≥ 80% и `ruff check` / `mypy` зелёные.
+3. Для security-изменений — обновите `threat-model.md` и добавьте тест с `TM Ref`.
+4. Для observability-изменений — обновите Observability Contract в `spec.md` и `test-specs`.
+
+## License
+
+MIT — см. [LICENSE](LICENSE).
+
+## Security
+
+See [SECURITY.md](SECURITY.md) for reporting vulnerabilities.