nirizan 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nirizan-0.1.0/.github/ISSUE_TEMPLATE/bug_report.yml +76 -0
- nirizan-0.1.0/.github/ISSUE_TEMPLATE/config.yml +8 -0
- nirizan-0.1.0/.github/ISSUE_TEMPLATE/contract_change.yml +89 -0
- nirizan-0.1.0/.github/ISSUE_TEMPLATE/documentation.yml +50 -0
- nirizan-0.1.0/.github/ISSUE_TEMPLATE/feature_request.yml +67 -0
- nirizan-0.1.0/.github/PULL_REQUEST_TEMPLATE.md +78 -0
- nirizan-0.1.0/.github/dependabot.yml +34 -0
- nirizan-0.1.0/.github/workflows/ci.yml +34 -0
- nirizan-0.1.0/.github/workflows/cross-platform.yml +54 -0
- nirizan-0.1.0/.github/workflows/dependabot-auto-merge.yml +128 -0
- nirizan-0.1.0/.github/workflows/nirizan_gate.yml +93 -0
- nirizan-0.1.0/.github/workflows/packaging.yml +43 -0
- nirizan-0.1.0/.github/workflows/publish.yml +37 -0
- nirizan-0.1.0/.github/workflows/test-pypi.yml +34 -0
- nirizan-0.1.0/.gitignore +265 -0
- nirizan-0.1.0/CITATION.cff +31 -0
- nirizan-0.1.0/CODE_OF_CONDUCT.md +128 -0
- nirizan-0.1.0/CODE_OF_ETHICS.md +116 -0
- nirizan-0.1.0/CONTRIBUTING.md +181 -0
- nirizan-0.1.0/LICENSE +674 -0
- nirizan-0.1.0/PKG-INFO +108 -0
- nirizan-0.1.0/README.md +76 -0
- nirizan-0.1.0/SECURITY.md +51 -0
- nirizan-0.1.0/console/linux/readme.md +1 -0
- nirizan-0.1.0/console/macOS/readme.md +1 -0
- nirizan-0.1.0/console/windows/readme.md +1 -0
- nirizan-0.1.0/docs/architecture.md +198 -0
- nirizan-0.1.0/docs/contracts.md +442 -0
- nirizan-0.1.0/docs/governance/DATA_POLICY.md +165 -0
- nirizan-0.1.0/docs/governance/KPI-Definitions.md +210 -0
- nirizan-0.1.0/docs/literature-review.md +179 -0
- nirizan-0.1.0/docs/standards/EUR-ACE.md +59 -0
- nirizan-0.1.0/docs/standards/ISO-IEC-IEEE.md +52 -0
- nirizan-0.1.0/docs/standards/UN-SDG.md +34 -0
- nirizan-0.1.0/docs/standards/Washington-Accord.md +38 -0
- nirizan-0.1.0/examples/rag_pipeline_demo/main.py +101 -0
- nirizan-0.1.0/experiments/01_instrumentation_trace_storage.ipynb +1039 -0
- nirizan-0.1.0/experiments/02_RAG_Triad.ipynb +1181 -0
- nirizan-0.1.0/experiments/03_Experiment_Tracking_and_Baselines.ipynb +818 -0
- nirizan-0.1.0/experiments/04_regression_detection_ci_gate.ipynb +724 -0
- nirizan-0.1.0/experiments/04b_statistical_gating.ipynb +571 -0
- nirizan-0.1.0/experiments/04v2_regression_detection_ci_gate.ipynb +2380 -0
- nirizan-0.1.0/experiments/05_drift_and_judge_reliability.ipynb +664 -0
- nirizan-0.1.0/experiments/06_testPYPI.ipynb +416 -0
- nirizan-0.1.0/experiments/07_evaluation_ablation_and_benchmarking.ipynb +1916 -0
- nirizan-0.1.0/experiments/readme.md +35 -0
- nirizan-0.1.0/pyproject.toml +108 -0
- nirizan-0.1.0/src/nirizan/__init__.py +18 -0
- nirizan-0.1.0/src/nirizan/_logging.py +113 -0
- nirizan-0.1.0/src/nirizan/gate/__init__.py +18 -0
- nirizan-0.1.0/src/nirizan/gate/ci.py +80 -0
- nirizan-0.1.0/src/nirizan/gate/verdict.py +170 -0
- nirizan-0.1.0/src/nirizan/instrumentation/__init__.py +6 -0
- nirizan-0.1.0/src/nirizan/instrumentation/exporters.py +47 -0
- nirizan-0.1.0/src/nirizan/instrumentation/sdk.py +117 -0
- nirizan-0.1.0/src/nirizan/instrumentation/sessions.py +18 -0
- nirizan-0.1.0/src/nirizan/instrumentation/spans.py +62 -0
- nirizan-0.1.0/src/nirizan/instrumentation/tracer.py +132 -0
- nirizan-0.1.0/src/nirizan/metrics/__init__.py +6 -0
- nirizan-0.1.0/src/nirizan/metrics/base.py +38 -0
- nirizan-0.1.0/src/nirizan/metrics/behavioral_anchor.py +71 -0
- nirizan-0.1.0/src/nirizan/metrics/lightweight_judge.py +87 -0
- nirizan-0.1.0/src/nirizan/metrics/llm_judge.py +89 -0
- nirizan-0.1.0/src/nirizan/metrics/rag_triad.py +107 -0
- nirizan-0.1.0/src/nirizan/metrics/statistical_gating.py +203 -0
- nirizan-0.1.0/src/nirizan/orchestrator/__init__.py +5 -0
- nirizan-0.1.0/src/nirizan/orchestrator/collector.py +98 -0
- nirizan-0.1.0/src/nirizan/orchestrator/dispatcher.py +23 -0
- nirizan-0.1.0/src/nirizan/orchestrator/scheduler.py +76 -0
- nirizan-0.1.0/src/nirizan/regression/__init__.py +22 -0
- nirizan-0.1.0/src/nirizan/regression/comparator.py +275 -0
- nirizan-0.1.0/src/nirizan/regression/thresholds.py +113 -0
- nirizan-0.1.0/src/nirizan/reporting/__init__.py +5 -0
- nirizan-0.1.0/src/nirizan/reporting/dashboard.py +113 -0
- nirizan-0.1.0/src/nirizan/reporting/health_score.py +22 -0
- nirizan-0.1.0/src/nirizan/reporting/judge_reliability.py +176 -0
- nirizan-0.1.0/src/nirizan/storage/__init__.py +4 -0
- nirizan-0.1.0/src/nirizan/storage/baselines.py +100 -0
- nirizan-0.1.0/src/nirizan/storage/experiment_store.py +122 -0
- nirizan-0.1.0/src/nirizan/storage/models.py +115 -0
- nirizan-0.1.0/src/nirizan/storage/run_repository.py +26 -0
- nirizan-0.1.0/src/nirizan/storage/session_repository.py +26 -0
- nirizan-0.1.0/src/nirizan/storage/trace_repository.py +252 -0
- nirizan-0.1.0/src/nirizan/trust/__init__.py +5 -0
- nirizan-0.1.0/src/nirizan/trust/anchor_set.py +22 -0
- nirizan-0.1.0/src/nirizan/trust/attribution.py +67 -0
- nirizan-0.1.0/tests/gate/test_ci.py +116 -0
- nirizan-0.1.0/tests/gate/test_verdict.py +168 -0
- nirizan-0.1.0/tests/instrumentation/test_exporters.py +51 -0
- nirizan-0.1.0/tests/instrumentation/test_sdk.py +97 -0
- nirizan-0.1.0/tests/instrumentation/test_spans.py +90 -0
- nirizan-0.1.0/tests/instrumentation/test_tracer.py +58 -0
- nirizan-0.1.0/tests/integration/test_end_to_end.py +537 -0
- nirizan-0.1.0/tests/metrics/test_behavioral_anchor.py +73 -0
- nirizan-0.1.0/tests/metrics/test_lightweight_judge.py +48 -0
- nirizan-0.1.0/tests/metrics/test_llm_judge.py +56 -0
- nirizan-0.1.0/tests/metrics/test_rag_triad.py +116 -0
- nirizan-0.1.0/tests/metrics/test_statistical_gating.py +105 -0
- nirizan-0.1.0/tests/regression/test_comparator.py +172 -0
- nirizan-0.1.0/tests/regression/test_thresholds.py +73 -0
- nirizan-0.1.0/tests/reporting/test_dashboard.py +139 -0
- nirizan-0.1.0/tests/reporting/test_health_score.py +50 -0
- nirizan-0.1.0/tests/reporting/test_judge_reliability.py +166 -0
- nirizan-0.1.0/tests/storage/test_baselines.py +93 -0
- nirizan-0.1.0/tests/storage/test_experiment_store.py +162 -0
- nirizan-0.1.0/tests/storage/test_models.py +137 -0
- nirizan-0.1.0/tests/trust/test_anchor_set.py +64 -0
- nirizan-0.1.0/tests/trust/test_attribution.py +75 -0
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
name: Bug Report
|
|
2
|
+
description: Something isn't working the way docs/contracts.md or docs/architecture.md says it should.
|
|
3
|
+
title: "[Bug]: "
|
|
4
|
+
labels: ["bug"]
|
|
5
|
+
body:
|
|
6
|
+
- type: markdown
|
|
7
|
+
attributes:
|
|
8
|
+
value: |
|
|
9
|
+
Thanks for reporting a bug. Phases 1-5 are implemented and CI-tested; if you're on a later phase, check CONTRIBUTING.md first since that component may be early or not-yet-built work.
|
|
10
|
+
|
|
11
|
+
- type: dropdown
|
|
12
|
+
id: phase
|
|
13
|
+
attributes:
|
|
14
|
+
label: Which phase does this affect?
|
|
15
|
+
description: See CONTRIBUTING.md if you're unsure which phase a component belongs to.
|
|
16
|
+
options:
|
|
17
|
+
- "Phase 1: Instrumentation & Trace Storage"
|
|
18
|
+
- "Phase 2: RAG Triad Metrics"
|
|
19
|
+
- "Phase 3: Experiment Tracking & Baselines"
|
|
20
|
+
- "Phase 4: Regression Detection & CI Gate"
|
|
21
|
+
- "Phase 5: Drift & Judge-Reliability Layer"
|
|
22
|
+
- "Phase 6: Evaluation Ablation & Benchmarking"
|
|
23
|
+
- Not sure / spans multiple phases
|
|
24
|
+
validations:
|
|
25
|
+
required: true
|
|
26
|
+
|
|
27
|
+
- type: input
|
|
28
|
+
id: component
|
|
29
|
+
attributes:
|
|
30
|
+
label: Component or file
|
|
31
|
+
description: e.g. instrumentation/tracer.py, metrics/rag_triad.py
|
|
32
|
+
placeholder: src/nirizan/...
|
|
33
|
+
validations:
|
|
34
|
+
required: false
|
|
35
|
+
|
|
36
|
+
- type: textarea
|
|
37
|
+
id: what-happened
|
|
38
|
+
attributes:
|
|
39
|
+
label: What happened?
|
|
40
|
+
description: What did you expect instead? Include the exact error message if there is one.
|
|
41
|
+
validations:
|
|
42
|
+
required: true
|
|
43
|
+
|
|
44
|
+
- type: textarea
|
|
45
|
+
id: repro
|
|
46
|
+
attributes:
|
|
47
|
+
label: Steps to reproduce
|
|
48
|
+
description: Minimal steps, or a minimal code snippet, that reproduces the issue.
|
|
49
|
+
render: python
|
|
50
|
+
validations:
|
|
51
|
+
required: false
|
|
52
|
+
|
|
53
|
+
- type: input
|
|
54
|
+
id: version
|
|
55
|
+
attributes:
|
|
56
|
+
label: NiriZan version or commit
|
|
57
|
+
placeholder: e.g. 0.1.0, or a git commit hash
|
|
58
|
+
validations:
|
|
59
|
+
required: false
|
|
60
|
+
|
|
61
|
+
- type: input
|
|
62
|
+
id: python-version
|
|
63
|
+
attributes:
|
|
64
|
+
label: Python version
|
|
65
|
+
placeholder: "3.11 or later (per pyproject.toml)"
|
|
66
|
+
validations:
|
|
67
|
+
required: false
|
|
68
|
+
|
|
69
|
+
- type: checkboxes
|
|
70
|
+
id: terms
|
|
71
|
+
attributes:
|
|
72
|
+
label: Code of Conduct
|
|
73
|
+
description: By submitting this issue, you agree to follow this project's [Code of Conduct](../../CODE_OF_CONDUCT.md).
|
|
74
|
+
options:
|
|
75
|
+
- label: I agree to follow this project's Code of Conduct.
|
|
76
|
+
required: true
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
blank_issues_enabled: true
|
|
2
|
+
contact_links:
|
|
3
|
+
- name: Security Vulnerability Report
|
|
4
|
+
url: https://github.com/Red1-Rahman/NiriZan/security/advisories/new
|
|
5
|
+
about: Do not open a public issue for security vulnerabilities. See SECURITY.md and report privately instead.
|
|
6
|
+
- name: General Question
|
|
7
|
+
url: https://github.com/Red1-Rahman/NiriZan/discussions
|
|
8
|
+
about: Ask a usage or design question that isn't a bug, feature request, or contract change.
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
name: Contract Change
|
|
2
|
+
description: Propose adding to or modifying a data model or interface defined in docs/contracts.md.
|
|
3
|
+
title: "[Contract]: "
|
|
4
|
+
labels: ["contract-change"]
|
|
5
|
+
body:
|
|
6
|
+
- type: markdown
|
|
7
|
+
attributes:
|
|
8
|
+
value: |
|
|
9
|
+
`docs/contracts.md` is the single source of truth for every pydantic model and plugin interface NiriZan's components agree on. Per CONTRIBUTING.md, a contract change is discussed and merged on its own, before any implementation PR that depends on it. This form is for proposing that discussion, not for shipping an implementation directly.
|
|
10
|
+
|
|
11
|
+
Note: `JudgeReliabilityMetrics` and `DashboardSnapshot` (Phase 5, `reporting/`) currently ship without a corresponding entry here, a known, tracked documentation gap, not an example to follow. A Contract Change issue backfilling those two is itself a welcome first contribution.
|
|
12
|
+
|
|
13
|
+
- type: input
|
|
14
|
+
id: contract-name
|
|
15
|
+
attributes:
|
|
16
|
+
label: Which contract does this affect?
|
|
17
|
+
description: Name the exact model, protocol, or interface (e.g. Span, MetricResult, RegressionVerdict).
|
|
18
|
+
placeholder: e.g. Span, Trace, Metric, RegressionVerdict, AttributionVerdict, JudgeReliabilityMetrics, DashboardSnapshot
|
|
19
|
+
validations:
|
|
20
|
+
required: true
|
|
21
|
+
|
|
22
|
+
- type: dropdown
|
|
23
|
+
id: phase
|
|
24
|
+
attributes:
|
|
25
|
+
label: Which phase's contracts does this belong to?
|
|
26
|
+
options:
|
|
27
|
+
- "Phase 1: Instrumentation & Trace Storage"
|
|
28
|
+
- "Phase 2: RAG Triad Metrics"
|
|
29
|
+
- "Phase 3: Experiment Tracking & Baselines"
|
|
30
|
+
- "Phase 4: Regression Detection & CI Gate"
|
|
31
|
+
- "Phase 5: Drift & Judge-Reliability Layer"
|
|
32
|
+
- "Phase 6: Evaluation Ablation & Benchmarking"
|
|
33
|
+
validations:
|
|
34
|
+
required: true
|
|
35
|
+
|
|
36
|
+
- type: dropdown
|
|
37
|
+
id: change-type
|
|
38
|
+
attributes:
|
|
39
|
+
label: Is this additive or breaking?
|
|
40
|
+
description: Per the Versioning Rule at the end of docs/contracts.md. Additive means a new optional field with a default. Breaking means anything else (removed field, changed meaning, narrowed type).
|
|
41
|
+
options:
|
|
42
|
+
- Additive (new optional field with a default value)
|
|
43
|
+
- Breaking (removed field, changed meaning, or narrowed type)
|
|
44
|
+
- Not sure
|
|
45
|
+
validations:
|
|
46
|
+
required: true
|
|
47
|
+
|
|
48
|
+
- type: textarea
|
|
49
|
+
id: current-shape
|
|
50
|
+
attributes:
|
|
51
|
+
label: Current contract (if modifying an existing one)
|
|
52
|
+
description: Paste the relevant model or protocol as it exists today in docs/contracts.md. If it's implemented in src/ but not yet documented here (see the note above), paste the real src/ definition instead and say so.
|
|
53
|
+
render: python
|
|
54
|
+
validations:
|
|
55
|
+
required: false
|
|
56
|
+
|
|
57
|
+
- type: textarea
|
|
58
|
+
id: proposed-shape
|
|
59
|
+
attributes:
|
|
60
|
+
label: Proposed contract
|
|
61
|
+
description: The new or changed model/protocol definition, in the same style as docs/contracts.md.
|
|
62
|
+
render: python
|
|
63
|
+
validations:
|
|
64
|
+
required: true
|
|
65
|
+
|
|
66
|
+
- type: textarea
|
|
67
|
+
id: rationale
|
|
68
|
+
attributes:
|
|
69
|
+
label: Why is this needed?
|
|
70
|
+
description: What does the current contract get wrong or fail to support?
|
|
71
|
+
validations:
|
|
72
|
+
required: true
|
|
73
|
+
|
|
74
|
+
- type: textarea
|
|
75
|
+
id: migration
|
|
76
|
+
attributes:
|
|
77
|
+
label: Migration path (required if breaking)
|
|
78
|
+
description: If this is a breaking change, describe how any already-persisted data using the old shape would be migrated.
|
|
79
|
+
validations:
|
|
80
|
+
required: false
|
|
81
|
+
|
|
82
|
+
- type: checkboxes
|
|
83
|
+
id: terms
|
|
84
|
+
attributes:
|
|
85
|
+
label: Code of Conduct
|
|
86
|
+
description: By submitting this issue, you agree to follow this project's [Code of Conduct](../../CODE_OF_CONDUCT.md).
|
|
87
|
+
options:
|
|
88
|
+
- label: I agree to follow this project's Code of Conduct.
|
|
89
|
+
required: true
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
name: Documentation
|
|
2
|
+
description: Report an issue with, or propose an improvement to, NiriZan's documentation.
|
|
3
|
+
title: "[Docs]: "
|
|
4
|
+
labels: ["documentation"]
|
|
5
|
+
body:
|
|
6
|
+
- type: dropdown
|
|
7
|
+
id: doc-area
|
|
8
|
+
attributes:
|
|
9
|
+
label: Which document does this concern?
|
|
10
|
+
options:
|
|
11
|
+
- README.md
|
|
12
|
+
- CONTRIBUTING.md
|
|
13
|
+
- SECURITY.md
|
|
14
|
+
- CODE_OF_CONDUCT.md
|
|
15
|
+
- CODE_OF_ETHICS.md
|
|
16
|
+
- docs/architecture.md
|
|
17
|
+
- docs/contracts.md
|
|
18
|
+
- docs/literature-review.md
|
|
19
|
+
- docs/modules/ (per-package reference docs)
|
|
20
|
+
- docs/governance/DATA_POLICY.md
|
|
21
|
+
- docs/governance/KPI-Definitions.md
|
|
22
|
+
- docs/standards/ (ISO-IEC-IEEE, Washington-Accord, EUR-ACE, UN-SDG)
|
|
23
|
+
- Something else / not listed
|
|
24
|
+
validations:
|
|
25
|
+
required: true
|
|
26
|
+
|
|
27
|
+
- type: textarea
|
|
28
|
+
id: issue
|
|
29
|
+
attributes:
|
|
30
|
+
label: What's wrong, missing, or unclear?
|
|
31
|
+
description: Quote the specific section if possible.
|
|
32
|
+
validations:
|
|
33
|
+
required: true
|
|
34
|
+
|
|
35
|
+
- type: textarea
|
|
36
|
+
id: suggestion
|
|
37
|
+
attributes:
|
|
38
|
+
label: Suggested fix
|
|
39
|
+
description: If you have a specific rewording or addition in mind, share it here.
|
|
40
|
+
validations:
|
|
41
|
+
required: false
|
|
42
|
+
|
|
43
|
+
- type: checkboxes
|
|
44
|
+
id: terms
|
|
45
|
+
attributes:
|
|
46
|
+
label: Code of Conduct
|
|
47
|
+
description: By submitting this issue, you agree to follow this project's [Code of Conduct](../../CODE_OF_CONDUCT.md).
|
|
48
|
+
options:
|
|
49
|
+
- label: I agree to follow this project's Code of Conduct.
|
|
50
|
+
required: true
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
name: Feature Request
|
|
2
|
+
description: Propose a new capability, metric, or improvement for NiriZan.
|
|
3
|
+
title: "[Feature]: "
|
|
4
|
+
labels: ["enhancement"]
|
|
5
|
+
body:
|
|
6
|
+
- type: markdown
|
|
7
|
+
attributes:
|
|
8
|
+
value: |
|
|
9
|
+
Before proposing new work, check `docs/architecture.md` and `docs/contracts.md`: if what you're proposing isn't reflected in either, that's worth flagging explicitly rather than assuming it fits.
|
|
10
|
+
|
|
11
|
+
- type: dropdown
|
|
12
|
+
id: phase
|
|
13
|
+
attributes:
|
|
14
|
+
label: Which phase does this belong to?
|
|
15
|
+
description: Prefer proposing work for the current phase (6) or closing a known gap in Phases 1-5 over jumping ahead. If this is a later-phase idea (Phase 7+), say so explicitly.
|
|
16
|
+
options:
|
|
17
|
+
- "Phase 1: Instrumentation & Trace Storage"
|
|
18
|
+
- "Phase 2: RAG Triad Metrics"
|
|
19
|
+
- "Phase 3: Experiment Tracking & Baselines"
|
|
20
|
+
- "Phase 4: Regression Detection & CI Gate"
|
|
21
|
+
- "Phase 5: Drift & Judge-Reliability Layer"
|
|
22
|
+
- "Phase 6: Evaluation Ablation & Benchmarking"
|
|
23
|
+
- Beyond the current roadmap / future work
|
|
24
|
+
validations:
|
|
25
|
+
required: true
|
|
26
|
+
|
|
27
|
+
- type: textarea
|
|
28
|
+
id: problem
|
|
29
|
+
attributes:
|
|
30
|
+
label: What problem does this solve?
|
|
31
|
+
description: Describe the gap or limitation this addresses. Reference a specific section of docs/architecture.md or docs/literature-review.md if relevant.
|
|
32
|
+
validations:
|
|
33
|
+
required: true
|
|
34
|
+
|
|
35
|
+
- type: textarea
|
|
36
|
+
id: proposal
|
|
37
|
+
attributes:
|
|
38
|
+
label: Proposed solution
|
|
39
|
+
description: What would you build? If this touches a data model or interface, name it explicitly.
|
|
40
|
+
validations:
|
|
41
|
+
required: true
|
|
42
|
+
|
|
43
|
+
- type: checkboxes
|
|
44
|
+
id: contract-impact
|
|
45
|
+
attributes:
|
|
46
|
+
label: Contract impact
|
|
47
|
+
description: Does this require changing anything in docs/contracts.md?
|
|
48
|
+
options:
|
|
49
|
+
- label: This requires a new or changed contract in docs/contracts.md (consider using the Contract Change template instead).
|
|
50
|
+
required: false
|
|
51
|
+
|
|
52
|
+
- type: textarea
|
|
53
|
+
id: alternatives
|
|
54
|
+
attributes:
|
|
55
|
+
label: Alternatives considered
|
|
56
|
+
description: Any other approaches you considered and why you didn't propose them instead.
|
|
57
|
+
validations:
|
|
58
|
+
required: false
|
|
59
|
+
|
|
60
|
+
- type: checkboxes
|
|
61
|
+
id: terms
|
|
62
|
+
attributes:
|
|
63
|
+
label: Code of Conduct
|
|
64
|
+
description: By submitting this issue, you agree to follow this project's [Code of Conduct](../../CODE_OF_CONDUCT.md).
|
|
65
|
+
options:
|
|
66
|
+
- label: I agree to follow this project's Code of Conduct.
|
|
67
|
+
required: true
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
<!--
|
|
2
|
+
Thanks for opening a PR. Phases 1-5 are implemented and CI-enforced;
|
|
3
|
+
Phase 6 (evaluation & benchmarking) is in progress. Please fill in every
|
|
4
|
+
section below rather than deleting what doesn't seem to apply; "N/A" is a
|
|
5
|
+
valid answer.
|
|
6
|
+
-->
|
|
7
|
+
|
|
8
|
+
## Summary
|
|
9
|
+
|
|
10
|
+
<!-- What does this PR do, in a sentence or two? -->
|
|
11
|
+
|
|
12
|
+
## Which phase does this belong to?
|
|
13
|
+
|
|
14
|
+
<!-- Match the issue templates' phase options. Pick one. -->
|
|
15
|
+
|
|
16
|
+
- [ ] Phase 1: Instrumentation & Trace Storage
|
|
17
|
+
- [ ] Phase 2: RAG Triad Metrics
|
|
18
|
+
- [ ] Phase 3: Experiment Tracking & Baselines
|
|
19
|
+
- [ ] Phase 4: Regression Detection & CI Gate
|
|
20
|
+
- [ ] Phase 5: Drift & Judge-Reliability Layer
|
|
21
|
+
- [ ] Phase 6: Evaluation Ablation & Benchmarking
|
|
22
|
+
- [ ] Not phase-specific (docs, tooling, dependency update, etc.)
|
|
23
|
+
|
|
24
|
+
## Related issue(s)
|
|
25
|
+
|
|
26
|
+
<!-- Link the issue(s) this PR closes or addresses, e.g. "Closes #12". -->
|
|
27
|
+
|
|
28
|
+
## Contract impact
|
|
29
|
+
|
|
30
|
+
<!--
|
|
31
|
+
Per docs/contracts.md and CONTRIBUTING.md: a change to any pydantic model
|
|
32
|
+
or plugin interface in docs/contracts.md is discussed and approved via a
|
|
33
|
+
Contract Change issue BEFORE the implementation PR, not alongside it. Note:
|
|
34
|
+
JudgeReliabilityMetrics and DashboardSnapshot currently ship without a
|
|
35
|
+
docs/contracts.md entry, a known, tracked gap; don't treat their existing
|
|
36
|
+
shape as precedent for skipping this process on new work.
|
|
37
|
+
-->
|
|
38
|
+
|
|
39
|
+
- [ ] This PR does **not** touch `docs/contracts.md`.
|
|
40
|
+
- [ ] This PR **does** touch `docs/contracts.md`, and the change was discussed and approved in Contract Change issue: #____
|
|
41
|
+
|
|
42
|
+
<!--
|
|
43
|
+
If this PR touches docs/contracts.md and there is no linked, approved
|
|
44
|
+
Contract Change issue, expect this PR to be asked to split into a
|
|
45
|
+
contract-change discussion first, per the process in CONTRIBUTING.md.
|
|
46
|
+
-->
|
|
47
|
+
|
|
48
|
+
## Import direction
|
|
49
|
+
|
|
50
|
+
<!-- Per docs/import-boundaries.md. -->
|
|
51
|
+
|
|
52
|
+
- [ ] I did not add any new cross-module imports.
|
|
53
|
+
- [ ] I added new cross-module import(s), and they follow the layer direction in `docs/import-boundaries.md` (`instrumentation → orchestrator → metrics → trust → storage → regression → gate → reporting`).
|
|
54
|
+
|
|
55
|
+
## Local checks
|
|
56
|
+
|
|
57
|
+
<!--
|
|
58
|
+
CI enforcement for human-authored PRs is not wired up yet. Until it is,
|
|
59
|
+
please confirm you've run these locally.
|
|
60
|
+
-->
|
|
61
|
+
|
|
62
|
+
- [ ] `pytest` passes
|
|
63
|
+
- [ ] `mypy --strict` passes
|
|
64
|
+
- [ ] `lint-imports` passes
|
|
65
|
+
|
|
66
|
+
## Contact (optional)
|
|
67
|
+
|
|
68
|
+
<!--
|
|
69
|
+
Optional. Leave blank if you'd rather not share it. If provided, this may
|
|
70
|
+
be used to reach out about this PR specifically (e.g. follow-up questions,
|
|
71
|
+
or potential paid/contract work related to this contribution).
|
|
72
|
+
-->
|
|
73
|
+
|
|
74
|
+
Email: ____
|
|
75
|
+
|
|
76
|
+
## Anything reviewers should look at closely?
|
|
77
|
+
|
|
78
|
+
<!-- Optional: tricky edge cases, deliberate tradeoffs, things you're unsure about. -->
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
version: 2
|
|
2
|
+
|
|
3
|
+
updates:
|
|
4
|
+
# Python dependencies declared in pyproject.toml.
|
|
5
|
+
- package-ecosystem: "pip"
|
|
6
|
+
directory: "/"
|
|
7
|
+
schedule:
|
|
8
|
+
interval: "weekly"
|
|
9
|
+
day: "monday"
|
|
10
|
+
open-pull-requests-limit: 10
|
|
11
|
+
groups:
|
|
12
|
+
python-patch-and-minor:
|
|
13
|
+
update-types:
|
|
14
|
+
- "patch"
|
|
15
|
+
- "minor"
|
|
16
|
+
labels:
|
|
17
|
+
- "dependencies"
|
|
18
|
+
- "python"
|
|
19
|
+
|
|
20
|
+
# GitHub Actions used in .github/workflows/
|
|
21
|
+
- package-ecosystem: "github-actions"
|
|
22
|
+
directory: "/"
|
|
23
|
+
schedule:
|
|
24
|
+
interval: "weekly"
|
|
25
|
+
day: "monday"
|
|
26
|
+
open-pull-requests-limit: 10
|
|
27
|
+
groups:
|
|
28
|
+
actions-patch-and-minor:
|
|
29
|
+
update-types:
|
|
30
|
+
- "patch"
|
|
31
|
+
- "minor"
|
|
32
|
+
labels:
|
|
33
|
+
- "dependencies"
|
|
34
|
+
- "github-actions"
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: ["main"]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: ["main"]
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
contents: read
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
checks:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v5
|
|
18
|
+
|
|
19
|
+
- name: Set up Python
|
|
20
|
+
uses: actions/setup-python@v6
|
|
21
|
+
with:
|
|
22
|
+
python-version: "3.11"
|
|
23
|
+
|
|
24
|
+
- name: Install project with dev dependencies
|
|
25
|
+
run: pip install -e ".[dev]"
|
|
26
|
+
|
|
27
|
+
- name: Run pytest
|
|
28
|
+
run: pytest
|
|
29
|
+
|
|
30
|
+
- name: Run mypy --strict
|
|
31
|
+
run: mypy
|
|
32
|
+
|
|
33
|
+
- name: Run import-linter (layer contract)
|
|
34
|
+
run: lint-imports
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
name: Cross-platform
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: ["main"]
|
|
6
|
+
tags: ["v*"]
|
|
7
|
+
|
|
8
|
+
pull_request:
|
|
9
|
+
branches: ["main"]
|
|
10
|
+
|
|
11
|
+
workflow_dispatch:
|
|
12
|
+
|
|
13
|
+
permissions:
|
|
14
|
+
contents: read
|
|
15
|
+
|
|
16
|
+
jobs:
|
|
17
|
+
smoke-test:
|
|
18
|
+
name: ${{ matrix.os }} • Python ${{ matrix.python-version }}
|
|
19
|
+
runs-on: ${{ matrix.os }}
|
|
20
|
+
|
|
21
|
+
strategy:
|
|
22
|
+
fail-fast: false
|
|
23
|
+
matrix:
|
|
24
|
+
os:
|
|
25
|
+
- ubuntu-latest
|
|
26
|
+
- windows-latest
|
|
27
|
+
- macos-latest
|
|
28
|
+
python-version:
|
|
29
|
+
- "3.11"
|
|
30
|
+
|
|
31
|
+
steps:
|
|
32
|
+
- name: Check out repository
|
|
33
|
+
uses: actions/checkout@v5
|
|
34
|
+
|
|
35
|
+
- name: Set up Python
|
|
36
|
+
uses: actions/setup-python@v6
|
|
37
|
+
with:
|
|
38
|
+
python-version: ${{ matrix.python-version }}
|
|
39
|
+
|
|
40
|
+
- name: Upgrade packaging tools
|
|
41
|
+
run: |
|
|
42
|
+
python -m pip install --upgrade pip
|
|
43
|
+
|
|
44
|
+
- name: Install NiriZan
|
|
45
|
+
run: |
|
|
46
|
+
pip install -e .
|
|
47
|
+
|
|
48
|
+
- name: Verify package import
|
|
49
|
+
run: |
|
|
50
|
+
python -c "import nirizan; print('Imported NiriZan successfully.')"
|
|
51
|
+
|
|
52
|
+
- name: Verify package metadata
|
|
53
|
+
run: |
|
|
54
|
+
python -c "import importlib.metadata as md; print(md.version('nirizan'))"
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
name: Dependabot auto-merge
|
|
2
|
+
|
|
3
|
+
# Runs only on pull requests opened by Dependabot. Every dependency bump,
|
|
4
|
+
# regardless of semver level, must pass the full check suite here before
|
|
5
|
+
# anything is auto-merged. Patch and minor updates that pass are merged
|
|
6
|
+
# automatically. Major updates are never auto-merged, even if every check
|
|
7
|
+
# is green, since a passing test suite does not prove a major bump (e.g.
|
|
8
|
+
# pydantic v2 -> v3) is safe. It only proves the tests we happened to
|
|
9
|
+
# write still pass against the new version.
|
|
10
|
+
on:
|
|
11
|
+
pull_request:
|
|
12
|
+
branches: ["main"]
|
|
13
|
+
|
|
14
|
+
permissions:
|
|
15
|
+
contents: write
|
|
16
|
+
pull-requests: write
|
|
17
|
+
checks: read
|
|
18
|
+
|
|
19
|
+
jobs:
|
|
20
|
+
# Guard: only ever run the rest of this workflow for Dependabot PRs.
|
|
21
|
+
# A human PR that happens to touch pyproject.toml must never be
|
|
22
|
+
# auto-merged by this workflow.
|
|
23
|
+
guard:
|
|
24
|
+
if: github.actor == 'dependabot[bot]'
|
|
25
|
+
runs-on: ubuntu-latest
|
|
26
|
+
steps:
|
|
27
|
+
- run: echo "Dependabot PR confirmed, proceeding to checks."
|
|
28
|
+
|
|
29
|
+
# Full check suite: pytest, mypy --strict, and the import-linter layer
|
|
30
|
+
# contract from docs/import-boundaries.md. All three must pass. This is
|
|
31
|
+
# the step that catches the failure modes a version bump can quietly
|
|
32
|
+
# introduce: a behavioral regression (pytest), a type-contract violation
|
|
33
|
+
# from an upstream API change (mypy --strict), or a new import that
|
|
34
|
+
# breaks the layer direction because a dependency changed how a module
|
|
35
|
+
# re-exports something (import-linter). None of the three subsumes the
|
|
36
|
+
# others, so all three run every time, not just on suspicion.
|
|
37
|
+
checks:
|
|
38
|
+
needs: guard
|
|
39
|
+
runs-on: ubuntu-latest
|
|
40
|
+
steps:
|
|
41
|
+
- uses: actions/checkout@v7
|
|
42
|
+
|
|
43
|
+
- name: Set up Python
|
|
44
|
+
uses: actions/setup-python@v7
|
|
45
|
+
with:
|
|
46
|
+
python-version: "3.11"
|
|
47
|
+
|
|
48
|
+
- name: Install project with dev dependencies
|
|
49
|
+
run: pip install -e ".[dev]"
|
|
50
|
+
|
|
51
|
+
- name: Run pytest
|
|
52
|
+
run: pytest
|
|
53
|
+
|
|
54
|
+
- name: Run mypy --strict
|
|
55
|
+
run: mypy
|
|
56
|
+
|
|
57
|
+
- name: Run import-linter (layer contract)
|
|
58
|
+
run: lint-imports
|
|
59
|
+
|
|
60
|
+
# Determine the semver level of the bump using dependabot's own
|
|
61
|
+
# metadata action, rather than parsing the PR title or branch name.
|
|
62
|
+
# This is the source of truth fetch-metadata is built for.
|
|
63
|
+
metadata:
|
|
64
|
+
needs: guard
|
|
65
|
+
runs-on: ubuntu-latest
|
|
66
|
+
outputs:
|
|
67
|
+
update-type: ${{ steps.metadata.outputs.update-type }}
|
|
68
|
+
dependency-names: ${{ steps.metadata.outputs.dependency-names }}
|
|
69
|
+
steps:
|
|
70
|
+
- name: Fetch Dependabot metadata
|
|
71
|
+
id: metadata
|
|
72
|
+
uses: dependabot/fetch-metadata@v3
|
|
73
|
+
with:
|
|
74
|
+
github-token: "${{ secrets.GITHUB_TOKEN }}"
|
|
75
|
+
|
|
76
|
+
# Patch and minor bumps: if checks passed, enable GitHub's native
|
|
77
|
+
# auto-merge. This does not force-merge immediately; it tells GitHub to
|
|
78
|
+
# merge once all required status checks (including the `checks` job
|
|
79
|
+
# above) report green, so this step is safe to run even if some other
|
|
80
|
+
# required check outside this workflow is still pending.
|
|
81
|
+
auto-merge-patch-minor:
|
|
82
|
+
needs: [checks, metadata]
|
|
83
|
+
if: |
|
|
84
|
+
needs.metadata.outputs.update-type == 'version-update:semver-patch' ||
|
|
85
|
+
needs.metadata.outputs.update-type == 'version-update:semver-minor'
|
|
86
|
+
runs-on: ubuntu-latest
|
|
87
|
+
steps:
|
|
88
|
+
- name: Enable auto-merge for patch/minor update
|
|
89
|
+
run: gh pr merge --auto --squash "$PR_URL"
|
|
90
|
+
env:
|
|
91
|
+
PR_URL: ${{ github.event.pull_request.html_url }}
|
|
92
|
+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
93
|
+
|
|
94
|
+
# Major bumps: never auto-merged, regardless of check outcome. Leave the
|
|
95
|
+
# PR open, label it, and post a comment explaining why, so it's obvious
|
|
96
|
+
# at a glance this needs a human rather than looking stalled or ignored.
|
|
97
|
+
flag-major-for-review:
|
|
98
|
+
needs: [checks, metadata]
|
|
99
|
+
if: needs.metadata.outputs.update-type == 'version-update:semver-major'
|
|
100
|
+
runs-on: ubuntu-latest
|
|
101
|
+
steps:
|
|
102
|
+
- name: Label PR for manual review
|
|
103
|
+
run: gh pr edit "$PR_URL" --add-label "needs-manual-review,major-update"
|
|
104
|
+
env:
|
|
105
|
+
PR_URL: ${{ github.event.pull_request.html_url }}
|
|
106
|
+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
107
|
+
|
|
108
|
+
- name: Comment explaining why this was not auto-merged
|
|
109
|
+
run: |
|
|
110
|
+
gh pr comment "$PR_URL" --body "$(cat <<'EOF'
|
|
111
|
+
This is a **major** semver update (${{ needs.metadata.outputs.dependency-names }}).
|
|
112
|
+
|
|
113
|
+
Major updates are never auto-merged by this workflow, even when
|
|
114
|
+
pytest, mypy --strict, and import-linter all pass. A green check
|
|
115
|
+
suite here only proves the existing tests still pass against the
|
|
116
|
+
new version, not that the new version is safe: major bumps are
|
|
117
|
+
exactly where ABI mismatches, changed default behavior, or
|
|
118
|
+
removed APIs tend to surface outside what the test suite covers.
|
|
119
|
+
|
|
120
|
+
This PR needs manual review before merging. See
|
|
121
|
+
docs/import-boundaries.md and docs/contracts.md if the update
|
|
122
|
+
touches a dependency those documents assume a specific version
|
|
123
|
+
or behavior of.
|
|
124
|
+
EOF
|
|
125
|
+
)"
|
|
126
|
+
env:
|
|
127
|
+
PR_URL: ${{ github.event.pull_request.html_url }}
|
|
128
|
+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|