data-reliability-index 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_reliability_index-0.3.0/PKG-INFO +333 -0
- data_reliability_index-0.3.0/README.md +282 -0
- {data_reliability_index-0.2.0 → data_reliability_index-0.3.0}/pyproject.toml +12 -8
- data_reliability_index-0.3.0/src/data_reliability/__init__.py +47 -0
- data_reliability_index-0.3.0/src/data_reliability/core.py +82 -0
- data_reliability_index-0.3.0/src/data_reliability/database.py +76 -0
- data_reliability_index-0.3.0/src/data_reliability/pandas_ext.py +29 -0
- data_reliability_index-0.3.0/src/data_reliability/scanner.py +333 -0
- data_reliability_index-0.3.0/src/data_reliability_index.egg-info/PKG-INFO +333 -0
- {data_reliability_index-0.2.0 → data_reliability_index-0.3.0}/src/data_reliability_index.egg-info/SOURCES.txt +1 -0
- {data_reliability_index-0.2.0 → data_reliability_index-0.3.0}/src/data_reliability_index.egg-info/requires.txt +10 -0
- data_reliability_index-0.3.0/tests/test_core.py +209 -0
- data_reliability_index-0.2.0/PKG-INFO +0 -176
- data_reliability_index-0.2.0/README.md +0 -136
- data_reliability_index-0.2.0/src/data_reliability/__init__.py +0 -15
- data_reliability_index-0.2.0/src/data_reliability/core.py +0 -45
- data_reliability_index-0.2.0/src/data_reliability/pandas_ext.py +0 -17
- data_reliability_index-0.2.0/src/data_reliability/scanner.py +0 -145
- data_reliability_index-0.2.0/src/data_reliability_index.egg-info/PKG-INFO +0 -176
- data_reliability_index-0.2.0/tests/test_core.py +0 -71
- {data_reliability_index-0.2.0 → data_reliability_index-0.3.0}/LICENSE +0 -0
- {data_reliability_index-0.2.0 → data_reliability_index-0.3.0}/NOTICE +0 -0
- {data_reliability_index-0.2.0 → data_reliability_index-0.3.0}/setup.cfg +0 -0
- {data_reliability_index-0.2.0 → data_reliability_index-0.3.0}/src/data_reliability/py.typed +0 -0
- {data_reliability_index-0.2.0 → data_reliability_index-0.3.0}/src/data_reliability_index.egg-info/dependency_links.txt +0 -0
- {data_reliability_index-0.2.0 → data_reliability_index-0.3.0}/src/data_reliability_index.egg-info/top_level.txt +0 -0
- {data_reliability_index-0.2.0 → data_reliability_index-0.3.0}/tests/test_pandas.py +0 -0
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: data-reliability-index
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: SDK for scoring, tiering, and filtering data reliability evidence.
|
|
5
|
+
Author: h3pdesign
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/h3pdesign/data-reliability-index
|
|
8
|
+
Project-URL: Documentation, https://h3pdesign.github.io/data-reliability-index/
|
|
9
|
+
Project-URL: Repository, https://github.com/h3pdesign/data-reliability-index
|
|
10
|
+
Project-URL: Issues, https://github.com/h3pdesign/data-reliability-index/issues
|
|
11
|
+
Keywords: data-quality,data-reliability,validation,pydantic,sdk,database
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Science/Research
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Classifier: Topic :: Scientific/Engineering
|
|
24
|
+
Classifier: Topic :: Database
|
|
25
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
26
|
+
Requires-Python: >=3.9
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
License-File: LICENSE
|
|
29
|
+
License-File: NOTICE
|
|
30
|
+
Requires-Dist: pydantic>=2.0
|
|
31
|
+
Provides-Extra: build
|
|
32
|
+
Requires-Dist: build; extra == "build"
|
|
33
|
+
Requires-Dist: twine; extra == "build"
|
|
34
|
+
Provides-Extra: docs
|
|
35
|
+
Requires-Dist: black; extra == "docs"
|
|
36
|
+
Requires-Dist: mkdocs-material; extra == "docs"
|
|
37
|
+
Requires-Dist: mkdocstrings[python]; extra == "docs"
|
|
38
|
+
Provides-Extra: pandas
|
|
39
|
+
Requires-Dist: pandas>=2.0; extra == "pandas"
|
|
40
|
+
Provides-Extra: api
|
|
41
|
+
Requires-Dist: fastapi>=0.100; extra == "api"
|
|
42
|
+
Requires-Dist: uvicorn>=0.20; extra == "api"
|
|
43
|
+
Provides-Extra: test
|
|
44
|
+
Requires-Dist: pytest; extra == "test"
|
|
45
|
+
Requires-Dist: pandas>=2.0; extra == "test"
|
|
46
|
+
Provides-Extra: all
|
|
47
|
+
Requires-Dist: pandas>=2.0; extra == "all"
|
|
48
|
+
Requires-Dist: fastapi>=0.100; extra == "all"
|
|
49
|
+
Requires-Dist: uvicorn>=0.20; extra == "all"
|
|
50
|
+
Dynamic: license-file
|
|
51
|
+
|
|
52
|
+
# Data Reliability Index
|
|
53
|
+
|
|
54
|
+
[](https://github.com/h3pdesign/data-reliability-index/actions/workflows/ci.yml)
|
|
55
|
+
|
|
56
|
+

|
|
57
|
+
|
|
58
|
+
Data Reliability Index is a typed Python SDK for attaching reliability metadata to data points, enforcing trust policies, and filtering unreliable records before they reach analysis, databases, or API boundaries.
|
|
59
|
+
|
|
60
|
+
The package is built around a simple rule: data should carry the evidence needed to decide whether it is safe to use.
|
|
61
|
+
|
|
62
|
+
Supported Python versions: `3.9` through `3.14`.
|
|
63
|
+
|
|
64
|
+
## Release Status
|
|
65
|
+
|
|
66
|
+
Latest release: [v0.3.0](https://github.com/h3pdesign/data-reliability-index/releases/tag/v0.3.0)
|
|
67
|
+
|
|
68
|
+
The `v0.3.0` GitHub Release includes signed source, a wheel, and a source distribution. The package is published on PyPI as [`data-reliability-index`](https://pypi.org/project/data-reliability-index/).
|
|
69
|
+
|
|
70
|
+
## Features
|
|
71
|
+
|
|
72
|
+
- Pydantic models for reliability metadata and policies.
|
|
73
|
+
- Scanning engine for computing reliability scores from validation evidence.
|
|
74
|
+
- Tiered trust classification with scores from 0 to 100.
|
|
75
|
+
- Automatic trust-tier assignment from score and verification signals.
|
|
76
|
+
- Trace hash computation and verification support.
|
|
77
|
+
- Policy-based acceptance checks for individual records.
|
|
78
|
+
- Structured accept/reject decisions for audits and ingestion logs.
|
|
79
|
+
- Driver-neutral database helpers for SQL, analytical, and document stores.
|
|
80
|
+
- Optional Pandas helpers for filtering DataFrames by reliability metadata.
|
|
81
|
+
- FastAPI example for rejecting low-reliability input at ingestion time.
|
|
82
|
+
- MkDocs documentation for concepts and API usage.
|
|
83
|
+
|
|
84
|
+
## Installation
|
|
85
|
+
|
|
86
|
+
Install from PyPI:
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
pip install data-reliability-index
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Optional integrations:
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
pip install "data-reliability-index[pandas]"
|
|
96
|
+
pip install "data-reliability-index[api]"
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
You can also install the latest GitHub Release wheel directly:
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
pip install https://github.com/h3pdesign/data-reliability-index/releases/download/v0.3.0/data_reliability_index-0.3.0-py3-none-any.whl
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
For local development from this repository:
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
pip install -e ".[test]"
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
## Quick Start
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
from data_reliability import DataTier, ReliabilityPolicy, ReliabilityScanner, ValidationEvidence
|
|
115
|
+
|
|
116
|
+
scanner = ReliabilityScanner()
|
|
117
|
+
data = scanner.scan(
|
|
118
|
+
{"temperature": 21.4, "unit": "celsius"},
|
|
119
|
+
source_id="sensor-a",
|
|
120
|
+
evidence=ValidationEvidence(
|
|
121
|
+
completeness=1.0,
|
|
122
|
+
consistency=1.0,
|
|
123
|
+
provenance=1.0,
|
|
124
|
+
cryptographic_verification=1.0,
|
|
125
|
+
calibration=1.0,
|
|
126
|
+
schema_compliance=1.0,
|
|
127
|
+
anomaly_detection=1.0,
|
|
128
|
+
duplicate_detection=1.0,
|
|
129
|
+
metadata_quality=1.0,
|
|
130
|
+
),
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
policy = ReliabilityPolicy(
|
|
134
|
+
minimum_score=90,
|
|
135
|
+
maximum_tier=DataTier.TIER_2,
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
assert policy.resolve(data) == {"temperature": 21.4, "unit": "celsius"}
|
|
139
|
+
assert policy.assess(data.reliability).accepted is True
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
## How the Data Reliability Index Works
|
|
143
|
+
|
|
144
|
+
The Data Reliability Index models the complete lifecycle of a data point as it moves from raw input into a trusted dataset. Rather than treating all data equally, the system evaluates, scores, and classifies every record before it becomes eligible for trusted use.
|
|
145
|
+
|
|
146
|
+
Raw data can enter from APIs, IoT devices, calibrated sensors, databases, files, and user submissions. Because these sources differ in quality and provenance, every incoming record starts as untrusted until it has been analyzed.
|
|
147
|
+
|
|
148
|
+
The scanning engine evaluates validation evidence for completeness, consistency, provenance, cryptographic verification, calibration, schema compliance, anomaly detection, duplicate detection, and metadata quality. These checks provide an objective assessment of how trustworthy each data point is.
|
|
149
|
+
|
|
150
|
+
Each scan produces:
|
|
151
|
+
|
|
152
|
+
- A numeric reliability score from `0` to `100`, representing overall confidence.
|
|
153
|
+
- A standardized trust tier from `TIER_1` to `TIER_3`, describing source and verification level.
|
|
154
|
+
- A trace hash that can be used to verify whether the data changed.
|
|
155
|
+
- A `ReliableData` wrapper containing the original value and its reliability metadata.
|
|
156
|
+
|
|
157
|
+
Together, the numeric score and trust tier provide more information than either metric alone. The score enables precise filtering and ranking, while the tier gives an immediately understandable description of the data's verification level.
|
|
158
|
+
|
|
159
|
+
## Trust Tiers
|
|
160
|
+
|
|
161
|
+
The system classifies data into three standardized trust levels:
|
|
162
|
+
|
|
163
|
+
| Tier | Trust level | Typical sources | Intended use |
|
|
164
|
+
| --- | --- | --- | --- |
|
|
165
|
+
| `TIER_1` | Highest trust | Cryptographically verified data, calibrated sensor measurements, direct measurements from trusted APIs | Scientific, safety-critical, and mission-critical applications |
|
|
166
|
+
| `TIER_2` | High trust | Cleaned secondary datasets, indirect measurements, partially validated or derived information | Analytics, forecasting, and most production workloads |
|
|
167
|
+
| `TIER_3` | Moderate trust | User-generated content, self-reported information, weakly verified external sources | Exploratory analysis and workflows requiring additional validation |
|
|
168
|
+
|
|
169
|
+
## Intelligent Decision Pipeline
|
|
170
|
+
|
|
171
|
+
Based on the analysis results, data follows one of two paths:
|
|
172
|
+
|
|
173
|
+
- Rejected data: records failing minimum reliability requirements are isolated and excluded from trusted datasets.
|
|
174
|
+
- Accepted data: records meeting validation thresholds can be stored in trusted repositories while remaining fully traceable.
|
|
175
|
+
|
|
176
|
+
Every accepted record retains its reliability score, trust tier, provenance metadata, validation evidence, cryptographic integrity information, and audit context. Trust is not only calculated once; it remains transparent and reproducible throughout the data lifecycle.
|
|
177
|
+
|
|
178
|
+
## Transparent Scoring
|
|
179
|
+
|
|
180
|
+
The scanner creates a numeric score by applying profile weights to the evidence dimensions. Each evidence value is normalized from `0.0` to `1.0`, multiplied by the active profile's weight, and converted to a `0` to `100` score. Missing timestamps apply an additional penalty when timestamp verification is required.
|
|
181
|
+
|
|
182
|
+
Trust tiers are then assigned by explicit profile criteria:
|
|
183
|
+
|
|
184
|
+
- `TIER_1`: the record must pass the Tier 1 minimum score and all required evidence thresholds.
|
|
185
|
+
- `TIER_2`: the record must pass the Tier 2 minimum score and all required evidence thresholds.
|
|
186
|
+
- `TIER_3`: fallback for records that do not satisfy Tier 1 or Tier 2.
|
|
187
|
+
|
|
188
|
+
The package includes three profiles:
|
|
189
|
+
|
|
190
|
+
| Profile | Purpose | Tier behavior |
|
|
191
|
+
| --- | --- | --- |
|
|
192
|
+
| `default_profile()` | General application, API, and analytics data | Tier 1 requires high score, provenance, cryptographic verification, schema compliance, and verified timestamp |
|
|
193
|
+
| `scientific_profile()` | Research data across scientific fields | Increases weight and thresholds for provenance, calibration, consistency, anomaly checks, and reproducibility signals |
|
|
194
|
+
| `climate_record_profile()` | Weather and climate record data | Emphasizes calibrated instruments, station metadata, consistency with comparison stations, anomaly checks, and provenance |
|
|
195
|
+
|
|
196
|
+
The scoring model is stable within a profile. Different use cases change their acceptance thresholds or active profile, not the meaning of reliability itself.
|
|
197
|
+
|
|
198
|
+
Examples:
|
|
199
|
+
|
|
200
|
+
- A medical research study may require `TIER_1` data with a score of `99` or higher.
|
|
201
|
+
- An autonomous vehicle system may accept only `TIER_1` data above `95`.
|
|
202
|
+
- Financial risk models may require `TIER_1` or high-quality `TIER_2` data.
|
|
203
|
+
- Product analytics might accept `TIER_2` data with a threshold of `75`.
|
|
204
|
+
- Research prototypes may intentionally include `TIER_3` data while applying lower confidence weights.
|
|
205
|
+
|
|
206
|
+
Because every dataset is evaluated using the same transparent methodology, organizations can define their own acceptance criteria without changing how reliability is measured.
|
|
207
|
+
|
|
208
|
+
Instead of asking whether a dataset can be trusted, users can inspect objective metrics: reliability score, standardized trust tier, provenance, and validation history.
|
|
209
|
+
|
|
210
|
+
You can inspect why a record received its tier:
|
|
211
|
+
|
|
212
|
+
```python
|
|
213
|
+
from data_reliability import ReliabilityScanner, ValidationEvidence, climate_record_profile
|
|
214
|
+
|
|
215
|
+
scanner = ReliabilityScanner(profile=climate_record_profile())
|
|
216
|
+
evidence = ValidationEvidence(
|
|
217
|
+
completeness=0.98,
|
|
218
|
+
consistency=0.96,
|
|
219
|
+
provenance=0.96,
|
|
220
|
+
calibration=0.96,
|
|
221
|
+
schema_compliance=0.92,
|
|
222
|
+
anomaly_detection=0.96,
|
|
223
|
+
metadata_quality=0.60,
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
print(scanner.tier_evaluation(evidence))
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
For climate records, this makes methodological uncertainty visible. A temperature record with strong provenance and calibration can still fail Tier 1 if station metadata, anomaly checks, timestamp verification, or comparison-station consistency are insufficient.
|
|
230
|
+
|
|
231
|
+
## Database Usage
|
|
232
|
+
|
|
233
|
+
The SDK does not require a database driver. It emits plain dictionaries so the same reliability metadata can be stored in small local databases such as SQLite and DuckDB, production SQL databases such as PostgreSQL and MySQL, analytical warehouses, or document stores.
|
|
234
|
+
|
|
235
|
+
```python
|
|
236
|
+
from data_reliability import ValidationEvidence, scan_row
|
|
237
|
+
|
|
238
|
+
row = {"temperature": 21.4, "unit": "celsius"}
|
|
239
|
+
scored_row = scan_row(
|
|
240
|
+
row,
|
|
241
|
+
source_id="sensor-a",
|
|
242
|
+
evidence=ValidationEvidence(cryptographic_verification=1.0, calibration=1.0),
|
|
243
|
+
required_fields=["temperature", "unit"],
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
assert scored_row["dri_score"] >= 90
|
|
247
|
+
assert scored_row["dri_tier"] == 1
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
For SQL tables, store the generated `dri_*` columns beside the source data. For document databases, store `metadata_to_document(reliable.reliability)` as a nested reliability object.
|
|
251
|
+
|
|
252
|
+
## SDK and Security Notes
|
|
253
|
+
|
|
254
|
+
This project is intended to be used as an SDK. The core package keeps dependencies small, exposes typed Pydantic models, avoids dynamic code execution, and uses deterministic SHA-256 trace hashes to detect changed payloads. Hashes are integrity signals, not proof of source identity by themselves; use authenticated ingestion, signed upstream payloads, database permissions, and private vulnerability reporting for production systems.
|
|
255
|
+
|
|
256
|
+
## Pandas Filtering
|
|
257
|
+
|
|
258
|
+
```python
|
|
259
|
+
import pandas as pd
|
|
260
|
+
from data_reliability import DataTier, ReliabilityMetadata, ReliabilityPolicy, filter_reliable_df
|
|
261
|
+
|
|
262
|
+
df = pd.DataFrame([
|
|
263
|
+
{
|
|
264
|
+
"value": 10,
|
|
265
|
+
"reliability": ReliabilityMetadata(
|
|
266
|
+
score=95,
|
|
267
|
+
tier=DataTier.TIER_1,
|
|
268
|
+
source_id="sensor-a",
|
|
269
|
+
trace_hash="abc123",
|
|
270
|
+
),
|
|
271
|
+
},
|
|
272
|
+
])
|
|
273
|
+
|
|
274
|
+
policy = ReliabilityPolicy(minimum_score=90, maximum_tier=DataTier.TIER_2)
|
|
275
|
+
trusted = filter_reliable_df(df, policy)
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
## Documentation
|
|
279
|
+
|
|
280
|
+
The project documentation lives in [`docs/`](docs/) and can be served locally with MkDocs:
|
|
281
|
+
|
|
282
|
+
```bash
|
|
283
|
+
pip install -e ".[docs]"
|
|
284
|
+
mkdocs serve
|
|
285
|
+
```
|
|
286
|
+
|
|
287
|
+
Start with:
|
|
288
|
+
|
|
289
|
+
- [Concepts](docs/concepts.md)
|
|
290
|
+
- [Core models](docs/api/core.md)
|
|
291
|
+
- [Scanning engine](docs/api/scanner.md)
|
|
292
|
+
- [Database helpers](docs/api/database.md)
|
|
293
|
+
- [Pandas extension](docs/api/pandas.md)
|
|
294
|
+
- [FastAPI example](docs/api/fastapi.md)
|
|
295
|
+
- [Release and publishing](docs/release.md)
|
|
296
|
+
|
|
297
|
+
The longer project rationale is available in [`data-reliability.md`](data-reliability.md).
|
|
298
|
+
|
|
299
|
+
## Development
|
|
300
|
+
|
|
301
|
+
Run the test suite:
|
|
302
|
+
|
|
303
|
+
```bash
|
|
304
|
+
pip install -e ".[test]"
|
|
305
|
+
pytest
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
Build package artifacts:
|
|
309
|
+
|
|
310
|
+
```bash
|
|
311
|
+
pip install -e ".[build]"
|
|
312
|
+
python -m build
|
|
313
|
+
python -m twine check dist/*
|
|
314
|
+
```
|
|
315
|
+
|
|
316
|
+
Run the FastAPI example:
|
|
317
|
+
|
|
318
|
+
```bash
|
|
319
|
+
pip install -e ".[api]"
|
|
320
|
+
uvicorn examples.fastapi_app:app --reload
|
|
321
|
+
```
|
|
322
|
+
|
|
323
|
+
## Contributing
|
|
324
|
+
|
|
325
|
+
Contributions are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) for the local workflow and pull request expectations.
|
|
326
|
+
|
|
327
|
+
## Security
|
|
328
|
+
|
|
329
|
+
Please report security issues privately. See [SECURITY.md](SECURITY.md).
|
|
330
|
+
|
|
331
|
+
## License
|
|
332
|
+
|
|
333
|
+
Licensed under the [Apache License 2.0](LICENSE).
|
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
# Data Reliability Index
|
|
2
|
+
|
|
3
|
+
[](https://github.com/h3pdesign/data-reliability-index/actions/workflows/ci.yml)
|
|
4
|
+
|
|
5
|
+

|
|
6
|
+
|
|
7
|
+
Data Reliability Index is a typed Python SDK for attaching reliability metadata to data points, enforcing trust policies, and filtering unreliable records before they reach analysis, databases, or API boundaries.
|
|
8
|
+
|
|
9
|
+
The package is built around a simple rule: data should carry the evidence needed to decide whether it is safe to use.
|
|
10
|
+
|
|
11
|
+
Supported Python versions: `3.9` through `3.14`.
|
|
12
|
+
|
|
13
|
+
## Release Status
|
|
14
|
+
|
|
15
|
+
Latest release: [v0.3.0](https://github.com/h3pdesign/data-reliability-index/releases/tag/v0.3.0)
|
|
16
|
+
|
|
17
|
+
The `v0.3.0` GitHub Release includes signed source, a wheel, and a source distribution. The package is published on PyPI as [`data-reliability-index`](https://pypi.org/project/data-reliability-index/).
|
|
18
|
+
|
|
19
|
+
## Features
|
|
20
|
+
|
|
21
|
+
- Pydantic models for reliability metadata and policies.
|
|
22
|
+
- Scanning engine for computing reliability scores from validation evidence.
|
|
23
|
+
- Tiered trust classification with scores from 0 to 100.
|
|
24
|
+
- Automatic trust-tier assignment from score and verification signals.
|
|
25
|
+
- Trace hash computation and verification support.
|
|
26
|
+
- Policy-based acceptance checks for individual records.
|
|
27
|
+
- Structured accept/reject decisions for audits and ingestion logs.
|
|
28
|
+
- Driver-neutral database helpers for SQL, analytical, and document stores.
|
|
29
|
+
- Optional Pandas helpers for filtering DataFrames by reliability metadata.
|
|
30
|
+
- FastAPI example for rejecting low-reliability input at ingestion time.
|
|
31
|
+
- MkDocs documentation for concepts and API usage.
|
|
32
|
+
|
|
33
|
+
## Installation
|
|
34
|
+
|
|
35
|
+
Install from PyPI:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
pip install data-reliability-index
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Optional integrations:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
pip install "data-reliability-index[pandas]"
|
|
45
|
+
pip install "data-reliability-index[api]"
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
You can also install the latest GitHub Release wheel directly:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
pip install https://github.com/h3pdesign/data-reliability-index/releases/download/v0.3.0/data_reliability_index-0.3.0-py3-none-any.whl
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
For local development from this repository:
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
pip install -e ".[test]"
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Quick Start
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
from data_reliability import DataTier, ReliabilityPolicy, ReliabilityScanner, ValidationEvidence
|
|
64
|
+
|
|
65
|
+
scanner = ReliabilityScanner()
|
|
66
|
+
data = scanner.scan(
|
|
67
|
+
{"temperature": 21.4, "unit": "celsius"},
|
|
68
|
+
source_id="sensor-a",
|
|
69
|
+
evidence=ValidationEvidence(
|
|
70
|
+
completeness=1.0,
|
|
71
|
+
consistency=1.0,
|
|
72
|
+
provenance=1.0,
|
|
73
|
+
cryptographic_verification=1.0,
|
|
74
|
+
calibration=1.0,
|
|
75
|
+
schema_compliance=1.0,
|
|
76
|
+
anomaly_detection=1.0,
|
|
77
|
+
duplicate_detection=1.0,
|
|
78
|
+
metadata_quality=1.0,
|
|
79
|
+
),
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
policy = ReliabilityPolicy(
|
|
83
|
+
minimum_score=90,
|
|
84
|
+
maximum_tier=DataTier.TIER_2,
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
assert policy.resolve(data) == {"temperature": 21.4, "unit": "celsius"}
|
|
88
|
+
assert policy.assess(data.reliability).accepted is True
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## How the Data Reliability Index Works
|
|
92
|
+
|
|
93
|
+
The Data Reliability Index models the complete lifecycle of a data point as it moves from raw input into a trusted dataset. Rather than treating all data equally, the system evaluates, scores, and classifies every record before it becomes eligible for trusted use.
|
|
94
|
+
|
|
95
|
+
Raw data can enter from APIs, IoT devices, calibrated sensors, databases, files, and user submissions. Because these sources differ in quality and provenance, every incoming record starts as untrusted until it has been analyzed.
|
|
96
|
+
|
|
97
|
+
The scanning engine evaluates validation evidence for completeness, consistency, provenance, cryptographic verification, calibration, schema compliance, anomaly detection, duplicate detection, and metadata quality. These checks provide an objective assessment of how trustworthy each data point is.
|
|
98
|
+
|
|
99
|
+
Each scan produces:
|
|
100
|
+
|
|
101
|
+
- A numeric reliability score from `0` to `100`, representing overall confidence.
|
|
102
|
+
- A standardized trust tier from `TIER_1` to `TIER_3`, describing source and verification level.
|
|
103
|
+
- A trace hash that can be used to verify whether the data changed.
|
|
104
|
+
- A `ReliableData` wrapper containing the original value and its reliability metadata.
|
|
105
|
+
|
|
106
|
+
Together, the numeric score and trust tier provide more information than either metric alone. The score enables precise filtering and ranking, while the tier gives an immediately understandable description of the data's verification level.
|
|
107
|
+
|
|
108
|
+
## Trust Tiers
|
|
109
|
+
|
|
110
|
+
The system classifies data into three standardized trust levels:
|
|
111
|
+
|
|
112
|
+
| Tier | Trust level | Typical sources | Intended use |
|
|
113
|
+
| --- | --- | --- | --- |
|
|
114
|
+
| `TIER_1` | Highest trust | Cryptographically verified data, calibrated sensor measurements, direct measurements from trusted APIs | Scientific, safety-critical, and mission-critical applications |
|
|
115
|
+
| `TIER_2` | High trust | Cleaned secondary datasets, indirect measurements, partially validated or derived information | Analytics, forecasting, and most production workloads |
|
|
116
|
+
| `TIER_3` | Moderate trust | User-generated content, self-reported information, weakly verified external sources | Exploratory analysis and workflows requiring additional validation |
|
|
117
|
+
|
|
118
|
+
## Intelligent Decision Pipeline
|
|
119
|
+
|
|
120
|
+
Based on the analysis results, data follows one of two paths:
|
|
121
|
+
|
|
122
|
+
- Rejected data: records failing minimum reliability requirements are isolated and excluded from trusted datasets.
|
|
123
|
+
- Accepted data: records meeting validation thresholds can be stored in trusted repositories while remaining fully traceable.
|
|
124
|
+
|
|
125
|
+
Every accepted record retains its reliability score, trust tier, provenance metadata, validation evidence, cryptographic integrity information, and audit context. Trust is not only calculated once; it remains transparent and reproducible throughout the data lifecycle.
|
|
126
|
+
|
|
127
|
+
## Transparent Scoring
|
|
128
|
+
|
|
129
|
+
The scanner creates a numeric score by applying profile weights to the evidence dimensions. Each evidence value is normalized from `0.0` to `1.0`, multiplied by the active profile's weight, and converted to a `0` to `100` score. Missing timestamps apply an additional penalty when timestamp verification is required.
|
|
130
|
+
|
|
131
|
+
Trust tiers are then assigned by explicit profile criteria:
|
|
132
|
+
|
|
133
|
+
- `TIER_1`: the record must pass the Tier 1 minimum score and all required evidence thresholds.
|
|
134
|
+
- `TIER_2`: the record must pass the Tier 2 minimum score and all required evidence thresholds.
|
|
135
|
+
- `TIER_3`: fallback for records that do not satisfy Tier 1 or Tier 2.
|
|
136
|
+
|
|
137
|
+
The package includes three profiles:
|
|
138
|
+
|
|
139
|
+
| Profile | Purpose | Tier behavior |
|
|
140
|
+
| --- | --- | --- |
|
|
141
|
+
| `default_profile()` | General application, API, and analytics data | Tier 1 requires high score, provenance, cryptographic verification, schema compliance, and verified timestamp |
|
|
142
|
+
| `scientific_profile()` | Research data across scientific fields | Increases weight and thresholds for provenance, calibration, consistency, anomaly checks, and reproducibility signals |
|
|
143
|
+
| `climate_record_profile()` | Weather and climate record data | Emphasizes calibrated instruments, station metadata, consistency with comparison stations, anomaly checks, and provenance |
|
|
144
|
+
|
|
145
|
+
The scoring model is stable within a profile. Different use cases change their acceptance thresholds or active profile, not the meaning of reliability itself.
|
|
146
|
+
|
|
147
|
+
Examples:
|
|
148
|
+
|
|
149
|
+
- A medical research study may require `TIER_1` data with a score of `99` or higher.
|
|
150
|
+
- An autonomous vehicle system may accept only `TIER_1` data above `95`.
|
|
151
|
+
- Financial risk models may require `TIER_1` or high-quality `TIER_2` data.
|
|
152
|
+
- Product analytics might accept `TIER_2` data with a threshold of `75`.
|
|
153
|
+
- Research prototypes may intentionally include `TIER_3` data while applying lower confidence weights.
|
|
154
|
+
|
|
155
|
+
Because every dataset is evaluated using the same transparent methodology, organizations can define their own acceptance criteria without changing how reliability is measured.
|
|
156
|
+
|
|
157
|
+
Instead of asking whether a dataset can be trusted, users can inspect objective metrics: reliability score, standardized trust tier, provenance, and validation history.
|
|
158
|
+
|
|
159
|
+
You can inspect why a record received its tier:
|
|
160
|
+
|
|
161
|
+
```python
|
|
162
|
+
from data_reliability import ReliabilityScanner, ValidationEvidence, climate_record_profile
|
|
163
|
+
|
|
164
|
+
scanner = ReliabilityScanner(profile=climate_record_profile())
|
|
165
|
+
evidence = ValidationEvidence(
|
|
166
|
+
completeness=0.98,
|
|
167
|
+
consistency=0.96,
|
|
168
|
+
provenance=0.96,
|
|
169
|
+
calibration=0.96,
|
|
170
|
+
schema_compliance=0.92,
|
|
171
|
+
anomaly_detection=0.96,
|
|
172
|
+
metadata_quality=0.60,
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
print(scanner.tier_evaluation(evidence))
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
For climate records, this makes methodological uncertainty visible. A temperature record with strong provenance and calibration can still fail Tier 1 if station metadata, anomaly checks, timestamp verification, or comparison-station consistency are insufficient.
|
|
179
|
+
|
|
180
|
+
## Database Usage
|
|
181
|
+
|
|
182
|
+
The SDK does not require a database driver. It emits plain dictionaries so the same reliability metadata can be stored in small local databases such as SQLite and DuckDB, production SQL databases such as PostgreSQL and MySQL, analytical warehouses, or document stores.
|
|
183
|
+
|
|
184
|
+
```python
|
|
185
|
+
from data_reliability import ValidationEvidence, scan_row
|
|
186
|
+
|
|
187
|
+
row = {"temperature": 21.4, "unit": "celsius"}
|
|
188
|
+
scored_row = scan_row(
|
|
189
|
+
row,
|
|
190
|
+
source_id="sensor-a",
|
|
191
|
+
evidence=ValidationEvidence(cryptographic_verification=1.0, calibration=1.0),
|
|
192
|
+
required_fields=["temperature", "unit"],
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
assert scored_row["dri_score"] >= 90
|
|
196
|
+
assert scored_row["dri_tier"] == 1
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
For SQL tables, store the generated `dri_*` columns beside the source data. For document databases, store `metadata_to_document(reliable.reliability)` as a nested reliability object.
|
|
200
|
+
|
|
201
|
+
## SDK and Security Notes
|
|
202
|
+
|
|
203
|
+
This project is intended to be used as an SDK. The core package keeps dependencies small, exposes typed Pydantic models, avoids dynamic code execution, and uses deterministic SHA-256 trace hashes to detect changed payloads. Hashes are integrity signals, not proof of source identity by themselves; use authenticated ingestion, signed upstream payloads, database permissions, and private vulnerability reporting for production systems.
|
|
204
|
+
|
|
205
|
+
## Pandas Filtering
|
|
206
|
+
|
|
207
|
+
```python
|
|
208
|
+
import pandas as pd
|
|
209
|
+
from data_reliability import DataTier, ReliabilityMetadata, ReliabilityPolicy, filter_reliable_df
|
|
210
|
+
|
|
211
|
+
df = pd.DataFrame([
|
|
212
|
+
{
|
|
213
|
+
"value": 10,
|
|
214
|
+
"reliability": ReliabilityMetadata(
|
|
215
|
+
score=95,
|
|
216
|
+
tier=DataTier.TIER_1,
|
|
217
|
+
source_id="sensor-a",
|
|
218
|
+
trace_hash="abc123",
|
|
219
|
+
),
|
|
220
|
+
},
|
|
221
|
+
])
|
|
222
|
+
|
|
223
|
+
policy = ReliabilityPolicy(minimum_score=90, maximum_tier=DataTier.TIER_2)
|
|
224
|
+
trusted = filter_reliable_df(df, policy)
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
## Documentation
|
|
228
|
+
|
|
229
|
+
The project documentation lives in [`docs/`](docs/) and can be served locally with MkDocs:
|
|
230
|
+
|
|
231
|
+
```bash
|
|
232
|
+
pip install -e ".[docs]"
|
|
233
|
+
mkdocs serve
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
Start with:
|
|
237
|
+
|
|
238
|
+
- [Concepts](docs/concepts.md)
|
|
239
|
+
- [Core models](docs/api/core.md)
|
|
240
|
+
- [Scanning engine](docs/api/scanner.md)
|
|
241
|
+
- [Database helpers](docs/api/database.md)
|
|
242
|
+
- [Pandas extension](docs/api/pandas.md)
|
|
243
|
+
- [FastAPI example](docs/api/fastapi.md)
|
|
244
|
+
- [Release and publishing](docs/release.md)
|
|
245
|
+
|
|
246
|
+
The longer project rationale is available in [`data-reliability.md`](data-reliability.md).
|
|
247
|
+
|
|
248
|
+
## Development
|
|
249
|
+
|
|
250
|
+
Run the test suite:
|
|
251
|
+
|
|
252
|
+
```bash
|
|
253
|
+
pip install -e ".[test]"
|
|
254
|
+
pytest
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
Build package artifacts:
|
|
258
|
+
|
|
259
|
+
```bash
|
|
260
|
+
pip install -e ".[build]"
|
|
261
|
+
python -m build
|
|
262
|
+
python -m twine check dist/*
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
Run the FastAPI example:
|
|
266
|
+
|
|
267
|
+
```bash
|
|
268
|
+
pip install -e ".[api]"
|
|
269
|
+
uvicorn examples.fastapi_app:app --reload
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
## Contributing
|
|
273
|
+
|
|
274
|
+
Contributions are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) for the local workflow and pull request expectations.
|
|
275
|
+
|
|
276
|
+
## Security
|
|
277
|
+
|
|
278
|
+
Please report security issues privately. See [SECURITY.md](SECURITY.md).
|
|
279
|
+
|
|
280
|
+
## License
|
|
281
|
+
|
|
282
|
+
Licensed under the [Apache License 2.0](LICENSE).
|
|
@@ -4,15 +4,15 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "data-reliability-index"
|
|
7
|
-
version = "0.
|
|
8
|
-
description = "
|
|
7
|
+
version = "0.3.0"
|
|
8
|
+
description = "SDK for scoring, tiering, and filtering data reliability evidence."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
11
11
|
license = "Apache-2.0"
|
|
12
12
|
authors = [
|
|
13
13
|
{ name = "h3pdesign" }
|
|
14
14
|
]
|
|
15
|
-
keywords = ["data-quality", "data-reliability", "validation", "pydantic", "
|
|
15
|
+
keywords = ["data-quality", "data-reliability", "validation", "pydantic", "sdk", "database"]
|
|
16
16
|
classifiers = [
|
|
17
17
|
"Development Status :: 3 - Alpha",
|
|
18
18
|
"Intended Audience :: Developers",
|
|
@@ -22,20 +22,24 @@ classifiers = [
|
|
|
22
22
|
"Programming Language :: Python :: 3.10",
|
|
23
23
|
"Programming Language :: Python :: 3.11",
|
|
24
24
|
"Programming Language :: Python :: 3.12",
|
|
25
|
+
"Programming Language :: Python :: 3.13",
|
|
26
|
+
"Programming Language :: Python :: 3.14",
|
|
27
|
+
"Typing :: Typed",
|
|
25
28
|
"Topic :: Scientific/Engineering",
|
|
29
|
+
"Topic :: Database",
|
|
26
30
|
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
27
31
|
]
|
|
28
32
|
dependencies = [
|
|
29
|
-
"pydantic>=2.0"
|
|
30
|
-
"pandas>=2.0",
|
|
31
|
-
"fastapi>=0.100",
|
|
32
|
-
"uvicorn>=0.20"
|
|
33
|
+
"pydantic>=2.0"
|
|
33
34
|
]
|
|
34
35
|
|
|
35
36
|
[project.optional-dependencies]
|
|
36
37
|
build = ["build", "twine"]
|
|
37
38
|
docs = ["black", "mkdocs-material", "mkdocstrings[python]"]
|
|
38
|
-
|
|
39
|
+
pandas = ["pandas>=2.0"]
|
|
40
|
+
api = ["fastapi>=0.100", "uvicorn>=0.20"]
|
|
41
|
+
test = ["pytest", "pandas>=2.0"]
|
|
42
|
+
all = ["pandas>=2.0", "fastapi>=0.100", "uvicorn>=0.20"]
|
|
39
43
|
|
|
40
44
|
[project.urls]
|
|
41
45
|
Homepage = "https://github.com/h3pdesign/data-reliability-index"
|