oribi 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oribi-0.1.0/CHANGELOG.md +10 -0
- oribi-0.1.0/LICENSE +21 -0
- oribi-0.1.0/MANIFEST.in +4 -0
- oribi-0.1.0/PKG-INFO +120 -0
- oribi-0.1.0/README.md +90 -0
- oribi-0.1.0/examples/quickstart.py +21 -0
- oribi-0.1.0/oribi/__init__.py +24 -0
- oribi-0.1.0/oribi/api.py +84 -0
- oribi-0.1.0/oribi/exceptions.py +9 -0
- oribi-0.1.0/oribi/models/__init__.py +7 -0
- oribi-0.1.0/oribi/models/context.py +59 -0
- oribi-0.1.0/oribi/models/evidence.py +80 -0
- oribi-0.1.0/oribi/models/profile.py +36 -0
- oribi-0.1.0/oribi/profiling/__init__.py +1 -0
- oribi-0.1.0/oribi/profiling/columns.py +49 -0
- oribi-0.1.0/oribi/profiling/observations.py +66 -0
- oribi-0.1.0/oribi/profiling/relationships.py +39 -0
- oribi-0.1.0/oribi/profiling/temporal.py +75 -0
- oribi-0.1.0/oribi/reasoning/__init__.py +1 -0
- oribi-0.1.0/oribi/reasoning/keys.py +169 -0
- oribi-0.1.0/oribi/reasoning/rules.py +30 -0
- oribi-0.1.0/oribi/reasoning/semantic.py +67 -0
- oribi-0.1.0/oribi/serialization/__init__.py +1 -0
- oribi-0.1.0/oribi/serialization/context.py +51 -0
- oribi-0.1.0/oribi/serialization/dict.py +39 -0
- oribi-0.1.0/oribi/serialization/markdown.py +36 -0
- oribi-0.1.0/oribi.egg-info/PKG-INFO +120 -0
- oribi-0.1.0/oribi.egg-info/SOURCES.txt +37 -0
- oribi-0.1.0/oribi.egg-info/dependency_links.txt +1 -0
- oribi-0.1.0/oribi.egg-info/requires.txt +6 -0
- oribi-0.1.0/oribi.egg-info/top_level.txt +1 -0
- oribi-0.1.0/pyproject.toml +41 -0
- oribi-0.1.0/setup.cfg +4 -0
- oribi-0.1.0/tests/test_api.py +27 -0
- oribi-0.1.0/tests/test_keys_grain.py +131 -0
- oribi-0.1.0/tests/test_profiling.py +109 -0
- oribi-0.1.0/tests/test_reasoning.py +17 -0
- oribi-0.1.0/tests/test_semantics.py +54 -0
- oribi-0.1.0/tests/test_serialization.py +33 -0
oribi-0.1.0/CHANGELOG.md
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to Oribi are recorded here.
|
|
4
|
+
|
|
5
|
+
## 0.1.0 — Initial release
|
|
6
|
+
|
|
7
|
+
- Introduce the `oribi` pandas DataFrame profiling API.
|
|
8
|
+
- Report column statistics, evidence-backed semantic roles, recommendations, warnings, relationships, candidate keys, and likely grain.
|
|
9
|
+
- Add conservative temporal-gap warnings for regular datetime series.
|
|
10
|
+
- Provide summary, dictionary, JSON, Markdown, and bounded AI-oriented context output.
|
oribi-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Oribi contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
oribi-0.1.0/MANIFEST.in
ADDED
oribi-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: oribi
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Evidence-backed analytical context for pandas DataFrames
|
|
5
|
+
Author: Oribi contributors
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/samhoon000/Oribi
|
|
8
|
+
Project-URL: Repository, https://github.com/samhoon000/Oribi
|
|
9
|
+
Project-URL: Issues, https://github.com/samhoon000/Oribi/issues
|
|
10
|
+
Keywords: pandas,data-profiling,data-analysis,data-quality,analytics
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
21
|
+
Requires-Python: <3.15,>=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Requires-Dist: pandas<4,>=1.5.3
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
27
|
+
Requires-Dist: pytest-cov>=6; extra == "dev"
|
|
28
|
+
Requires-Dist: build>=1.2; extra == "dev"
|
|
29
|
+
Dynamic: license-file
|
|
30
|
+
|
|
31
|
+
# Oribi
|
|
32
|
+
|
|
33
|
+
Oribi profiles a pandas DataFrame and returns a structured, evidence-backed snapshot of its shape, column statistics, likely semantic roles, candidate keys, possible grain, relationships, and analytical warnings. Use it to inspect data before writing an analysis or to give downstream tools a compact description of a dataset.
|
|
34
|
+
|
|
35
|
+
## Install
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
python -m pip install oribi
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Oribi supports Python 3.10–3.14 and pandas 1.5.3 or newer (below pandas 4).
|
|
42
|
+
|
|
43
|
+
## Quick start
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
import pandas as pd
|
|
47
|
+
from oribi import inspect
|
|
48
|
+
|
|
49
|
+
df = pd.DataFrame({
|
|
50
|
+
"order_id": [101, 101, 102, 102],
|
|
51
|
+
"product_id": [7, 8, 7, 8],
|
|
52
|
+
"revenue": [25.0, 40.0, 30.0, -5.0],
|
|
53
|
+
})
|
|
54
|
+
|
|
55
|
+
context = inspect(df)
|
|
56
|
+
print(context.summary())
|
|
57
|
+
print(context.to_context(max_chars=2_000))
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
`inspect()` does not modify the DataFrame or keep a reference to it in the result. The returned snapshot can also be rendered as a full Markdown report or serialized to JSON-compatible data:
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
markdown = context.to_markdown()
|
|
64
|
+
payload = context.to_dict()
|
|
65
|
+
json_text = context.to_json()
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## What Oribi currently reports
|
|
69
|
+
|
|
70
|
+
- Dataset row/column counts and duplicate rows.
|
|
71
|
+
- Per-column missingness, distinct counts, numeric summaries, categorical top values, and datetime bounds.
|
|
72
|
+
- Likely roles such as identifier, measure, dimension, categorical, temporal, boolean, text, or unknown, with evidence.
|
|
73
|
+
- Candidate single-column and bounded two-column keys, plus a separate likely-grain estimate.
|
|
74
|
+
- Conservative within-DataFrame functional-dependency relationships.
|
|
75
|
+
- Recommendations for likely numeric measures and identifier counts.
|
|
76
|
+
- Warnings for duplicate rows, missingness, non-unique identifiers, negative or skewed measures, high-cardinality categories, and possible gaps in regular datetime series.
|
|
77
|
+
|
|
78
|
+
Inferences are heuristics, not guarantees about business meaning. Temporal-gap warnings require a sufficiently long series with a dominant fixed cadence; irregular event timestamps and short series are left without a gap claim. Relationships describe observed value mappings within one frame and do not prove that a join is valid or safe.
|
|
79
|
+
|
|
80
|
+
## Public API
|
|
81
|
+
|
|
82
|
+
The main operation is `oribi.inspect(df, *, options=None)`. It returns a `DatasetContext` snapshot. `ProfileOptions` can bound top values and exact key/quantile analysis:
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
import pandas as pd
|
|
86
|
+
from oribi import ProfileOptions, inspect
|
|
87
|
+
|
|
88
|
+
df = pd.DataFrame({"amount": [12.5, 18.0, 20.0]})
|
|
89
|
+
context = inspect(df, options=ProfileOptions(top_k=5, max_exact_rows=50_000))
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Options include `top_k`, `include_quantiles`, `max_exact_rows`, and `max_key_columns`. Column labels must be unique strings. Datetime-like text is not parsed automatically. Profiling is in-memory and may require several full-column scans; exact composite-key checks are bounded by the configured row and column limits.
|
|
93
|
+
|
|
94
|
+
Confidence values describe heuristic strength; they are not calibrated probabilities. A candidate key is evidence of observed uniqueness under the profiling rules, not proof of a business key. Serialized output includes `schema_version`; treat changes to that output as version-sensitive until a stable compatibility policy is declared.
|
|
95
|
+
|
|
96
|
+
## Development
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
git clone https://github.com/samhoon000/Oribi.git
|
|
100
|
+
cd Oribi
|
|
101
|
+
python -m venv .venv
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Activate the environment, then install the package and development tools:
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
python -m pip install -e ".[dev]"
|
|
108
|
+
pytest -q
|
|
109
|
+
python -m build
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for the contribution workflow and [DESIGN.md](DESIGN.md) for architecture notes. Run the end-to-end example from the repository root with `python -m examples.quickstart`.
|
|
113
|
+
|
|
114
|
+
## Planned work
|
|
115
|
+
|
|
116
|
+
Additional data sources, configurable semantic rules, cross-table foreign-key analysis, and richer profiling options are future work. The current release focuses on pandas DataFrames and deterministic, evidence-bearing heuristics.
|
|
117
|
+
|
|
118
|
+
## License
|
|
119
|
+
|
|
120
|
+
Oribi is released under the [MIT License](LICENSE).
|
oribi-0.1.0/README.md
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# Oribi
|
|
2
|
+
|
|
3
|
+
Oribi profiles a pandas DataFrame and returns a structured, evidence-backed snapshot of its shape, column statistics, likely semantic roles, candidate keys, possible grain, relationships, and analytical warnings. Use it to inspect data before writing an analysis or to give downstream tools a compact description of a dataset.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
python -m pip install oribi
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
Oribi supports Python 3.10–3.14 and pandas 1.5.3 or newer (below pandas 4).
|
|
12
|
+
|
|
13
|
+
## Quick start
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
import pandas as pd
|
|
17
|
+
from oribi import inspect
|
|
18
|
+
|
|
19
|
+
df = pd.DataFrame({
|
|
20
|
+
"order_id": [101, 101, 102, 102],
|
|
21
|
+
"product_id": [7, 8, 7, 8],
|
|
22
|
+
"revenue": [25.0, 40.0, 30.0, -5.0],
|
|
23
|
+
})
|
|
24
|
+
|
|
25
|
+
context = inspect(df)
|
|
26
|
+
print(context.summary())
|
|
27
|
+
print(context.to_context(max_chars=2_000))
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
`inspect()` does not modify the DataFrame or keep a reference to it in the result. The returned snapshot can also be rendered as a full Markdown report or serialized to JSON-compatible data:
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
markdown = context.to_markdown()
|
|
34
|
+
payload = context.to_dict()
|
|
35
|
+
json_text = context.to_json()
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## What Oribi currently reports
|
|
39
|
+
|
|
40
|
+
- Dataset row/column counts and duplicate rows.
|
|
41
|
+
- Per-column missingness, distinct counts, numeric summaries, categorical top values, and datetime bounds.
|
|
42
|
+
- Likely roles such as identifier, measure, dimension, categorical, temporal, boolean, text, or unknown, with evidence.
|
|
43
|
+
- Candidate single-column and bounded two-column keys, plus a separate likely-grain estimate.
|
|
44
|
+
- Conservative within-DataFrame functional-dependency relationships.
|
|
45
|
+
- Recommendations for likely numeric measures and identifier counts.
|
|
46
|
+
- Warnings for duplicate rows, missingness, non-unique identifiers, negative or skewed measures, high-cardinality categories, and possible gaps in regular datetime series.
|
|
47
|
+
|
|
48
|
+
Inferences are heuristics, not guarantees about business meaning. Temporal-gap warnings require a sufficiently long series with a dominant fixed cadence; irregular event timestamps and short series are left without a gap claim. Relationships describe observed value mappings within one frame and do not prove that a join is valid or safe.
|
|
49
|
+
|
|
50
|
+
## Public API
|
|
51
|
+
|
|
52
|
+
The main operation is `oribi.inspect(df, *, options=None)`. It returns a `DatasetContext` snapshot. `ProfileOptions` can bound top values and exact key/quantile analysis:
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
import pandas as pd
|
|
56
|
+
from oribi import ProfileOptions, inspect
|
|
57
|
+
|
|
58
|
+
df = pd.DataFrame({"amount": [12.5, 18.0, 20.0]})
|
|
59
|
+
context = inspect(df, options=ProfileOptions(top_k=5, max_exact_rows=50_000))
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Options include `top_k`, `include_quantiles`, `max_exact_rows`, and `max_key_columns`. Column labels must be unique strings. Datetime-like text is not parsed automatically. Profiling is in-memory and may require several full-column scans; exact composite-key checks are bounded by the configured row and column limits.
|
|
63
|
+
|
|
64
|
+
Confidence values describe heuristic strength; they are not calibrated probabilities. A candidate key is evidence of observed uniqueness under the profiling rules, not proof of a business key. Serialized output includes `schema_version`; treat changes to that output as version-sensitive until a stable compatibility policy is declared.
|
|
65
|
+
|
|
66
|
+
## Development
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
git clone https://github.com/samhoon000/Oribi.git
|
|
70
|
+
cd Oribi
|
|
71
|
+
python -m venv .venv
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
Activate the environment, then install the package and development tools:
|
|
75
|
+
|
|
76
|
+
```bash
|
|
77
|
+
python -m pip install -e ".[dev]"
|
|
78
|
+
pytest -q
|
|
79
|
+
python -m build
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for the contribution workflow and [DESIGN.md](DESIGN.md) for architecture notes. Run the end-to-end example from the repository root with `python -m examples.quickstart`.
|
|
83
|
+
|
|
84
|
+
## Planned work
|
|
85
|
+
|
|
86
|
+
Additional data sources, configurable semantic rules, cross-table foreign-key analysis, and richer profiling options are future work. The current release focuses on pandas DataFrames and deterministic, evidence-bearing heuristics.
|
|
87
|
+
|
|
88
|
+
## License
|
|
89
|
+
|
|
90
|
+
Oribi is released under the [MIT License](LICENSE).
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Runnable Oribi example: profile a small order-items DataFrame."""
|
|
2
|
+
|
|
3
|
+
import pandas as pd
|
|
4
|
+
|
|
5
|
+
from oribi import inspect
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
orders = pd.DataFrame(
|
|
9
|
+
{
|
|
10
|
+
"order_id": [101, 101, 102, 102],
|
|
11
|
+
"product_id": [7, 8, 7, 8],
|
|
12
|
+
"revenue": [25.0, 40.0, 30.0, -5.0],
|
|
13
|
+
"order_date": pd.to_datetime(
|
|
14
|
+
["2026-01-01", "2026-01-01", "2026-01-02", "2026-01-02"]
|
|
15
|
+
),
|
|
16
|
+
}
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
context = inspect(orders)
|
|
20
|
+
print(context.summary())
|
|
21
|
+
print(context.to_context(max_chars=2_000))
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""Build structured analytical context from pandas DataFrames."""
|
|
2
|
+
|
|
3
|
+
from .api import ProfileOptions, inspect
|
|
4
|
+
from .exceptions import InvalidInputError, OribiError
|
|
5
|
+
from .models import (
|
|
6
|
+
ColumnContext,
|
|
7
|
+
DatasetContext,
|
|
8
|
+
Evidence,
|
|
9
|
+
GrainContext,
|
|
10
|
+
KeyCandidate,
|
|
11
|
+
Recommendation,
|
|
12
|
+
RelationshipContext,
|
|
13
|
+
SemanticContext,
|
|
14
|
+
StatisticsContext,
|
|
15
|
+
Warning,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"ColumnContext", "DatasetContext", "Evidence", "GrainContext", "InvalidInputError",
|
|
20
|
+
"KeyCandidate", "ProfileOptions", "Recommendation", "RelationshipContext",
|
|
21
|
+
"OribiError", "SemanticContext", "StatisticsContext", "Warning", "inspect",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
__version__ = "0.1.0"
|
oribi-0.1.0/oribi/api.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Public inspection API and pipeline orchestration."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
from .exceptions import InvalidInputError
|
|
8
|
+
from .models.context import ColumnContext, DatasetContext
|
|
9
|
+
from .models.evidence import Warning
|
|
10
|
+
from .models.profile import DatasetProfile
|
|
11
|
+
from .profiling.columns import profile_column
|
|
12
|
+
from .profiling.observations import AnalysisSession
|
|
13
|
+
from .profiling.relationships import detect_relationships
|
|
14
|
+
from .profiling.temporal import temporal_gap_warnings
|
|
15
|
+
from .reasoning.keys import detect_keys
|
|
16
|
+
from .reasoning.rules import dataset_warnings, reason_column
|
|
17
|
+
from .reasoning.semantic import infer_semantic
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True)
|
|
21
|
+
class ProfileOptions:
|
|
22
|
+
"""Controls profiling work while keeping results deterministic."""
|
|
23
|
+
|
|
24
|
+
top_k: int = 10
|
|
25
|
+
include_quantiles: bool = True
|
|
26
|
+
max_exact_rows: int = 100_000
|
|
27
|
+
max_key_columns: int = 8
|
|
28
|
+
|
|
29
|
+
def __post_init__(self) -> None:
|
|
30
|
+
if self.top_k < 1:
|
|
31
|
+
raise ValueError("top_k must be at least 1")
|
|
32
|
+
if self.max_exact_rows < 1 or self.max_key_columns < 2:
|
|
33
|
+
raise ValueError("max_exact_rows must be positive and max_key_columns at least 2")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def inspect(df: pd.DataFrame, *, options: ProfileOptions | None = None) -> DatasetContext:
|
|
37
|
+
"""Profile a pandas DataFrame and return its analytical context snapshot.
|
|
38
|
+
|
|
39
|
+
The returned context contains no reference to ``df``. Object columns that
|
|
40
|
+
appear datetime-like are not parsed automatically.
|
|
41
|
+
"""
|
|
42
|
+
options = options or ProfileOptions()
|
|
43
|
+
if not isinstance(options, ProfileOptions):
|
|
44
|
+
raise InvalidInputError("options must be a ProfileOptions instance")
|
|
45
|
+
session = AnalysisSession(df, options)
|
|
46
|
+
nrows, ncols = df.shape
|
|
47
|
+
try:
|
|
48
|
+
duplicates = int(df.duplicated().sum())
|
|
49
|
+
except (TypeError, ValueError):
|
|
50
|
+
duplicates = None
|
|
51
|
+
duplicate_rate = (duplicates / nrows) if duplicates is not None and nrows else (0.0 if duplicates == 0 else None)
|
|
52
|
+
names = list(df.columns)
|
|
53
|
+
observations = {name: session.observe(name) for name in names}
|
|
54
|
+
stats = {name: profile_column(obs, nrows, options.top_k, options.include_quantiles and nrows <= options.max_exact_rows) for name, obs in observations.items()}
|
|
55
|
+
semantics = {name: infer_semantic(name, observations[name].logical_type, stats[name], nrows) for name in names}
|
|
56
|
+
key_candidates, grain = detect_keys(
|
|
57
|
+
df, names, stats, {name: semantics[name].role for name in names},
|
|
58
|
+
max_columns=options.max_key_columns, max_exact_rows=options.max_exact_rows,
|
|
59
|
+
)
|
|
60
|
+
role_map = {name: semantics[name].role for name in names}
|
|
61
|
+
relationships = detect_relationships(df, role_map, max_rows=options.max_exact_rows)
|
|
62
|
+
columns = []
|
|
63
|
+
for name in names:
|
|
64
|
+
obs = observations[name]
|
|
65
|
+
recommendations, warnings = reason_column(name, obs.logical_type, semantics[name], stats[name])
|
|
66
|
+
if obs.logical_type == "temporal":
|
|
67
|
+
warnings += temporal_gap_warnings(obs.series, name)
|
|
68
|
+
columns.append(ColumnContext(name, str(obs.series.dtype), obs.logical_type, stats[name], semantics[name], recommendations, warnings))
|
|
69
|
+
global_warnings = list(dataset_warnings(duplicates, nrows))
|
|
70
|
+
for col in columns:
|
|
71
|
+
global_warnings.extend(col.warnings)
|
|
72
|
+
if duplicates is None:
|
|
73
|
+
global_warnings.append(Warning(
|
|
74
|
+
"duplicate_check_unavailable", "low", "Duplicate-row check unavailable",
|
|
75
|
+
"Pandas could not compare rows with the available cell values.",
|
|
76
|
+
))
|
|
77
|
+
return DatasetContext(
|
|
78
|
+
profile=DatasetProfile(nrows, ncols, duplicates, duplicate_rate),
|
|
79
|
+
columns=tuple(columns),
|
|
80
|
+
candidate_keys=key_candidates,
|
|
81
|
+
grain=grain,
|
|
82
|
+
relationships=relationships,
|
|
83
|
+
warnings=tuple(global_warnings),
|
|
84
|
+
)
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Public context models."""
|
|
2
|
+
|
|
3
|
+
from .context import ColumnContext, DatasetContext
|
|
4
|
+
from .evidence import Evidence, GrainContext, KeyCandidate, Recommendation, RelationshipContext, SemanticContext, Warning
|
|
5
|
+
from .profile import StatisticsContext
|
|
6
|
+
|
|
7
|
+
__all__ = ["ColumnContext", "DatasetContext", "Evidence", "GrainContext", "KeyCandidate", "Recommendation", "RelationshipContext", "SemanticContext", "StatisticsContext", "Warning"]
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Canonical result returned by :func:`oribi.inspect`."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from .evidence import GrainContext, KeyCandidate, Recommendation, RelationshipContext, SemanticContext, Warning
|
|
7
|
+
from .profile import DatasetProfile, StatisticsContext
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True)
|
|
11
|
+
class ColumnContext:
|
|
12
|
+
name: str
|
|
13
|
+
dtype: str
|
|
14
|
+
logical_type: str
|
|
15
|
+
statistics: StatisticsContext
|
|
16
|
+
semantic: SemanticContext
|
|
17
|
+
recommendations: tuple[Recommendation, ...] = ()
|
|
18
|
+
warnings: tuple[Warning, ...] = ()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class DatasetContext:
|
|
23
|
+
profile: DatasetProfile
|
|
24
|
+
columns: tuple[ColumnContext, ...]
|
|
25
|
+
candidate_keys: tuple[KeyCandidate, ...] = ()
|
|
26
|
+
grain: GrainContext | None = None
|
|
27
|
+
relationships: tuple[RelationshipContext, ...] = ()
|
|
28
|
+
warnings: tuple[Warning, ...] = ()
|
|
29
|
+
schema_version: str = "1"
|
|
30
|
+
|
|
31
|
+
def to_dict(self) -> dict[str, Any]:
|
|
32
|
+
from ..serialization.dict import to_dict
|
|
33
|
+
return to_dict(self)
|
|
34
|
+
|
|
35
|
+
def to_json(self, *, indent: int | None = 2) -> str:
|
|
36
|
+
from ..serialization.dict import to_json
|
|
37
|
+
return to_json(self, indent=indent)
|
|
38
|
+
|
|
39
|
+
def to_markdown(self) -> str:
|
|
40
|
+
from ..serialization.markdown import to_markdown
|
|
41
|
+
return to_markdown(self)
|
|
42
|
+
|
|
43
|
+
def to_context(self, *, max_chars: int = 8000) -> str:
|
|
44
|
+
from ..serialization.context import to_context
|
|
45
|
+
return to_context(self, max_chars=max_chars)
|
|
46
|
+
|
|
47
|
+
def summary(self) -> str:
|
|
48
|
+
rows = self.profile.row_count
|
|
49
|
+
cols = self.profile.column_count
|
|
50
|
+
parts = [f"Dataset: {rows:,} rows × {cols:,} columns."]
|
|
51
|
+
if self.profile.duplicate_row_count:
|
|
52
|
+
parts.append(f"{self.profile.duplicate_row_count:,} duplicate rows observed.")
|
|
53
|
+
if self.grain and self.grain.status == "estimated" and self.grain.label:
|
|
54
|
+
parts.append(f"Likely grain: {self.grain.label} (confidence {self.grain.confidence:.2f}).")
|
|
55
|
+
elif self.grain:
|
|
56
|
+
parts.append("Likely grain could not be determined confidently.")
|
|
57
|
+
if self.warnings:
|
|
58
|
+
parts.append(f"{len(self.warnings)} analytical warning(s) detected.")
|
|
59
|
+
return " ".join(parts)
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""Evidence-bearing inferences and recommendations."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from collections.abc import Mapping
|
|
5
|
+
from types import MappingProxyType
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _freeze(value: Any) -> Any:
|
|
10
|
+
if isinstance(value, Mapping):
|
|
11
|
+
return MappingProxyType({key: _freeze(item) for key, item in value.items()})
|
|
12
|
+
if isinstance(value, (list, tuple)):
|
|
13
|
+
return tuple(_freeze(item) for item in value)
|
|
14
|
+
if isinstance(value, set):
|
|
15
|
+
return frozenset(_freeze(item) for item in value)
|
|
16
|
+
return value
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True)
|
|
20
|
+
class Evidence:
|
|
21
|
+
code: str
|
|
22
|
+
statement: str
|
|
23
|
+
details: Mapping[str, Any] = field(default_factory=dict)
|
|
24
|
+
|
|
25
|
+
def __post_init__(self) -> None:
|
|
26
|
+
object.__setattr__(self, "details", _freeze(self.details))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class SemanticContext:
|
|
31
|
+
role: str
|
|
32
|
+
confidence: float
|
|
33
|
+
evidence: tuple[Evidence, ...] = ()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class Recommendation:
|
|
38
|
+
operation: str
|
|
39
|
+
disposition: str
|
|
40
|
+
rationale: str
|
|
41
|
+
evidence: tuple[Evidence, ...] = ()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True)
|
|
45
|
+
class Warning:
|
|
46
|
+
code: str
|
|
47
|
+
severity: str
|
|
48
|
+
title: str
|
|
49
|
+
message: str
|
|
50
|
+
evidence: tuple[Evidence, ...] = ()
|
|
51
|
+
implication: str = ""
|
|
52
|
+
recommendation: str | None = None
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass(frozen=True)
|
|
56
|
+
class KeyCandidate:
|
|
57
|
+
columns: tuple[str, ...]
|
|
58
|
+
kind: str
|
|
59
|
+
confidence: float
|
|
60
|
+
uniqueness_rate: float
|
|
61
|
+
evidence: tuple[Evidence, ...] = ()
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass(frozen=True)
|
|
65
|
+
class GrainContext:
|
|
66
|
+
status: str
|
|
67
|
+
label: str | None
|
|
68
|
+
columns: tuple[str, ...]
|
|
69
|
+
confidence: float
|
|
70
|
+
evidence: tuple[Evidence, ...] = ()
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass(frozen=True)
|
|
74
|
+
class RelationshipContext:
|
|
75
|
+
source_column: str
|
|
76
|
+
target_column: str
|
|
77
|
+
kind: str
|
|
78
|
+
coverage: float
|
|
79
|
+
confidence: float
|
|
80
|
+
evidence: tuple[Evidence, ...] = ()
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Observed dataset and column statistics."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from collections.abc import Mapping
|
|
5
|
+
from types import MappingProxyType
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass(frozen=True)
|
|
10
|
+
class StatisticsContext:
|
|
11
|
+
missing_count: int
|
|
12
|
+
missing_rate: float
|
|
13
|
+
distinct_count: int
|
|
14
|
+
distinct_rate: float
|
|
15
|
+
minimum: Any = None
|
|
16
|
+
maximum: Any = None
|
|
17
|
+
mean: float | None = None
|
|
18
|
+
median: float | None = None
|
|
19
|
+
standard_deviation: float | None = None
|
|
20
|
+
quantiles: Mapping[str, float] = field(default_factory=dict)
|
|
21
|
+
zero_count: int | None = None
|
|
22
|
+
negative_count: int | None = None
|
|
23
|
+
skewness: float | None = None
|
|
24
|
+
top_values: tuple[tuple[Any, int], ...] = ()
|
|
25
|
+
date_range: str | None = None
|
|
26
|
+
|
|
27
|
+
def __post_init__(self) -> None:
|
|
28
|
+
object.__setattr__(self, "quantiles", MappingProxyType(dict(self.quantiles)))
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class DatasetProfile:
|
|
33
|
+
row_count: int
|
|
34
|
+
column_count: int
|
|
35
|
+
duplicate_row_count: int | None
|
|
36
|
+
duplicate_row_rate: float | None
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Internal profiling components."""
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Statistics for observed columns."""
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
from ..models.profile import StatisticsContext
|
|
9
|
+
from .observations import ColumnObservation
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def profile_column(obs: ColumnObservation, row_count: int, top_k: int, include_quantiles: bool) -> StatisticsContext:
|
|
13
|
+
s = obs.series
|
|
14
|
+
denom = row_count or 1
|
|
15
|
+
non_null = s.dropna()
|
|
16
|
+
values: dict[str, Any] = {}
|
|
17
|
+
if obs.logical_type == "numeric" and len(non_null):
|
|
18
|
+
numeric = pd.to_numeric(non_null, errors="coerce").dropna()
|
|
19
|
+
if len(numeric):
|
|
20
|
+
def finite_or_none(value: Any) -> float | None:
|
|
21
|
+
result = float(value)
|
|
22
|
+
return result if math.isfinite(result) else None
|
|
23
|
+
|
|
24
|
+
values.update(minimum=numeric.min(), maximum=numeric.max(), mean=finite_or_none(numeric.mean()), median=finite_or_none(numeric.median()), standard_deviation=finite_or_none(numeric.std()))
|
|
25
|
+
values["zero_count"] = int((numeric == 0).sum())
|
|
26
|
+
values["negative_count"] = int((numeric < 0).sum())
|
|
27
|
+
if include_quantiles:
|
|
28
|
+
qs = numeric.quantile([.25, .5, .75])
|
|
29
|
+
values["quantiles"] = {str(q): float(v) for q, v in qs.items()}
|
|
30
|
+
if len(numeric) >= 3 and numeric.nunique() > 1:
|
|
31
|
+
skew = float(numeric.skew())
|
|
32
|
+
values["skewness"] = skew if pd.notna(skew) else None
|
|
33
|
+
elif obs.logical_type == "temporal" and len(non_null):
|
|
34
|
+
values["minimum"] = non_null.min()
|
|
35
|
+
values["maximum"] = non_null.max()
|
|
36
|
+
values["date_range"] = f"{values['minimum']} to {values['maximum']}"
|
|
37
|
+
if len(non_null) and (obs.logical_type in {"text", "categorical", "boolean", "unknown"} or obs.distinct_count <= top_k * 4):
|
|
38
|
+
try:
|
|
39
|
+
counts = non_null.value_counts(dropna=True).head(top_k)
|
|
40
|
+
values["top_values"] = tuple((k, int(v)) for k, v in counts.items())
|
|
41
|
+
except (TypeError, ValueError):
|
|
42
|
+
pass
|
|
43
|
+
return StatisticsContext(
|
|
44
|
+
missing_count=obs.missing_count,
|
|
45
|
+
missing_rate=obs.missing_count / denom,
|
|
46
|
+
distinct_count=obs.distinct_count,
|
|
47
|
+
distinct_rate=obs.distinct_count / denom,
|
|
48
|
+
**values,
|
|
49
|
+
)
|