oribi 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oribi/__init__.py +24 -0
- oribi/api.py +84 -0
- oribi/exceptions.py +9 -0
- oribi/models/__init__.py +7 -0
- oribi/models/context.py +59 -0
- oribi/models/evidence.py +80 -0
- oribi/models/profile.py +36 -0
- oribi/profiling/__init__.py +1 -0
- oribi/profiling/columns.py +49 -0
- oribi/profiling/observations.py +66 -0
- oribi/profiling/relationships.py +39 -0
- oribi/profiling/temporal.py +75 -0
- oribi/reasoning/__init__.py +1 -0
- oribi/reasoning/keys.py +169 -0
- oribi/reasoning/rules.py +30 -0
- oribi/reasoning/semantic.py +67 -0
- oribi/serialization/__init__.py +1 -0
- oribi/serialization/context.py +51 -0
- oribi/serialization/dict.py +39 -0
- oribi/serialization/markdown.py +36 -0
- oribi-0.1.0.dist-info/METADATA +120 -0
- oribi-0.1.0.dist-info/RECORD +25 -0
- oribi-0.1.0.dist-info/WHEEL +5 -0
- oribi-0.1.0.dist-info/licenses/LICENSE +21 -0
- oribi-0.1.0.dist-info/top_level.txt +1 -0
oribi/__init__.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""Build structured analytical context from pandas DataFrames."""
|
|
2
|
+
|
|
3
|
+
from .api import ProfileOptions, inspect
|
|
4
|
+
from .exceptions import InvalidInputError, OribiError
|
|
5
|
+
from .models import (
|
|
6
|
+
ColumnContext,
|
|
7
|
+
DatasetContext,
|
|
8
|
+
Evidence,
|
|
9
|
+
GrainContext,
|
|
10
|
+
KeyCandidate,
|
|
11
|
+
Recommendation,
|
|
12
|
+
RelationshipContext,
|
|
13
|
+
SemanticContext,
|
|
14
|
+
StatisticsContext,
|
|
15
|
+
Warning,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"ColumnContext", "DatasetContext", "Evidence", "GrainContext", "InvalidInputError",
|
|
20
|
+
"KeyCandidate", "ProfileOptions", "Recommendation", "RelationshipContext",
|
|
21
|
+
"OribiError", "SemanticContext", "StatisticsContext", "Warning", "inspect",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
__version__ = "0.1.0"
|
oribi/api.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Public inspection API and pipeline orchestration."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
from .exceptions import InvalidInputError
|
|
8
|
+
from .models.context import ColumnContext, DatasetContext
|
|
9
|
+
from .models.evidence import Warning
|
|
10
|
+
from .models.profile import DatasetProfile
|
|
11
|
+
from .profiling.columns import profile_column
|
|
12
|
+
from .profiling.observations import AnalysisSession
|
|
13
|
+
from .profiling.relationships import detect_relationships
|
|
14
|
+
from .profiling.temporal import temporal_gap_warnings
|
|
15
|
+
from .reasoning.keys import detect_keys
|
|
16
|
+
from .reasoning.rules import dataset_warnings, reason_column
|
|
17
|
+
from .reasoning.semantic import infer_semantic
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True)
|
|
21
|
+
class ProfileOptions:
|
|
22
|
+
"""Controls profiling work while keeping results deterministic."""
|
|
23
|
+
|
|
24
|
+
top_k: int = 10
|
|
25
|
+
include_quantiles: bool = True
|
|
26
|
+
max_exact_rows: int = 100_000
|
|
27
|
+
max_key_columns: int = 8
|
|
28
|
+
|
|
29
|
+
def __post_init__(self) -> None:
|
|
30
|
+
if self.top_k < 1:
|
|
31
|
+
raise ValueError("top_k must be at least 1")
|
|
32
|
+
if self.max_exact_rows < 1 or self.max_key_columns < 2:
|
|
33
|
+
raise ValueError("max_exact_rows must be positive and max_key_columns at least 2")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def inspect(df: pd.DataFrame, *, options: ProfileOptions | None = None) -> DatasetContext:
|
|
37
|
+
"""Profile a pandas DataFrame and return its analytical context snapshot.
|
|
38
|
+
|
|
39
|
+
The returned context contains no reference to ``df``. Object columns that
|
|
40
|
+
appear datetime-like are not parsed automatically.
|
|
41
|
+
"""
|
|
42
|
+
options = options or ProfileOptions()
|
|
43
|
+
if not isinstance(options, ProfileOptions):
|
|
44
|
+
raise InvalidInputError("options must be a ProfileOptions instance")
|
|
45
|
+
session = AnalysisSession(df, options)
|
|
46
|
+
nrows, ncols = df.shape
|
|
47
|
+
try:
|
|
48
|
+
duplicates = int(df.duplicated().sum())
|
|
49
|
+
except (TypeError, ValueError):
|
|
50
|
+
duplicates = None
|
|
51
|
+
duplicate_rate = (duplicates / nrows) if duplicates is not None and nrows else (0.0 if duplicates == 0 else None)
|
|
52
|
+
names = list(df.columns)
|
|
53
|
+
observations = {name: session.observe(name) for name in names}
|
|
54
|
+
stats = {name: profile_column(obs, nrows, options.top_k, options.include_quantiles and nrows <= options.max_exact_rows) for name, obs in observations.items()}
|
|
55
|
+
semantics = {name: infer_semantic(name, observations[name].logical_type, stats[name], nrows) for name in names}
|
|
56
|
+
key_candidates, grain = detect_keys(
|
|
57
|
+
df, names, stats, {name: semantics[name].role for name in names},
|
|
58
|
+
max_columns=options.max_key_columns, max_exact_rows=options.max_exact_rows,
|
|
59
|
+
)
|
|
60
|
+
role_map = {name: semantics[name].role for name in names}
|
|
61
|
+
relationships = detect_relationships(df, role_map, max_rows=options.max_exact_rows)
|
|
62
|
+
columns = []
|
|
63
|
+
for name in names:
|
|
64
|
+
obs = observations[name]
|
|
65
|
+
recommendations, warnings = reason_column(name, obs.logical_type, semantics[name], stats[name])
|
|
66
|
+
if obs.logical_type == "temporal":
|
|
67
|
+
warnings += temporal_gap_warnings(obs.series, name)
|
|
68
|
+
columns.append(ColumnContext(name, str(obs.series.dtype), obs.logical_type, stats[name], semantics[name], recommendations, warnings))
|
|
69
|
+
global_warnings = list(dataset_warnings(duplicates, nrows))
|
|
70
|
+
for col in columns:
|
|
71
|
+
global_warnings.extend(col.warnings)
|
|
72
|
+
if duplicates is None:
|
|
73
|
+
global_warnings.append(Warning(
|
|
74
|
+
"duplicate_check_unavailable", "low", "Duplicate-row check unavailable",
|
|
75
|
+
"Pandas could not compare rows with the available cell values.",
|
|
76
|
+
))
|
|
77
|
+
return DatasetContext(
|
|
78
|
+
profile=DatasetProfile(nrows, ncols, duplicates, duplicate_rate),
|
|
79
|
+
columns=tuple(columns),
|
|
80
|
+
candidate_keys=key_candidates,
|
|
81
|
+
grain=grain,
|
|
82
|
+
relationships=relationships,
|
|
83
|
+
warnings=tuple(global_warnings),
|
|
84
|
+
)
|
oribi/exceptions.py
ADDED
oribi/models/__init__.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Public context models."""
|
|
2
|
+
|
|
3
|
+
from .context import ColumnContext, DatasetContext
|
|
4
|
+
from .evidence import Evidence, GrainContext, KeyCandidate, Recommendation, RelationshipContext, SemanticContext, Warning
|
|
5
|
+
from .profile import StatisticsContext
|
|
6
|
+
|
|
7
|
+
__all__ = ["ColumnContext", "DatasetContext", "Evidence", "GrainContext", "KeyCandidate", "Recommendation", "RelationshipContext", "SemanticContext", "StatisticsContext", "Warning"]
|
oribi/models/context.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Canonical result returned by :func:`oribi.inspect`."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from .evidence import GrainContext, KeyCandidate, Recommendation, RelationshipContext, SemanticContext, Warning
|
|
7
|
+
from .profile import DatasetProfile, StatisticsContext
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True)
|
|
11
|
+
class ColumnContext:
|
|
12
|
+
name: str
|
|
13
|
+
dtype: str
|
|
14
|
+
logical_type: str
|
|
15
|
+
statistics: StatisticsContext
|
|
16
|
+
semantic: SemanticContext
|
|
17
|
+
recommendations: tuple[Recommendation, ...] = ()
|
|
18
|
+
warnings: tuple[Warning, ...] = ()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class DatasetContext:
|
|
23
|
+
profile: DatasetProfile
|
|
24
|
+
columns: tuple[ColumnContext, ...]
|
|
25
|
+
candidate_keys: tuple[KeyCandidate, ...] = ()
|
|
26
|
+
grain: GrainContext | None = None
|
|
27
|
+
relationships: tuple[RelationshipContext, ...] = ()
|
|
28
|
+
warnings: tuple[Warning, ...] = ()
|
|
29
|
+
schema_version: str = "1"
|
|
30
|
+
|
|
31
|
+
def to_dict(self) -> dict[str, Any]:
|
|
32
|
+
from ..serialization.dict import to_dict
|
|
33
|
+
return to_dict(self)
|
|
34
|
+
|
|
35
|
+
def to_json(self, *, indent: int | None = 2) -> str:
|
|
36
|
+
from ..serialization.dict import to_json
|
|
37
|
+
return to_json(self, indent=indent)
|
|
38
|
+
|
|
39
|
+
def to_markdown(self) -> str:
|
|
40
|
+
from ..serialization.markdown import to_markdown
|
|
41
|
+
return to_markdown(self)
|
|
42
|
+
|
|
43
|
+
def to_context(self, *, max_chars: int = 8000) -> str:
|
|
44
|
+
from ..serialization.context import to_context
|
|
45
|
+
return to_context(self, max_chars=max_chars)
|
|
46
|
+
|
|
47
|
+
def summary(self) -> str:
|
|
48
|
+
rows = self.profile.row_count
|
|
49
|
+
cols = self.profile.column_count
|
|
50
|
+
parts = [f"Dataset: {rows:,} rows × {cols:,} columns."]
|
|
51
|
+
if self.profile.duplicate_row_count:
|
|
52
|
+
parts.append(f"{self.profile.duplicate_row_count:,} duplicate rows observed.")
|
|
53
|
+
if self.grain and self.grain.status == "estimated" and self.grain.label:
|
|
54
|
+
parts.append(f"Likely grain: {self.grain.label} (confidence {self.grain.confidence:.2f}).")
|
|
55
|
+
elif self.grain:
|
|
56
|
+
parts.append("Likely grain could not be determined confidently.")
|
|
57
|
+
if self.warnings:
|
|
58
|
+
parts.append(f"{len(self.warnings)} analytical warning(s) detected.")
|
|
59
|
+
return " ".join(parts)
|
oribi/models/evidence.py
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""Evidence-bearing inferences and recommendations."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from collections.abc import Mapping
|
|
5
|
+
from types import MappingProxyType
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _freeze(value: Any) -> Any:
|
|
10
|
+
if isinstance(value, Mapping):
|
|
11
|
+
return MappingProxyType({key: _freeze(item) for key, item in value.items()})
|
|
12
|
+
if isinstance(value, (list, tuple)):
|
|
13
|
+
return tuple(_freeze(item) for item in value)
|
|
14
|
+
if isinstance(value, set):
|
|
15
|
+
return frozenset(_freeze(item) for item in value)
|
|
16
|
+
return value
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True)
|
|
20
|
+
class Evidence:
|
|
21
|
+
code: str
|
|
22
|
+
statement: str
|
|
23
|
+
details: Mapping[str, Any] = field(default_factory=dict)
|
|
24
|
+
|
|
25
|
+
def __post_init__(self) -> None:
|
|
26
|
+
object.__setattr__(self, "details", _freeze(self.details))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class SemanticContext:
|
|
31
|
+
role: str
|
|
32
|
+
confidence: float
|
|
33
|
+
evidence: tuple[Evidence, ...] = ()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class Recommendation:
|
|
38
|
+
operation: str
|
|
39
|
+
disposition: str
|
|
40
|
+
rationale: str
|
|
41
|
+
evidence: tuple[Evidence, ...] = ()
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True)
|
|
45
|
+
class Warning:
|
|
46
|
+
code: str
|
|
47
|
+
severity: str
|
|
48
|
+
title: str
|
|
49
|
+
message: str
|
|
50
|
+
evidence: tuple[Evidence, ...] = ()
|
|
51
|
+
implication: str = ""
|
|
52
|
+
recommendation: str | None = None
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass(frozen=True)
|
|
56
|
+
class KeyCandidate:
|
|
57
|
+
columns: tuple[str, ...]
|
|
58
|
+
kind: str
|
|
59
|
+
confidence: float
|
|
60
|
+
uniqueness_rate: float
|
|
61
|
+
evidence: tuple[Evidence, ...] = ()
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass(frozen=True)
|
|
65
|
+
class GrainContext:
|
|
66
|
+
status: str
|
|
67
|
+
label: str | None
|
|
68
|
+
columns: tuple[str, ...]
|
|
69
|
+
confidence: float
|
|
70
|
+
evidence: tuple[Evidence, ...] = ()
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass(frozen=True)
|
|
74
|
+
class RelationshipContext:
|
|
75
|
+
source_column: str
|
|
76
|
+
target_column: str
|
|
77
|
+
kind: str
|
|
78
|
+
coverage: float
|
|
79
|
+
confidence: float
|
|
80
|
+
evidence: tuple[Evidence, ...] = ()
|
oribi/models/profile.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Observed dataset and column statistics."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from collections.abc import Mapping
|
|
5
|
+
from types import MappingProxyType
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass(frozen=True)
|
|
10
|
+
class StatisticsContext:
|
|
11
|
+
missing_count: int
|
|
12
|
+
missing_rate: float
|
|
13
|
+
distinct_count: int
|
|
14
|
+
distinct_rate: float
|
|
15
|
+
minimum: Any = None
|
|
16
|
+
maximum: Any = None
|
|
17
|
+
mean: float | None = None
|
|
18
|
+
median: float | None = None
|
|
19
|
+
standard_deviation: float | None = None
|
|
20
|
+
quantiles: Mapping[str, float] = field(default_factory=dict)
|
|
21
|
+
zero_count: int | None = None
|
|
22
|
+
negative_count: int | None = None
|
|
23
|
+
skewness: float | None = None
|
|
24
|
+
top_values: tuple[tuple[Any, int], ...] = ()
|
|
25
|
+
date_range: str | None = None
|
|
26
|
+
|
|
27
|
+
def __post_init__(self) -> None:
|
|
28
|
+
object.__setattr__(self, "quantiles", MappingProxyType(dict(self.quantiles)))
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class DatasetProfile:
|
|
33
|
+
row_count: int
|
|
34
|
+
column_count: int
|
|
35
|
+
duplicate_row_count: int | None
|
|
36
|
+
duplicate_row_rate: float | None
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Internal profiling components."""
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""Statistics for observed columns."""
|
|
2
|
+
|
|
3
|
+
import math
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
from ..models.profile import StatisticsContext
|
|
9
|
+
from .observations import ColumnObservation
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def profile_column(obs: ColumnObservation, row_count: int, top_k: int, include_quantiles: bool) -> StatisticsContext:
|
|
13
|
+
s = obs.series
|
|
14
|
+
denom = row_count or 1
|
|
15
|
+
non_null = s.dropna()
|
|
16
|
+
values: dict[str, Any] = {}
|
|
17
|
+
if obs.logical_type == "numeric" and len(non_null):
|
|
18
|
+
numeric = pd.to_numeric(non_null, errors="coerce").dropna()
|
|
19
|
+
if len(numeric):
|
|
20
|
+
def finite_or_none(value: Any) -> float | None:
|
|
21
|
+
result = float(value)
|
|
22
|
+
return result if math.isfinite(result) else None
|
|
23
|
+
|
|
24
|
+
values.update(minimum=numeric.min(), maximum=numeric.max(), mean=finite_or_none(numeric.mean()), median=finite_or_none(numeric.median()), standard_deviation=finite_or_none(numeric.std()))
|
|
25
|
+
values["zero_count"] = int((numeric == 0).sum())
|
|
26
|
+
values["negative_count"] = int((numeric < 0).sum())
|
|
27
|
+
if include_quantiles:
|
|
28
|
+
qs = numeric.quantile([.25, .5, .75])
|
|
29
|
+
values["quantiles"] = {str(q): float(v) for q, v in qs.items()}
|
|
30
|
+
if len(numeric) >= 3 and numeric.nunique() > 1:
|
|
31
|
+
skew = float(numeric.skew())
|
|
32
|
+
values["skewness"] = skew if pd.notna(skew) else None
|
|
33
|
+
elif obs.logical_type == "temporal" and len(non_null):
|
|
34
|
+
values["minimum"] = non_null.min()
|
|
35
|
+
values["maximum"] = non_null.max()
|
|
36
|
+
values["date_range"] = f"{values['minimum']} to {values['maximum']}"
|
|
37
|
+
if len(non_null) and (obs.logical_type in {"text", "categorical", "boolean", "unknown"} or obs.distinct_count <= top_k * 4):
|
|
38
|
+
try:
|
|
39
|
+
counts = non_null.value_counts(dropna=True).head(top_k)
|
|
40
|
+
values["top_values"] = tuple((k, int(v)) for k, v in counts.items())
|
|
41
|
+
except (TypeError, ValueError):
|
|
42
|
+
pass
|
|
43
|
+
return StatisticsContext(
|
|
44
|
+
missing_count=obs.missing_count,
|
|
45
|
+
missing_rate=obs.missing_count / denom,
|
|
46
|
+
distinct_count=obs.distinct_count,
|
|
47
|
+
distinct_rate=obs.distinct_count / denom,
|
|
48
|
+
**values,
|
|
49
|
+
)
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Per-call memoized observations shared across detectors."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from typing import TYPE_CHECKING
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
import pandas as pd
|
|
10
|
+
|
|
11
|
+
from ..exceptions import InvalidInputError
|
|
12
|
+
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
from ..api import ProfileOptions
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class ColumnObservation:
|
|
19
|
+
name: str
|
|
20
|
+
series: pd.Series
|
|
21
|
+
missing_count: int
|
|
22
|
+
distinct_count: int
|
|
23
|
+
logical_type: str
|
|
24
|
+
stats: dict[str, Any] = field(default_factory=dict)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class AnalysisSession:
|
|
28
|
+
def __init__(self, frame: pd.DataFrame, options: ProfileOptions):
|
|
29
|
+
if not isinstance(frame, pd.DataFrame):
|
|
30
|
+
raise InvalidInputError("inspect() expects a pandas DataFrame")
|
|
31
|
+
if not frame.columns.is_unique:
|
|
32
|
+
raise InvalidInputError("DataFrame column labels must be unique")
|
|
33
|
+
if any(not isinstance(c, str) for c in frame.columns):
|
|
34
|
+
raise InvalidInputError("DataFrame column labels must be strings")
|
|
35
|
+
self.frame = frame
|
|
36
|
+
self.options = options
|
|
37
|
+
self.observations: dict[Any, ColumnObservation] = {}
|
|
38
|
+
|
|
39
|
+
def observe(self, name: Any) -> ColumnObservation:
|
|
40
|
+
if name in self.observations:
|
|
41
|
+
return self.observations[name]
|
|
42
|
+
series = self.frame[name]
|
|
43
|
+
missing = int(series.isna().sum())
|
|
44
|
+
try:
|
|
45
|
+
distinct = int(series.nunique(dropna=True))
|
|
46
|
+
except (TypeError, ValueError):
|
|
47
|
+
distinct = int(series.dropna().astype(str).nunique())
|
|
48
|
+
if pd.api.types.is_bool_dtype(series.dtype):
|
|
49
|
+
logical = "boolean"
|
|
50
|
+
elif pd.api.types.is_datetime64_any_dtype(series.dtype):
|
|
51
|
+
logical = "temporal"
|
|
52
|
+
elif pd.api.types.is_numeric_dtype(series.dtype):
|
|
53
|
+
logical = "numeric"
|
|
54
|
+
elif isinstance(series.dtype, pd.CategoricalDtype):
|
|
55
|
+
logical = "categorical"
|
|
56
|
+
elif series.dtype == object or pd.api.types.is_string_dtype(series.dtype):
|
|
57
|
+
values = series.dropna()
|
|
58
|
+
kinds = {type(v) for v in values.iloc[:100]}
|
|
59
|
+
logical = "unknown" if len(kinds) > 1 and len({"str" if issubclass(t, str) else t.__name__ for t in kinds}) > 1 else "text"
|
|
60
|
+
if values.empty:
|
|
61
|
+
logical = "unknown"
|
|
62
|
+
else:
|
|
63
|
+
logical = "unknown"
|
|
64
|
+
obs = ColumnObservation(str(name), series, missing, distinct, logical)
|
|
65
|
+
self.observations[name] = obs
|
|
66
|
+
return obs
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Conservative within-frame column dependency detection."""
|
|
2
|
+
|
|
3
|
+
import pandas as pd
|
|
4
|
+
|
|
5
|
+
from ..models.evidence import Evidence, RelationshipContext
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def detect_relationships(frame: pd.DataFrame, roles: dict[str, str], max_rows: int = 100_000) -> tuple[RelationshipContext, ...]:
|
|
9
|
+
"""Find near-complete one-to-one value mappings among low-cardinality columns."""
|
|
10
|
+
if len(frame) == 0 or len(frame) > max_rows:
|
|
11
|
+
return ()
|
|
12
|
+
eligible = [name for name, role in roles.items() if role in {"identifier", "dimension", "categorical"}]
|
|
13
|
+
eligible = [name for name in eligible if frame[name].nunique(dropna=True) <= min(500, max(20, len(frame) // 2))]
|
|
14
|
+
results: list[RelationshipContext] = []
|
|
15
|
+
for source in eligible:
|
|
16
|
+
for target in eligible:
|
|
17
|
+
if source == target:
|
|
18
|
+
continue
|
|
19
|
+
pairs = frame[[source, target]].dropna().drop_duplicates()
|
|
20
|
+
source_values = frame[source].dropna().nunique()
|
|
21
|
+
if source_values < 2:
|
|
22
|
+
continue
|
|
23
|
+
unique_mapping = int(pairs.groupby(source, dropna=True)[target].nunique().le(1).sum())
|
|
24
|
+
coverage = unique_mapping / source_values
|
|
25
|
+
target_values = frame[target].dropna().nunique()
|
|
26
|
+
if coverage >= .98 and target_values > 1 and source_values <= target_values:
|
|
27
|
+
results.append(RelationshipContext(
|
|
28
|
+
source, target, "functional_dependency", coverage,
|
|
29
|
+
round(min(.95, .65 + coverage * .3), 2),
|
|
30
|
+
(Evidence("value_mapping", f"{coverage:.1%} of observed {source} values map to one {target} value", {"source_distinct": int(source_values), "target_distinct": int(target_values), "coverage": coverage}),),
|
|
31
|
+
))
|
|
32
|
+
# Keep only one orientation per pair to avoid duplicate, reversed claims.
|
|
33
|
+
deduped: dict[frozenset[str], RelationshipContext] = {}
|
|
34
|
+
for relation in results:
|
|
35
|
+
key = frozenset((relation.source_column, relation.target_column))
|
|
36
|
+
current = deduped.get(key)
|
|
37
|
+
if current is None or relation.source_column < current.source_column:
|
|
38
|
+
deduped[key] = relation
|
|
39
|
+
return tuple(sorted(deduped.values(), key=lambda r: (r.source_column, r.target_column)))
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""Conservative temporal cadence and gap observations."""
|
|
2
|
+
|
|
3
|
+
from collections import Counter
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
from ..models.evidence import Evidence, Warning
|
|
8
|
+
|
|
9
|
+
_MIN_UNIQUE_TIMESTAMPS = 4
|
|
10
|
+
_MIN_CADENCE_INTERVALS = 2
|
|
11
|
+
_MIN_CADENCE_SHARE = 0.6
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def temporal_gap_warnings(series: pd.Series, column_name: str) -> tuple[Warning, ...]:
|
|
15
|
+
"""Report possible gaps only when a dominant fixed cadence is supported."""
|
|
16
|
+
timestamps = pd.DatetimeIndex(series.dropna().drop_duplicates().sort_values())
|
|
17
|
+
if len(timestamps) < _MIN_UNIQUE_TIMESTAMPS:
|
|
18
|
+
return ()
|
|
19
|
+
|
|
20
|
+
deltas = [timestamps[i] - timestamps[i - 1] for i in range(1, len(timestamps))]
|
|
21
|
+
ranked = Counter(deltas).most_common()
|
|
22
|
+
cadence, dominant_count = ranked[0]
|
|
23
|
+
if len(ranked) > 1 and ranked[1][1] == dominant_count:
|
|
24
|
+
return ()
|
|
25
|
+
if dominant_count < _MIN_CADENCE_INTERVALS or dominant_count / len(deltas) < _MIN_CADENCE_SHARE:
|
|
26
|
+
return ()
|
|
27
|
+
|
|
28
|
+
steps_by_interval: list[int] = []
|
|
29
|
+
for delta in deltas:
|
|
30
|
+
steps = delta / cadence
|
|
31
|
+
if steps < 1 or steps != int(steps):
|
|
32
|
+
return ()
|
|
33
|
+
steps_by_interval.append(int(steps))
|
|
34
|
+
if not any(steps > 1 for steps in steps_by_interval):
|
|
35
|
+
return ()
|
|
36
|
+
|
|
37
|
+
cadence_text = _format_timedelta(cadence)
|
|
38
|
+
warnings: list[Warning] = []
|
|
39
|
+
for index, steps in enumerate(steps_by_interval):
|
|
40
|
+
missing_count = steps - 1
|
|
41
|
+
if not missing_count:
|
|
42
|
+
continue
|
|
43
|
+
before, after = timestamps[index], timestamps[index + 1]
|
|
44
|
+
expected = [before + cadence * step for step in range(1, steps)]
|
|
45
|
+
evidence = Evidence(
|
|
46
|
+
"temporal_gap",
|
|
47
|
+
f"{missing_count} timestamp(s) may be absent relative to an inferred {cadence_text} cadence",
|
|
48
|
+
{
|
|
49
|
+
"cadence": cadence_text,
|
|
50
|
+
"gap_start": expected[0].isoformat(),
|
|
51
|
+
"gap_end": expected[-1].isoformat(),
|
|
52
|
+
"missing_intervals": missing_count,
|
|
53
|
+
"expected_timestamps": [value.isoformat() for value in expected],
|
|
54
|
+
"observed_before": before.isoformat(),
|
|
55
|
+
"observed_after": after.isoformat(),
|
|
56
|
+
},
|
|
57
|
+
)
|
|
58
|
+
warnings.append(Warning(
|
|
59
|
+
"potential_temporal_gap", "low", "Potential temporal gap",
|
|
60
|
+
f"{column_name} has a possible temporal gap relative to an inferred {cadence_text} cadence.",
|
|
61
|
+
(evidence,),
|
|
62
|
+
"The interval may reflect missing observations or a normal pause in data collection.",
|
|
63
|
+
"Confirm the expected sampling schedule before treating the interval as missing data.",
|
|
64
|
+
))
|
|
65
|
+
return tuple(warnings)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _format_timedelta(value: pd.Timedelta) -> str:
|
|
69
|
+
"""Format common fixed cadence durations consistently."""
|
|
70
|
+
seconds = value.total_seconds()
|
|
71
|
+
for size, unit in ((86400, "day"), (3600, "hour"), (60, "minute"), (1, "second")):
|
|
72
|
+
if seconds % size == 0:
|
|
73
|
+
amount = int(seconds / size)
|
|
74
|
+
return f"{amount} {unit}{'' if amount == 1 else 's'}"
|
|
75
|
+
return str(value)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Deterministic analytical reasoning."""
|
oribi/reasoning/keys.py
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"""Candidate-key collection and separate grain selection."""
|
|
2
|
+
|
|
3
|
+
from itertools import combinations
|
|
4
|
+
import re
|
|
5
|
+
|
|
6
|
+
import pandas as pd
|
|
7
|
+
|
|
8
|
+
from ..models.evidence import Evidence, GrainContext, KeyCandidate
|
|
9
|
+
|
|
10
|
+
_MIN_SINGLE_UNIQUENESS = 0.98
|
|
11
|
+
_MAX_SINGLE_MISSING_RATE = 0.02
|
|
12
|
+
_MIN_COMPOSITE_UNIQUENESS = 0.95
|
|
13
|
+
_COMPOSITE_ROLES = {"identifier", "dimension", "categorical", "boolean"}
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def detect_candidate_keys(
|
|
17
|
+
frame: pd.DataFrame,
|
|
18
|
+
names: list[str],
|
|
19
|
+
stats: dict[str, object],
|
|
20
|
+
roles: dict[str, str],
|
|
21
|
+
max_columns: int = 8,
|
|
22
|
+
max_exact_rows: int = 100_000,
|
|
23
|
+
) -> tuple[KeyCandidate, ...]:
|
|
24
|
+
"""Collect qualifying single and bounded pair candidates.
|
|
25
|
+
|
|
26
|
+
Pair search considers at most ``max_columns`` non-unique identifier or
|
|
27
|
+
dimension-like columns (28 pairs at the default limit of eight), and only
|
|
28
|
+
runs below ``max_exact_rows``. This bounds exact combination scans while
|
|
29
|
+
allowing all qualifying candidates in the searched set to be retained.
|
|
30
|
+
"""
|
|
31
|
+
row_count = len(frame)
|
|
32
|
+
if row_count == 0:
|
|
33
|
+
return ()
|
|
34
|
+
sample_support = min(1.0, (row_count / 100) ** 0.5)
|
|
35
|
+
|
|
36
|
+
single_candidates: list[KeyCandidate] = []
|
|
37
|
+
for name in sorted(names):
|
|
38
|
+
column_stats = stats[name]
|
|
39
|
+
rate = column_stats.distinct_count / row_count
|
|
40
|
+
if (
|
|
41
|
+
roles.get(name) == "identifier"
|
|
42
|
+
and rate >= _MIN_SINGLE_UNIQUENESS
|
|
43
|
+
and column_stats.missing_rate <= _MAX_SINGLE_MISSING_RATE
|
|
44
|
+
):
|
|
45
|
+
evidence = Evidence(
|
|
46
|
+
"single_column_uniqueness",
|
|
47
|
+
f"{column_stats.distinct_count} distinct values across {row_count} rows",
|
|
48
|
+
{"distinct_count": column_stats.distinct_count, "row_count": row_count},
|
|
49
|
+
)
|
|
50
|
+
single_candidates.append(KeyCandidate(
|
|
51
|
+
(name,),
|
|
52
|
+
"single",
|
|
53
|
+
round(min(.95, .5 + (.6 + .4 * rate - .5) * sample_support), 2),
|
|
54
|
+
rate,
|
|
55
|
+
(evidence,),
|
|
56
|
+
))
|
|
57
|
+
|
|
58
|
+
# A component already qualifying alone adds no useful composite candidate.
|
|
59
|
+
single_columns = {candidate.columns[0] for candidate in single_candidates}
|
|
60
|
+
potential = sorted(
|
|
61
|
+
(
|
|
62
|
+
name for name in names
|
|
63
|
+
if name not in single_columns
|
|
64
|
+
and _is_key_component(name, roles.get(name))
|
|
65
|
+
and stats[name].distinct_count > 1
|
|
66
|
+
),
|
|
67
|
+
key=lambda name: (-stats[name].distinct_rate, stats[name].missing_rate, name),
|
|
68
|
+
)[:max_columns]
|
|
69
|
+
|
|
70
|
+
pair_candidates: list[KeyCandidate] = []
|
|
71
|
+
if row_count <= max_exact_rows:
|
|
72
|
+
for pair in combinations(potential, 2):
|
|
73
|
+
columns = tuple(sorted(pair))
|
|
74
|
+
try:
|
|
75
|
+
distinct_pairs = int(frame[list(columns)].drop_duplicates().shape[0])
|
|
76
|
+
except (TypeError, ValueError):
|
|
77
|
+
continue
|
|
78
|
+
rate = distinct_pairs / row_count
|
|
79
|
+
if rate >= _MIN_COMPOSITE_UNIQUENESS:
|
|
80
|
+
base_confidence = min(.9, .55 + (rate - _MIN_COMPOSITE_UNIQUENESS) * 5)
|
|
81
|
+
confidence = .5 + (base_confidence - .5) * sample_support
|
|
82
|
+
evidence = Evidence(
|
|
83
|
+
"composite_uniqueness",
|
|
84
|
+
f"{distinct_pairs} distinct pairs across {row_count} rows",
|
|
85
|
+
{
|
|
86
|
+
"distinct_pairs": distinct_pairs,
|
|
87
|
+
"row_count": row_count,
|
|
88
|
+
"uniqueness_rate": rate,
|
|
89
|
+
},
|
|
90
|
+
)
|
|
91
|
+
pair_candidates.append(KeyCandidate(
|
|
92
|
+
columns, "composite", round(confidence, 2), rate, (evidence,),
|
|
93
|
+
))
|
|
94
|
+
|
|
95
|
+
# Candidate retention does not depend on this ordering. Simpler keys are
|
|
96
|
+
# listed first, with deterministic uniqueness/confidence/name tie-breakers.
|
|
97
|
+
return tuple(sorted(
|
|
98
|
+
(*single_candidates, *pair_candidates),
|
|
99
|
+
key=lambda candidate: (
|
|
100
|
+
len(candidate.columns),
|
|
101
|
+
-candidate.uniqueness_rate,
|
|
102
|
+
-candidate.confidence,
|
|
103
|
+
candidate.columns,
|
|
104
|
+
),
|
|
105
|
+
))
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _is_key_component(name: str, role: str | None) -> bool:
|
|
109
|
+
"""Allow established key-like name hints when role votes are ambiguous."""
|
|
110
|
+
if role in _COMPOSITE_ROLES:
|
|
111
|
+
return True
|
|
112
|
+
tokens = set(re.split(r"[^a-z0-9]+", name.lower()))
|
|
113
|
+
return "id" in tokens or name.lower().endswith(("_key", "_code"))
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def select_grain(candidates: tuple[KeyCandidate, ...]) -> GrainContext:
|
|
117
|
+
"""Choose one preferred grain from the retained candidates."""
|
|
118
|
+
if not candidates:
|
|
119
|
+
return GrainContext("undetermined", None, (), 0.0, ())
|
|
120
|
+
|
|
121
|
+
# Preserve the existing preference for a single-column identifier when
|
|
122
|
+
# available; candidate detection still retains every composite candidate.
|
|
123
|
+
singles = [candidate for candidate in candidates if candidate.kind == "single"]
|
|
124
|
+
pool = singles or [candidate for candidate in candidates if candidate.kind == "composite"]
|
|
125
|
+
best = min(
|
|
126
|
+
pool,
|
|
127
|
+
key=lambda candidate: (
|
|
128
|
+
-candidate.uniqueness_rate,
|
|
129
|
+
-candidate.confidence,
|
|
130
|
+
candidate.columns,
|
|
131
|
+
),
|
|
132
|
+
)
|
|
133
|
+
label = " × ".join(best.columns)
|
|
134
|
+
return GrainContext("estimated", label, best.columns, best.confidence, best.evidence)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def detect_keys(
|
|
138
|
+
frame: pd.DataFrame,
|
|
139
|
+
names: list[str],
|
|
140
|
+
stats: dict[str, object],
|
|
141
|
+
roles: dict[str, str],
|
|
142
|
+
max_columns: int = 8,
|
|
143
|
+
max_exact_rows: int = 100_000,
|
|
144
|
+
) -> tuple[tuple[KeyCandidate, ...], GrainContext]:
|
|
145
|
+
"""Collect candidate keys, then independently select a grain hypothesis."""
|
|
146
|
+
candidates = detect_candidate_keys(
|
|
147
|
+
frame, names, stats, roles,
|
|
148
|
+
max_columns=max_columns,
|
|
149
|
+
max_exact_rows=max_exact_rows,
|
|
150
|
+
)
|
|
151
|
+
if len(frame) == 0:
|
|
152
|
+
grain = GrainContext(
|
|
153
|
+
"undetermined", None, (), 0.0,
|
|
154
|
+
(Evidence("empty_dataset", "No rows are available to estimate a key or grain"),),
|
|
155
|
+
)
|
|
156
|
+
elif candidates:
|
|
157
|
+
grain = select_grain(candidates)
|
|
158
|
+
else:
|
|
159
|
+
code = "key_search_bounded" if len(frame) > max_exact_rows else "no_strong_key"
|
|
160
|
+
message = (
|
|
161
|
+
"Composite key search was skipped because the DataFrame exceeds max_exact_rows"
|
|
162
|
+
if len(frame) > max_exact_rows
|
|
163
|
+
else "No single column or tested column pair was sufficiently unique"
|
|
164
|
+
)
|
|
165
|
+
grain = GrainContext(
|
|
166
|
+
"undetermined", None, (), 0.0,
|
|
167
|
+
(Evidence(code, message, {"row_count": len(frame), "max_exact_rows": max_exact_rows}),),
|
|
168
|
+
)
|
|
169
|
+
return candidates, grain
|
oribi/reasoning/rules.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Structured warnings and aggregation recommendations."""
|
|
2
|
+
|
|
3
|
+
from ..models.evidence import Evidence, Recommendation, Warning
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def reason_column(name: str, logical_type: str, semantic: object, stats: object) -> tuple[tuple[Recommendation, ...], tuple[Warning, ...]]:
|
|
7
|
+
recommendations: list[Recommendation] = []
|
|
8
|
+
warnings: list[Warning] = []
|
|
9
|
+
if stats.missing_rate >= .2:
|
|
10
|
+
warnings.append(Warning("high_missingness", "medium", "High missingness", f"{name} is missing {stats.missing_rate:.1%} of its values.", (Evidence("missing_rate", f"{stats.missing_count} of values are missing", {"missing_count": stats.missing_count, "missing_rate": stats.missing_rate}),), "Analyses using this column may exclude many rows.", "Check whether missing values have a meaningful interpretation."))
|
|
11
|
+
if semantic.role == "identifier":
|
|
12
|
+
recommendations.append(Recommendation("COUNT DISTINCT", "recommended", "Distinct counting is more appropriate for counting entities than summing identifier values.", semantic.evidence))
|
|
13
|
+
if stats.distinct_rate < .98:
|
|
14
|
+
warnings.append(Warning("non_unique_identifier", "medium", "Identifier-like column is not unique", f"{name} has {stats.distinct_count} distinct values across rows and may identify repeated entities.", semantic.evidence, "Row counts may exceed entity counts.", "Use a distinct count when counting these entities."))
|
|
15
|
+
if semantic.role == "measure":
|
|
16
|
+
for op, disposition in (("SUM", "recommended"), ("MEAN", "recommended"), ("MEDIAN", "useful"), ("MIN", "valid"), ("MAX", "valid")):
|
|
17
|
+
recommendations.append(Recommendation(op, disposition, "The column is numeric and inferred as a likely measure.", semantic.evidence))
|
|
18
|
+
if stats.skewness is not None and abs(stats.skewness) >= 2:
|
|
19
|
+
warnings.append(Warning("high_skew", "low", "Highly skewed measure", f"{name} has skewness {stats.skewness:.2f}.", (Evidence("skewness", "Observed distribution is strongly skewed", {"skewness": stats.skewness}),), "The mean may be sensitive to a small number of large values.", "Consider reporting the median alongside the mean."))
|
|
20
|
+
if stats.negative_count:
|
|
21
|
+
warnings.append(Warning("negative_measure_values", "low", f"Negative values in {name}", f"{stats.negative_count} negative values were observed in likely measure {name}.", (Evidence("negative_count", f"{stats.negative_count} values are below zero", {"negative_count": stats.negative_count}),), "Negative values may represent refunds, reversals, or data issues.", "Check domain meaning before filtering negative values."))
|
|
22
|
+
if logical_type == "categorical" and stats.distinct_count > 100:
|
|
23
|
+
warnings.append(Warning("high_cardinality_category", "low", "High-cardinality category", f"{name} has {stats.distinct_count} distinct categories.", (), "Grouping by this column may produce many small groups."))
|
|
24
|
+
return tuple(recommendations), tuple(warnings)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def dataset_warnings(duplicate_count: int | None, row_count: int) -> tuple[Warning, ...]:
|
|
28
|
+
if duplicate_count:
|
|
29
|
+
return (Warning("duplicate_rows", "medium", "Duplicate rows detected", f"{duplicate_count} fully duplicated rows were observed.", (Evidence("duplicate_count", f"{duplicate_count} duplicate rows across {row_count} total rows", {"duplicate_row_count": duplicate_count, "row_count": row_count}),), "Duplicates can inflate aggregates.", "Confirm whether repeated rows are expected before aggregation."),)
|
|
30
|
+
return ()
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Composable rules for likely column roles."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
|
|
5
|
+
from ..models.evidence import Evidence, SemanticContext
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
_MEASURE_NAME_HINTS = ("amount", "price", "revenue", "sales", "quantity", "count", "total", "cost", "rate", "profit", "margin")
|
|
9
|
+
_MIN_ROLE_SCORE = 0.55
|
|
10
|
+
_AMBIGUITY_MARGIN = 0.12
|
|
11
|
+
|
|
12
|
+
def infer_semantic(name: str, logical_type: str, stats: object, row_count: int) -> SemanticContext:
|
|
13
|
+
votes: dict[str, float] = {}
|
|
14
|
+
evidence: list[Evidence] = []
|
|
15
|
+
lname = name.lower()
|
|
16
|
+
distinct = stats.distinct_count
|
|
17
|
+
rate = stats.distinct_rate
|
|
18
|
+
tokens = set(re.split(r"[^a-z0-9]+", lname))
|
|
19
|
+
explicit_identifier_name = "id" in tokens or lname.endswith("_key")
|
|
20
|
+
code_name = lname.endswith("_code")
|
|
21
|
+
if explicit_identifier_name:
|
|
22
|
+
votes["identifier"] = .88
|
|
23
|
+
evidence.append(Evidence("name_identifier_hint", "Column name strongly suggests an identifier"))
|
|
24
|
+
elif code_name:
|
|
25
|
+
votes["identifier"] = .78
|
|
26
|
+
evidence.append(Evidence("name_identifier_hint", "Column name suggests an identifier"))
|
|
27
|
+
if row_count and rate >= .98 and distinct > 1:
|
|
28
|
+
# Uniqueness is supporting evidence, not proof of business identity.
|
|
29
|
+
# It is deliberately weaker for numeric columns, which commonly hold
|
|
30
|
+
# continuous measures that happen to be unique in small samples.
|
|
31
|
+
unique_vote = .28 if logical_type == "numeric" else .62
|
|
32
|
+
votes["identifier"] = max(votes.get("identifier", 0), unique_vote)
|
|
33
|
+
evidence.append(Evidence("mostly_unique", f"{distinct} distinct non-null values across {row_count} rows", {"distinct_count": distinct, "row_count": row_count, "distinct_rate": rate}))
|
|
34
|
+
if logical_type == "numeric":
|
|
35
|
+
votes["measure"] = .48
|
|
36
|
+
evidence.append(Evidence("numeric_type", "Column has a numeric dtype"))
|
|
37
|
+
if any(w in tokens for w in _MEASURE_NAME_HINTS):
|
|
38
|
+
votes["measure"] += .28
|
|
39
|
+
evidence.append(Evidence("name_measure_hint", "Column name suggests a quantitative measure"))
|
|
40
|
+
elif logical_type == "temporal":
|
|
41
|
+
votes["temporal"] = .86
|
|
42
|
+
evidence.append(Evidence("datetime_dtype", "Column has a datetime dtype"))
|
|
43
|
+
elif logical_type == "boolean":
|
|
44
|
+
votes["boolean"] = .9
|
|
45
|
+
evidence.append(Evidence("boolean_dtype", "Column has a boolean dtype"))
|
|
46
|
+
elif logical_type == "categorical":
|
|
47
|
+
votes["categorical"] = .78
|
|
48
|
+
evidence.append(Evidence("categorical_dtype", "Column has a categorical dtype"))
|
|
49
|
+
elif logical_type == "text":
|
|
50
|
+
if rate > .5 and distinct > 20:
|
|
51
|
+
votes["text"] = .62
|
|
52
|
+
evidence.append(Evidence("high_cardinality_text", "Text values have relatively high cardinality"))
|
|
53
|
+
else:
|
|
54
|
+
votes["dimension"] = .62
|
|
55
|
+
evidence.append(Evidence("text_dimension", "Text values are reusable across rows"))
|
|
56
|
+
if logical_type == "numeric" and distinct <= min(20, max(3, row_count // 10)):
|
|
57
|
+
votes["dimension"] = max(votes.get("dimension", 0), .64)
|
|
58
|
+
evidence.append(Evidence("low_cardinality_numeric", "Numeric values have low cardinality"))
|
|
59
|
+
if not votes:
|
|
60
|
+
return SemanticContext("unknown", 0.0, tuple(evidence))
|
|
61
|
+
role, score = max(votes.items(), key=lambda item: (item[1], item[0]))
|
|
62
|
+
ranked_votes = sorted(votes.items(), key=lambda item: (-item[1], item[0]))
|
|
63
|
+
if score < _MIN_ROLE_SCORE:
|
|
64
|
+
return SemanticContext("unknown", round(score, 2), tuple(evidence))
|
|
65
|
+
if len(ranked_votes) > 1 and score - ranked_votes[1][1] < _AMBIGUITY_MARGIN:
|
|
66
|
+
return SemanticContext("unknown", round(min(score, .5), 2), tuple(evidence))
|
|
67
|
+
return SemanticContext(role, round(min(score, .98), 2), tuple(evidence))
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Pure renderers for DatasetContext."""
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""Compact AI-oriented rendering of an existing context snapshot."""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def to_context(context: Any, *, max_chars: int = 8000) -> str:
|
|
7
|
+
if max_chars < 100:
|
|
8
|
+
raise ValueError("max_chars must be at least 100")
|
|
9
|
+
p = context.profile
|
|
10
|
+
header = [f"DATASET (observed): {p.row_count} rows, {p.column_count} columns."]
|
|
11
|
+
if p.duplicate_row_count:
|
|
12
|
+
header.append(f"Duplicate rows (observed): {p.duplicate_row_count}.")
|
|
13
|
+
grain_lines = []
|
|
14
|
+
if context.grain:
|
|
15
|
+
if context.grain.status == "estimated":
|
|
16
|
+
grain_lines.append(f"Likely grain (inference, confidence {context.grain.confidence:.2f}): {context.grain.label}.")
|
|
17
|
+
grain_lines.extend(f" Evidence: {e.statement}." for e in context.grain.evidence)
|
|
18
|
+
else:
|
|
19
|
+
grain_lines.append("Likely grain: undetermined from available evidence.")
|
|
20
|
+
warning_lines = ["WARNINGS"] if context.warnings else []
|
|
21
|
+
for warning in context.warnings:
|
|
22
|
+
warning_lines.append(f"- [{warning.severity}] {warning.message} {warning.implication}".strip())
|
|
23
|
+
column_lines = ["COLUMNS"]
|
|
24
|
+
ranked = sorted(context.columns, key=lambda c: (c.semantic.role not in {"identifier", "measure", "temporal"}, -len(c.warnings), c.name))
|
|
25
|
+
for col in ranked:
|
|
26
|
+
s = col.statistics
|
|
27
|
+
column_lines.append(f"- {col.name} [{col.dtype}]: likely {col.semantic.role} ({col.semantic.confidence:.2f}); observed {s.distinct_count} distinct, {s.missing_rate:.1%} missing.")
|
|
28
|
+
for e in col.semantic.evidence[:2]:
|
|
29
|
+
column_lines.append(f" Evidence: {e.statement}.")
|
|
30
|
+
if s.minimum is not None or s.maximum is not None:
|
|
31
|
+
column_lines.append(f" Observed range: {s.minimum} to {s.maximum}.")
|
|
32
|
+
for rec in col.recommendations:
|
|
33
|
+
if rec.disposition in {"recommended", "caution"}:
|
|
34
|
+
column_lines.append(f" Suggested {rec.operation}: {rec.rationale}.")
|
|
35
|
+
if context.relationships:
|
|
36
|
+
column_lines.append("OBSERVED RELATIONSHIPS")
|
|
37
|
+
for rel in context.relationships:
|
|
38
|
+
column_lines.append(f"- {rel.source_column} -> {rel.target_column}: {rel.kind}, coverage {rel.coverage:.1%}.")
|
|
39
|
+
lines = header + grain_lines + warning_lines + column_lines
|
|
40
|
+
result = "\n".join(lines)
|
|
41
|
+
if len(result) <= max_chars:
|
|
42
|
+
return result
|
|
43
|
+
marker = "[Context truncated.]"
|
|
44
|
+
selected: list[str] = []
|
|
45
|
+
for line in header + grain_lines + warning_lines + column_lines:
|
|
46
|
+
candidate = "\n".join((*selected, line, marker))
|
|
47
|
+
if len(candidate) <= max_chars:
|
|
48
|
+
selected.append(line)
|
|
49
|
+
if not selected:
|
|
50
|
+
return marker[:max_chars]
|
|
51
|
+
return "\n".join((*selected, marker))
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""JSON-compatible projection of the canonical model."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import math
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
from dataclasses import fields, is_dataclass
|
|
7
|
+
from datetime import date, datetime
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _normalize(value: Any) -> Any:
|
|
12
|
+
if is_dataclass(value):
|
|
13
|
+
return {field.name: _normalize(getattr(value, field.name)) for field in fields(value)}
|
|
14
|
+
if isinstance(value, Mapping):
|
|
15
|
+
return {str(k): _normalize(v) for k, v in value.items()}
|
|
16
|
+
if isinstance(value, (tuple, list)):
|
|
17
|
+
return [_normalize(v) for v in value]
|
|
18
|
+
if isinstance(value, (datetime, date)):
|
|
19
|
+
return value.isoformat()
|
|
20
|
+
if hasattr(value, "item"):
|
|
21
|
+
try:
|
|
22
|
+
return _normalize(value.item())
|
|
23
|
+
except (ValueError, TypeError):
|
|
24
|
+
pass
|
|
25
|
+
if isinstance(value, float) and not math.isfinite(value):
|
|
26
|
+
return None
|
|
27
|
+
if value is None or isinstance(value, (str, int, float, bool)):
|
|
28
|
+
return value
|
|
29
|
+
return str(value)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def to_dict(context: Any) -> dict[str, Any]:
|
|
33
|
+
"""Return a stable, JSON-compatible representation."""
|
|
34
|
+
return _normalize(context)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def to_json(context: Any, *, indent: int | None = 2) -> str:
|
|
38
|
+
"""Serialize a context using the standard JSON encoder."""
|
|
39
|
+
return json.dumps(to_dict(context), indent=indent, sort_keys=True, ensure_ascii=False)
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Human-readable context report."""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def to_markdown(context: Any) -> str:
|
|
7
|
+
profile = context.profile
|
|
8
|
+
lines = ["# Dataset context", "", f"- Rows: {profile.row_count:,}", f"- Columns: {profile.column_count:,}"]
|
|
9
|
+
if profile.duplicate_row_count is not None:
|
|
10
|
+
lines.append(f"- Duplicate rows: {profile.duplicate_row_count:,}")
|
|
11
|
+
lines += ["", "## Likely grain", ""]
|
|
12
|
+
if context.grain and context.grain.status == "estimated":
|
|
13
|
+
lines.append(f"{context.grain.label} (confidence {context.grain.confidence:.2f})")
|
|
14
|
+
lines.extend(f"- {e.statement}" for e in context.grain.evidence)
|
|
15
|
+
else:
|
|
16
|
+
lines.append("Undetermined")
|
|
17
|
+
lines += ["", "## Columns", ""]
|
|
18
|
+
for col in context.columns:
|
|
19
|
+
s = col.statistics
|
|
20
|
+
lines += [f"### {col.name}", f"- Type: {col.dtype} ({col.logical_type})", f"- Likely role: {col.semantic.role} ({col.semantic.confidence:.2f})", f"- Missing: {s.missing_count:,} ({s.missing_rate:.1%})", f"- Distinct: {s.distinct_count:,} ({s.distinct_rate:.1%})"]
|
|
21
|
+
if s.minimum is not None or s.maximum is not None:
|
|
22
|
+
lines.append(f"- Range: {s.minimum} to {s.maximum}")
|
|
23
|
+
for rec in col.recommendations:
|
|
24
|
+
lines.append(f"- {rec.disposition.title()}: {rec.operation} — {rec.rationale}")
|
|
25
|
+
for warning in col.warnings:
|
|
26
|
+
lines.append(f"- Warning [{warning.severity}]: {warning.message}")
|
|
27
|
+
lines.append("")
|
|
28
|
+
if context.warnings:
|
|
29
|
+
lines += ["## Analytical warnings", ""]
|
|
30
|
+
for warning in context.warnings:
|
|
31
|
+
lines.append(f"- **{warning.title}** ({warning.severity}): {warning.message}")
|
|
32
|
+
if warning.implication:
|
|
33
|
+
lines.append(f" - Implication: {warning.implication}")
|
|
34
|
+
if warning.recommendation:
|
|
35
|
+
lines.append(f" - Suggested: {warning.recommendation}")
|
|
36
|
+
return "\n".join(lines).strip()
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: oribi
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Evidence-backed analytical context for pandas DataFrames
|
|
5
|
+
Author: Oribi contributors
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/samhoon000/Oribi
|
|
8
|
+
Project-URL: Repository, https://github.com/samhoon000/Oribi
|
|
9
|
+
Project-URL: Issues, https://github.com/samhoon000/Oribi/issues
|
|
10
|
+
Keywords: pandas,data-profiling,data-analysis,data-quality,analytics
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
21
|
+
Requires-Python: <3.15,>=3.10
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Requires-Dist: pandas<4,>=1.5.3
|
|
25
|
+
Provides-Extra: dev
|
|
26
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
27
|
+
Requires-Dist: pytest-cov>=6; extra == "dev"
|
|
28
|
+
Requires-Dist: build>=1.2; extra == "dev"
|
|
29
|
+
Dynamic: license-file
|
|
30
|
+
|
|
31
|
+
# Oribi
|
|
32
|
+
|
|
33
|
+
Oribi profiles a pandas DataFrame and returns a structured, evidence-backed snapshot of its shape, column statistics, likely semantic roles, candidate keys, possible grain, relationships, and analytical warnings. Use it to inspect data before writing an analysis or to give downstream tools a compact description of a dataset.
|
|
34
|
+
|
|
35
|
+
## Install
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
python -m pip install oribi
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Oribi supports Python 3.10–3.14 and pandas 1.5.3 or newer (below pandas 4).
|
|
42
|
+
|
|
43
|
+
## Quick start
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
import pandas as pd
|
|
47
|
+
from oribi import inspect
|
|
48
|
+
|
|
49
|
+
df = pd.DataFrame({
|
|
50
|
+
"order_id": [101, 101, 102, 102],
|
|
51
|
+
"product_id": [7, 8, 7, 8],
|
|
52
|
+
"revenue": [25.0, 40.0, 30.0, -5.0],
|
|
53
|
+
})
|
|
54
|
+
|
|
55
|
+
context = inspect(df)
|
|
56
|
+
print(context.summary())
|
|
57
|
+
print(context.to_context(max_chars=2_000))
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
`inspect()` does not modify the DataFrame or keep a reference to it in the result. The returned snapshot can also be rendered as a full Markdown report or serialized to JSON-compatible data:
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
markdown = context.to_markdown()
|
|
64
|
+
payload = context.to_dict()
|
|
65
|
+
json_text = context.to_json()
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## What Oribi currently reports
|
|
69
|
+
|
|
70
|
+
- Dataset row/column counts and duplicate rows.
|
|
71
|
+
- Per-column missingness, distinct counts, numeric summaries, categorical top values, and datetime bounds.
|
|
72
|
+
- Likely roles such as identifier, measure, dimension, categorical, temporal, boolean, text, or unknown, with evidence.
|
|
73
|
+
- Candidate single-column and bounded two-column keys, plus a separate likely-grain estimate.
|
|
74
|
+
- Conservative within-DataFrame functional-dependency relationships.
|
|
75
|
+
- Recommendations for likely numeric measures and identifier counts.
|
|
76
|
+
- Warnings for duplicate rows, missingness, non-unique identifiers, negative or skewed measures, high-cardinality categories, and possible gaps in regular datetime series.
|
|
77
|
+
|
|
78
|
+
Inferences are heuristics, not guarantees about business meaning. Temporal-gap warnings require a sufficiently long series with a dominant fixed cadence; irregular event timestamps and short series are left without a gap claim. Relationships describe observed value mappings within one frame and do not prove that a join is valid or safe.
|
|
79
|
+
|
|
80
|
+
## Public API
|
|
81
|
+
|
|
82
|
+
The main operation is `oribi.inspect(df, *, options=None)`. It returns a `DatasetContext` snapshot. `ProfileOptions` can bound top values and exact key/quantile analysis:
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
import pandas as pd
|
|
86
|
+
from oribi import ProfileOptions, inspect
|
|
87
|
+
|
|
88
|
+
df = pd.DataFrame({"amount": [12.5, 18.0, 20.0]})
|
|
89
|
+
context = inspect(df, options=ProfileOptions(top_k=5, max_exact_rows=50_000))
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Options include `top_k`, `include_quantiles`, `max_exact_rows`, and `max_key_columns`. Column labels must be unique strings. Datetime-like text is not parsed automatically. Profiling is in-memory and may require several full-column scans; exact composite-key checks are bounded by the configured row and column limits.
|
|
93
|
+
|
|
94
|
+
Confidence values describe heuristic strength; they are not calibrated probabilities. A candidate key is evidence of observed uniqueness under the profiling rules, not proof of a business key. Serialized output includes `schema_version`; treat changes to that output as version-sensitive until a stable compatibility policy is declared.
|
|
95
|
+
|
|
96
|
+
## Development
|
|
97
|
+
|
|
98
|
+
```bash
|
|
99
|
+
git clone https://github.com/samhoon000/Oribi.git
|
|
100
|
+
cd Oribi
|
|
101
|
+
python -m venv .venv
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Activate the environment, then install the package and development tools:
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
python -m pip install -e ".[dev]"
|
|
108
|
+
pytest -q
|
|
109
|
+
python -m build
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for the contribution workflow and [DESIGN.md](DESIGN.md) for architecture notes. Run the end-to-end example from the repository root with `python -m examples.quickstart`.
|
|
113
|
+
|
|
114
|
+
## Planned work
|
|
115
|
+
|
|
116
|
+
Additional data sources, configurable semantic rules, cross-table foreign-key analysis, and richer profiling options are future work. The current release focuses on pandas DataFrames and deterministic, evidence-bearing heuristics.
|
|
117
|
+
|
|
118
|
+
## License
|
|
119
|
+
|
|
120
|
+
Oribi is released under the [MIT License](LICENSE).
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
oribi/__init__.py,sha256=es5tbOxceggHrVz8QVGeSyflcAJOnkKcTzdeWATZcKQ,662
|
|
2
|
+
oribi/api.py,sha256=tVWjiEpV0yASJP0y3HH19UQB15XOnQA4c3hsmgXUHB0,3764
|
|
3
|
+
oribi/exceptions.py,sha256=6Pi1lSbfQma9yEPa-mYYE5xjtuZQULNm-l1K6J71ODc,224
|
|
4
|
+
oribi/models/__init__.py,sha256=46DaTjg-OnJUp7jq9ytSGI4vs1uusX7AYYaLnfCkGYc,425
|
|
5
|
+
oribi/models/context.py,sha256=E6bmOTfhK6gfI3sTKk9k0NF0vHgFZW9LykgUQed--wA,2168
|
|
6
|
+
oribi/models/evidence.py,sha256=Q8QwxzPy1LeYFi3hr6Z643l5oTgAewrZoFVTxrZoWtY,1816
|
|
7
|
+
oribi/models/profile.py,sha256=TeU2fj0nhydNdZOjW79xfT96xxqZt7lF8HH8fFyFr0Q,1023
|
|
8
|
+
oribi/profiling/__init__.py,sha256=nlxBHD5s-Vqs-BS_Oiw0VRGnqghG4GvYbdoWegyHGy4,37
|
|
9
|
+
oribi/profiling/columns.py,sha256=uWEtIsIQJzja9EKFybZIew4pg_NsFo2ZIvRa0KqRnJM,2199
|
|
10
|
+
oribi/profiling/observations.py,sha256=BFEzlhuG7T3VxBm4Mca4F0Wxz1cGG86_p1VZ2iedYdY,2432
|
|
11
|
+
oribi/profiling/relationships.py,sha256=VrnqPBEcyN1LRfPkVHcl3-UP6zDjoREJ0ouoMXT4y7U,2220
|
|
12
|
+
oribi/profiling/temporal.py,sha256=OgiZP9lXnAiN50XQWe-mHBErTcO0-dYqtPPqXP8OnY4,3028
|
|
13
|
+
oribi/reasoning/__init__.py,sha256=ypyGZJECBGlzwBftcOxhpTIY4wcqZyywNQcMjOgmQ7I,42
|
|
14
|
+
oribi/reasoning/keys.py,sha256=VZ3x0KYg0hw0Gr3CCJuDUp_JVhmf_uqZ5Spa0x4eirs,6498
|
|
15
|
+
oribi/reasoning/rules.py,sha256=xvCKv4rGFNFT0mwB9tafE6nPAG5y-tsaZBR1HDv1s-s,3470
|
|
16
|
+
oribi/reasoning/semantic.py,sha256=tyFrayTZPVELUSgkS4UepFa3zOS0D_nSnR8HmFysq0Y,3651
|
|
17
|
+
oribi/serialization/__init__.py,sha256=cX3RUUt6PghDMtSjP2irG0TfT7ST7HggL5NO8qK_3H4,41
|
|
18
|
+
oribi/serialization/context.py,sha256=U3dR2QfiF2Sh5DPwMYVQx2SQ85ey7zmkEZ2JKAHTpyA,2677
|
|
19
|
+
oribi/serialization/dict.py,sha256=8RnIEd44U4YLQWEs22ROYTVrxla_WrLZq08qVZScVjY,1338
|
|
20
|
+
oribi/serialization/markdown.py,sha256=4zpWsKBw6EZyOgehMNSDggd2AYaPNFxx8U8UcoDUfQ0,1907
|
|
21
|
+
oribi-0.1.0.dist-info/licenses/LICENSE,sha256=YFnvQifh-Fdzfcy6NxNEQOWa2kCVQ6jLq6nW2YfUiQ4,1075
|
|
22
|
+
oribi-0.1.0.dist-info/METADATA,sha256=9Q9UmgFkgWsBeKcdY8pWGZx1y0Lv5WnJ13_zFjDSTkY,5184
|
|
23
|
+
oribi-0.1.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
24
|
+
oribi-0.1.0.dist-info/top_level.txt,sha256=5QKQlmM4Q0-TU5jVrQCI9V0xMcfrHV_RosRx6lVyt3I,6
|
|
25
|
+
oribi-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Oribi contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
oribi
|