oribi 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. oribi-0.1.0/CHANGELOG.md +10 -0
  2. oribi-0.1.0/LICENSE +21 -0
  3. oribi-0.1.0/MANIFEST.in +4 -0
  4. oribi-0.1.0/PKG-INFO +120 -0
  5. oribi-0.1.0/README.md +90 -0
  6. oribi-0.1.0/examples/quickstart.py +21 -0
  7. oribi-0.1.0/oribi/__init__.py +24 -0
  8. oribi-0.1.0/oribi/api.py +84 -0
  9. oribi-0.1.0/oribi/exceptions.py +9 -0
  10. oribi-0.1.0/oribi/models/__init__.py +7 -0
  11. oribi-0.1.0/oribi/models/context.py +59 -0
  12. oribi-0.1.0/oribi/models/evidence.py +80 -0
  13. oribi-0.1.0/oribi/models/profile.py +36 -0
  14. oribi-0.1.0/oribi/profiling/__init__.py +1 -0
  15. oribi-0.1.0/oribi/profiling/columns.py +49 -0
  16. oribi-0.1.0/oribi/profiling/observations.py +66 -0
  17. oribi-0.1.0/oribi/profiling/relationships.py +39 -0
  18. oribi-0.1.0/oribi/profiling/temporal.py +75 -0
  19. oribi-0.1.0/oribi/reasoning/__init__.py +1 -0
  20. oribi-0.1.0/oribi/reasoning/keys.py +169 -0
  21. oribi-0.1.0/oribi/reasoning/rules.py +30 -0
  22. oribi-0.1.0/oribi/reasoning/semantic.py +67 -0
  23. oribi-0.1.0/oribi/serialization/__init__.py +1 -0
  24. oribi-0.1.0/oribi/serialization/context.py +51 -0
  25. oribi-0.1.0/oribi/serialization/dict.py +39 -0
  26. oribi-0.1.0/oribi/serialization/markdown.py +36 -0
  27. oribi-0.1.0/oribi.egg-info/PKG-INFO +120 -0
  28. oribi-0.1.0/oribi.egg-info/SOURCES.txt +37 -0
  29. oribi-0.1.0/oribi.egg-info/dependency_links.txt +1 -0
  30. oribi-0.1.0/oribi.egg-info/requires.txt +6 -0
  31. oribi-0.1.0/oribi.egg-info/top_level.txt +1 -0
  32. oribi-0.1.0/pyproject.toml +41 -0
  33. oribi-0.1.0/setup.cfg +4 -0
  34. oribi-0.1.0/tests/test_api.py +27 -0
  35. oribi-0.1.0/tests/test_keys_grain.py +131 -0
  36. oribi-0.1.0/tests/test_profiling.py +109 -0
  37. oribi-0.1.0/tests/test_reasoning.py +17 -0
  38. oribi-0.1.0/tests/test_semantics.py +54 -0
  39. oribi-0.1.0/tests/test_serialization.py +33 -0
@@ -0,0 +1,10 @@
1
+ # Changelog
2
+
3
+ All notable changes to Oribi are recorded here.
4
+
5
+ ## 0.1.0 — Initial release
6
+
7
+ - Introduce the `oribi` pandas DataFrame profiling API.
8
+ - Report column statistics, evidence-backed semantic roles, recommendations, warnings, relationships, candidate keys, and likely grain.
9
+ - Add conservative temporal-gap warnings for regular datetime series.
10
+ - Provide summary, dictionary, JSON, Markdown, and bounded AI-oriented context output.
oribi-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Oribi contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,4 @@
1
+ include CHANGELOG.md
2
+ include LICENSE
3
+ include README.md
4
+ include examples/quickstart.py
oribi-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,120 @@
1
+ Metadata-Version: 2.4
2
+ Name: oribi
3
+ Version: 0.1.0
4
+ Summary: Evidence-backed analytical context for pandas DataFrames
5
+ Author: Oribi contributors
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/samhoon000/Oribi
8
+ Project-URL: Repository, https://github.com/samhoon000/Oribi
9
+ Project-URL: Issues, https://github.com/samhoon000/Oribi/issues
10
+ Keywords: pandas,data-profiling,data-analysis,data-quality,analytics
11
+ Classifier: Development Status :: 3 - Alpha
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3 :: Only
15
+ Classifier: Programming Language :: Python :: 3.10
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Programming Language :: Python :: 3.14
20
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
21
+ Requires-Python: <3.15,>=3.10
22
+ Description-Content-Type: text/markdown
23
+ License-File: LICENSE
24
+ Requires-Dist: pandas<4,>=1.5.3
25
+ Provides-Extra: dev
26
+ Requires-Dist: pytest>=8; extra == "dev"
27
+ Requires-Dist: pytest-cov>=6; extra == "dev"
28
+ Requires-Dist: build>=1.2; extra == "dev"
29
+ Dynamic: license-file
30
+
31
+ # Oribi
32
+
33
+ Oribi profiles a pandas DataFrame and returns a structured, evidence-backed snapshot of its shape, column statistics, likely semantic roles, candidate keys, possible grain, relationships, and analytical warnings. Use it to inspect data before writing an analysis or to give downstream tools a compact description of a dataset.
34
+
35
+ ## Install
36
+
37
+ ```bash
38
+ python -m pip install oribi
39
+ ```
40
+
41
+ Oribi supports Python 3.10–3.14 and pandas 1.5.3 or newer (below pandas 4).
42
+
43
+ ## Quick start
44
+
45
+ ```python
46
+ import pandas as pd
47
+ from oribi import inspect
48
+
49
+ df = pd.DataFrame({
50
+ "order_id": [101, 101, 102, 102],
51
+ "product_id": [7, 8, 7, 8],
52
+ "revenue": [25.0, 40.0, 30.0, -5.0],
53
+ })
54
+
55
+ context = inspect(df)
56
+ print(context.summary())
57
+ print(context.to_context(max_chars=2_000))
58
+ ```
59
+
60
+ `inspect()` does not modify the DataFrame or keep a reference to it in the result. The returned snapshot can also be rendered as a full Markdown report or serialized to JSON-compatible data:
61
+
62
+ ```python
63
+ markdown = context.to_markdown()
64
+ payload = context.to_dict()
65
+ json_text = context.to_json()
66
+ ```
67
+
68
+ ## What Oribi currently reports
69
+
70
+ - Dataset row/column counts and duplicate rows.
71
+ - Per-column missingness, distinct counts, numeric summaries, categorical top values, and datetime bounds.
72
+ - Likely roles such as identifier, measure, dimension, categorical, temporal, boolean, text, or unknown, with evidence.
73
+ - Candidate single-column and bounded two-column keys, plus a separate likely-grain estimate.
74
+ - Conservative within-DataFrame functional-dependency relationships.
75
+ - Recommendations for likely numeric measures and identifier counts.
76
+ - Warnings for duplicate rows, missingness, non-unique identifiers, negative or skewed measures, high-cardinality categories, and possible gaps in regular datetime series.
77
+
78
+ Inferences are heuristics, not guarantees about business meaning. Temporal-gap warnings require a sufficiently long series with a dominant fixed cadence; irregular event timestamps and short series are left without a gap claim. Relationships describe observed value mappings within one frame and do not prove that a join is valid or safe.
79
+
80
+ ## Public API
81
+
82
+ The main operation is `oribi.inspect(df, *, options=None)`. It returns a `DatasetContext` snapshot. `ProfileOptions` can bound top values and exact key/quantile analysis:
83
+
84
+ ```python
85
+ import pandas as pd
86
+ from oribi import ProfileOptions, inspect
87
+
88
+ df = pd.DataFrame({"amount": [12.5, 18.0, 20.0]})
89
+ context = inspect(df, options=ProfileOptions(top_k=5, max_exact_rows=50_000))
90
+ ```
91
+
92
+ Options include `top_k`, `include_quantiles`, `max_exact_rows`, and `max_key_columns`. Column labels must be unique strings. Datetime-like text is not parsed automatically. Profiling is in-memory and may require several full-column scans; exact composite-key checks are bounded by the configured row and column limits.
93
+
94
+ Confidence values describe heuristic strength; they are not calibrated probabilities. A candidate key is evidence of observed uniqueness under the profiling rules, not proof of a business key. Serialized output includes `schema_version`; treat changes to that output as version-sensitive until a stable compatibility policy is declared.
95
+
96
+ ## Development
97
+
98
+ ```bash
99
+ git clone https://github.com/samhoon000/Oribi.git
100
+ cd Oribi
101
+ python -m venv .venv
102
+ ```
103
+
104
+ Activate the environment, then install the package and development tools:
105
+
106
+ ```bash
107
+ python -m pip install -e ".[dev]"
108
+ pytest -q
109
+ python -m build
110
+ ```
111
+
112
+ See [CONTRIBUTING.md](CONTRIBUTING.md) for the contribution workflow and [DESIGN.md](DESIGN.md) for architecture notes. Run the end-to-end example from the repository root with `python -m examples.quickstart`.
113
+
114
+ ## Planned work
115
+
116
+ Additional data sources, configurable semantic rules, cross-table foreign-key analysis, and richer profiling options are future work. The current release focuses on pandas DataFrames and deterministic, evidence-bearing heuristics.
117
+
118
+ ## License
119
+
120
+ Oribi is released under the [MIT License](LICENSE).
oribi-0.1.0/README.md ADDED
@@ -0,0 +1,90 @@
1
+ # Oribi
2
+
3
+ Oribi profiles a pandas DataFrame and returns a structured, evidence-backed snapshot of its shape, column statistics, likely semantic roles, candidate keys, possible grain, relationships, and analytical warnings. Use it to inspect data before writing an analysis or to give downstream tools a compact description of a dataset.
4
+
5
+ ## Install
6
+
7
+ ```bash
8
+ python -m pip install oribi
9
+ ```
10
+
11
+ Oribi supports Python 3.10–3.14 and pandas 1.5.3 or newer (below pandas 4).
12
+
13
+ ## Quick start
14
+
15
+ ```python
16
+ import pandas as pd
17
+ from oribi import inspect
18
+
19
+ df = pd.DataFrame({
20
+ "order_id": [101, 101, 102, 102],
21
+ "product_id": [7, 8, 7, 8],
22
+ "revenue": [25.0, 40.0, 30.0, -5.0],
23
+ })
24
+
25
+ context = inspect(df)
26
+ print(context.summary())
27
+ print(context.to_context(max_chars=2_000))
28
+ ```
29
+
30
+ `inspect()` does not modify the DataFrame or keep a reference to it in the result. The returned snapshot can also be rendered as a full Markdown report or serialized to JSON-compatible data:
31
+
32
+ ```python
33
+ markdown = context.to_markdown()
34
+ payload = context.to_dict()
35
+ json_text = context.to_json()
36
+ ```
37
+
38
+ ## What Oribi currently reports
39
+
40
+ - Dataset row/column counts and duplicate rows.
41
+ - Per-column missingness, distinct counts, numeric summaries, categorical top values, and datetime bounds.
42
+ - Likely roles such as identifier, measure, dimension, categorical, temporal, boolean, text, or unknown, with evidence.
43
+ - Candidate single-column and bounded two-column keys, plus a separate likely-grain estimate.
44
+ - Conservative within-DataFrame functional-dependency relationships.
45
+ - Recommendations for likely numeric measures and identifier counts.
46
+ - Warnings for duplicate rows, missingness, non-unique identifiers, negative or skewed measures, high-cardinality categories, and possible gaps in regular datetime series.
47
+
48
+ Inferences are heuristics, not guarantees about business meaning. Temporal-gap warnings require a sufficiently long series with a dominant fixed cadence; irregular event timestamps and short series are left without a gap claim. Relationships describe observed value mappings within one frame and do not prove that a join is valid or safe.
49
+
50
+ ## Public API
51
+
52
+ The main operation is `oribi.inspect(df, *, options=None)`. It returns a `DatasetContext` snapshot. `ProfileOptions` can bound top values and exact key/quantile analysis:
53
+
54
+ ```python
55
+ import pandas as pd
56
+ from oribi import ProfileOptions, inspect
57
+
58
+ df = pd.DataFrame({"amount": [12.5, 18.0, 20.0]})
59
+ context = inspect(df, options=ProfileOptions(top_k=5, max_exact_rows=50_000))
60
+ ```
61
+
62
+ Options include `top_k`, `include_quantiles`, `max_exact_rows`, and `max_key_columns`. Column labels must be unique strings. Datetime-like text is not parsed automatically. Profiling is in-memory and may require several full-column scans; exact composite-key checks are bounded by the configured row and column limits.
63
+
64
+ Confidence values describe heuristic strength; they are not calibrated probabilities. A candidate key is evidence of observed uniqueness under the profiling rules, not proof of a business key. Serialized output includes `schema_version`; treat changes to that output as version-sensitive until a stable compatibility policy is declared.
65
+
66
+ ## Development
67
+
68
+ ```bash
69
+ git clone https://github.com/samhoon000/Oribi.git
70
+ cd Oribi
71
+ python -m venv .venv
72
+ ```
73
+
74
+ Activate the environment, then install the package and development tools:
75
+
76
+ ```bash
77
+ python -m pip install -e ".[dev]"
78
+ pytest -q
79
+ python -m build
80
+ ```
81
+
82
+ See [CONTRIBUTING.md](CONTRIBUTING.md) for the contribution workflow and [DESIGN.md](DESIGN.md) for architecture notes. Run the end-to-end example from the repository root with `python -m examples.quickstart`.
83
+
84
+ ## Planned work
85
+
86
+ Additional data sources, configurable semantic rules, cross-table foreign-key analysis, and richer profiling options are future work. The current release focuses on pandas DataFrames and deterministic, evidence-bearing heuristics.
87
+
88
+ ## License
89
+
90
+ Oribi is released under the [MIT License](LICENSE).
@@ -0,0 +1,21 @@
1
+ """Runnable Oribi example: profile a small order-items DataFrame."""
2
+
3
+ import pandas as pd
4
+
5
+ from oribi import inspect
6
+
7
+
8
+ orders = pd.DataFrame(
9
+ {
10
+ "order_id": [101, 101, 102, 102],
11
+ "product_id": [7, 8, 7, 8],
12
+ "revenue": [25.0, 40.0, 30.0, -5.0],
13
+ "order_date": pd.to_datetime(
14
+ ["2026-01-01", "2026-01-01", "2026-01-02", "2026-01-02"]
15
+ ),
16
+ }
17
+ )
18
+
19
+ context = inspect(orders)
20
+ print(context.summary())
21
+ print(context.to_context(max_chars=2_000))
@@ -0,0 +1,24 @@
1
+ """Build structured analytical context from pandas DataFrames."""
2
+
3
+ from .api import ProfileOptions, inspect
4
+ from .exceptions import InvalidInputError, OribiError
5
+ from .models import (
6
+ ColumnContext,
7
+ DatasetContext,
8
+ Evidence,
9
+ GrainContext,
10
+ KeyCandidate,
11
+ Recommendation,
12
+ RelationshipContext,
13
+ SemanticContext,
14
+ StatisticsContext,
15
+ Warning,
16
+ )
17
+
18
+ __all__ = [
19
+ "ColumnContext", "DatasetContext", "Evidence", "GrainContext", "InvalidInputError",
20
+ "KeyCandidate", "ProfileOptions", "Recommendation", "RelationshipContext",
21
+ "OribiError", "SemanticContext", "StatisticsContext", "Warning", "inspect",
22
+ ]
23
+
24
+ __version__ = "0.1.0"
@@ -0,0 +1,84 @@
1
+ """Public inspection API and pipeline orchestration."""
2
+
3
+ from dataclasses import dataclass
4
+
5
+ import pandas as pd
6
+
7
+ from .exceptions import InvalidInputError
8
+ from .models.context import ColumnContext, DatasetContext
9
+ from .models.evidence import Warning
10
+ from .models.profile import DatasetProfile
11
+ from .profiling.columns import profile_column
12
+ from .profiling.observations import AnalysisSession
13
+ from .profiling.relationships import detect_relationships
14
+ from .profiling.temporal import temporal_gap_warnings
15
+ from .reasoning.keys import detect_keys
16
+ from .reasoning.rules import dataset_warnings, reason_column
17
+ from .reasoning.semantic import infer_semantic
18
+
19
+
20
+ @dataclass(frozen=True)
21
+ class ProfileOptions:
22
+ """Controls profiling work while keeping results deterministic."""
23
+
24
+ top_k: int = 10
25
+ include_quantiles: bool = True
26
+ max_exact_rows: int = 100_000
27
+ max_key_columns: int = 8
28
+
29
+ def __post_init__(self) -> None:
30
+ if self.top_k < 1:
31
+ raise ValueError("top_k must be at least 1")
32
+ if self.max_exact_rows < 1 or self.max_key_columns < 2:
33
+ raise ValueError("max_exact_rows must be positive and max_key_columns at least 2")
34
+
35
+
36
+ def inspect(df: pd.DataFrame, *, options: ProfileOptions | None = None) -> DatasetContext:
37
+ """Profile a pandas DataFrame and return its analytical context snapshot.
38
+
39
+ The returned context contains no reference to ``df``. Object columns that
40
+ appear datetime-like are not parsed automatically.
41
+ """
42
+ options = options or ProfileOptions()
43
+ if not isinstance(options, ProfileOptions):
44
+ raise InvalidInputError("options must be a ProfileOptions instance")
45
+ session = AnalysisSession(df, options)
46
+ nrows, ncols = df.shape
47
+ try:
48
+ duplicates = int(df.duplicated().sum())
49
+ except (TypeError, ValueError):
50
+ duplicates = None
51
+ duplicate_rate = (duplicates / nrows) if duplicates is not None and nrows else (0.0 if duplicates == 0 else None)
52
+ names = list(df.columns)
53
+ observations = {name: session.observe(name) for name in names}
54
+ stats = {name: profile_column(obs, nrows, options.top_k, options.include_quantiles and nrows <= options.max_exact_rows) for name, obs in observations.items()}
55
+ semantics = {name: infer_semantic(name, observations[name].logical_type, stats[name], nrows) for name in names}
56
+ key_candidates, grain = detect_keys(
57
+ df, names, stats, {name: semantics[name].role for name in names},
58
+ max_columns=options.max_key_columns, max_exact_rows=options.max_exact_rows,
59
+ )
60
+ role_map = {name: semantics[name].role for name in names}
61
+ relationships = detect_relationships(df, role_map, max_rows=options.max_exact_rows)
62
+ columns = []
63
+ for name in names:
64
+ obs = observations[name]
65
+ recommendations, warnings = reason_column(name, obs.logical_type, semantics[name], stats[name])
66
+ if obs.logical_type == "temporal":
67
+ warnings += temporal_gap_warnings(obs.series, name)
68
+ columns.append(ColumnContext(name, str(obs.series.dtype), obs.logical_type, stats[name], semantics[name], recommendations, warnings))
69
+ global_warnings = list(dataset_warnings(duplicates, nrows))
70
+ for col in columns:
71
+ global_warnings.extend(col.warnings)
72
+ if duplicates is None:
73
+ global_warnings.append(Warning(
74
+ "duplicate_check_unavailable", "low", "Duplicate-row check unavailable",
75
+ "Pandas could not compare rows with the available cell values.",
76
+ ))
77
+ return DatasetContext(
78
+ profile=DatasetProfile(nrows, ncols, duplicates, duplicate_rate),
79
+ columns=tuple(columns),
80
+ candidate_keys=key_candidates,
81
+ grain=grain,
82
+ relationships=relationships,
83
+ warnings=tuple(global_warnings),
84
+ )
@@ -0,0 +1,9 @@
1
+ """Public exceptions raised by Oribi."""
2
+
3
+
4
+ class OribiError(Exception):
5
+ """Base exception for Oribi errors."""
6
+
7
+
8
+ class InvalidInputError(OribiError, ValueError):
9
+ """Raised when an input cannot be profiled safely."""
@@ -0,0 +1,7 @@
1
+ """Public context models."""
2
+
3
+ from .context import ColumnContext, DatasetContext
4
+ from .evidence import Evidence, GrainContext, KeyCandidate, Recommendation, RelationshipContext, SemanticContext, Warning
5
+ from .profile import StatisticsContext
6
+
7
+ __all__ = ["ColumnContext", "DatasetContext", "Evidence", "GrainContext", "KeyCandidate", "Recommendation", "RelationshipContext", "SemanticContext", "StatisticsContext", "Warning"]
@@ -0,0 +1,59 @@
1
+ """Canonical result returned by :func:`oribi.inspect`."""
2
+
3
+ from dataclasses import dataclass
4
+ from typing import Any
5
+
6
+ from .evidence import GrainContext, KeyCandidate, Recommendation, RelationshipContext, SemanticContext, Warning
7
+ from .profile import DatasetProfile, StatisticsContext
8
+
9
+
10
+ @dataclass(frozen=True)
11
+ class ColumnContext:
12
+ name: str
13
+ dtype: str
14
+ logical_type: str
15
+ statistics: StatisticsContext
16
+ semantic: SemanticContext
17
+ recommendations: tuple[Recommendation, ...] = ()
18
+ warnings: tuple[Warning, ...] = ()
19
+
20
+
21
+ @dataclass(frozen=True)
22
+ class DatasetContext:
23
+ profile: DatasetProfile
24
+ columns: tuple[ColumnContext, ...]
25
+ candidate_keys: tuple[KeyCandidate, ...] = ()
26
+ grain: GrainContext | None = None
27
+ relationships: tuple[RelationshipContext, ...] = ()
28
+ warnings: tuple[Warning, ...] = ()
29
+ schema_version: str = "1"
30
+
31
+ def to_dict(self) -> dict[str, Any]:
32
+ from ..serialization.dict import to_dict
33
+ return to_dict(self)
34
+
35
+ def to_json(self, *, indent: int | None = 2) -> str:
36
+ from ..serialization.dict import to_json
37
+ return to_json(self, indent=indent)
38
+
39
+ def to_markdown(self) -> str:
40
+ from ..serialization.markdown import to_markdown
41
+ return to_markdown(self)
42
+
43
+ def to_context(self, *, max_chars: int = 8000) -> str:
44
+ from ..serialization.context import to_context
45
+ return to_context(self, max_chars=max_chars)
46
+
47
+ def summary(self) -> str:
48
+ rows = self.profile.row_count
49
+ cols = self.profile.column_count
50
+ parts = [f"Dataset: {rows:,} rows × {cols:,} columns."]
51
+ if self.profile.duplicate_row_count:
52
+ parts.append(f"{self.profile.duplicate_row_count:,} duplicate rows observed.")
53
+ if self.grain and self.grain.status == "estimated" and self.grain.label:
54
+ parts.append(f"Likely grain: {self.grain.label} (confidence {self.grain.confidence:.2f}).")
55
+ elif self.grain:
56
+ parts.append("Likely grain could not be determined confidently.")
57
+ if self.warnings:
58
+ parts.append(f"{len(self.warnings)} analytical warning(s) detected.")
59
+ return " ".join(parts)
@@ -0,0 +1,80 @@
1
+ """Evidence-bearing inferences and recommendations."""
2
+
3
+ from dataclasses import dataclass, field
4
+ from collections.abc import Mapping
5
+ from types import MappingProxyType
6
+ from typing import Any
7
+
8
+
9
+ def _freeze(value: Any) -> Any:
10
+ if isinstance(value, Mapping):
11
+ return MappingProxyType({key: _freeze(item) for key, item in value.items()})
12
+ if isinstance(value, (list, tuple)):
13
+ return tuple(_freeze(item) for item in value)
14
+ if isinstance(value, set):
15
+ return frozenset(_freeze(item) for item in value)
16
+ return value
17
+
18
+
19
+ @dataclass(frozen=True)
20
+ class Evidence:
21
+ code: str
22
+ statement: str
23
+ details: Mapping[str, Any] = field(default_factory=dict)
24
+
25
+ def __post_init__(self) -> None:
26
+ object.__setattr__(self, "details", _freeze(self.details))
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class SemanticContext:
31
+ role: str
32
+ confidence: float
33
+ evidence: tuple[Evidence, ...] = ()
34
+
35
+
36
+ @dataclass(frozen=True)
37
+ class Recommendation:
38
+ operation: str
39
+ disposition: str
40
+ rationale: str
41
+ evidence: tuple[Evidence, ...] = ()
42
+
43
+
44
+ @dataclass(frozen=True)
45
+ class Warning:
46
+ code: str
47
+ severity: str
48
+ title: str
49
+ message: str
50
+ evidence: tuple[Evidence, ...] = ()
51
+ implication: str = ""
52
+ recommendation: str | None = None
53
+
54
+
55
+ @dataclass(frozen=True)
56
+ class KeyCandidate:
57
+ columns: tuple[str, ...]
58
+ kind: str
59
+ confidence: float
60
+ uniqueness_rate: float
61
+ evidence: tuple[Evidence, ...] = ()
62
+
63
+
64
+ @dataclass(frozen=True)
65
+ class GrainContext:
66
+ status: str
67
+ label: str | None
68
+ columns: tuple[str, ...]
69
+ confidence: float
70
+ evidence: tuple[Evidence, ...] = ()
71
+
72
+
73
+ @dataclass(frozen=True)
74
+ class RelationshipContext:
75
+ source_column: str
76
+ target_column: str
77
+ kind: str
78
+ coverage: float
79
+ confidence: float
80
+ evidence: tuple[Evidence, ...] = ()
@@ -0,0 +1,36 @@
1
+ """Observed dataset and column statistics."""
2
+
3
+ from dataclasses import dataclass, field
4
+ from collections.abc import Mapping
5
+ from types import MappingProxyType
6
+ from typing import Any
7
+
8
+
9
+ @dataclass(frozen=True)
10
+ class StatisticsContext:
11
+ missing_count: int
12
+ missing_rate: float
13
+ distinct_count: int
14
+ distinct_rate: float
15
+ minimum: Any = None
16
+ maximum: Any = None
17
+ mean: float | None = None
18
+ median: float | None = None
19
+ standard_deviation: float | None = None
20
+ quantiles: Mapping[str, float] = field(default_factory=dict)
21
+ zero_count: int | None = None
22
+ negative_count: int | None = None
23
+ skewness: float | None = None
24
+ top_values: tuple[tuple[Any, int], ...] = ()
25
+ date_range: str | None = None
26
+
27
+ def __post_init__(self) -> None:
28
+ object.__setattr__(self, "quantiles", MappingProxyType(dict(self.quantiles)))
29
+
30
+
31
+ @dataclass(frozen=True)
32
+ class DatasetProfile:
33
+ row_count: int
34
+ column_count: int
35
+ duplicate_row_count: int | None
36
+ duplicate_row_rate: float | None
@@ -0,0 +1 @@
1
+ """Internal profiling components."""
@@ -0,0 +1,49 @@
1
+ """Statistics for observed columns."""
2
+
3
+ import math
4
+ from typing import Any
5
+
6
+ import pandas as pd
7
+
8
+ from ..models.profile import StatisticsContext
9
+ from .observations import ColumnObservation
10
+
11
+
12
+ def profile_column(obs: ColumnObservation, row_count: int, top_k: int, include_quantiles: bool) -> StatisticsContext:
13
+ s = obs.series
14
+ denom = row_count or 1
15
+ non_null = s.dropna()
16
+ values: dict[str, Any] = {}
17
+ if obs.logical_type == "numeric" and len(non_null):
18
+ numeric = pd.to_numeric(non_null, errors="coerce").dropna()
19
+ if len(numeric):
20
+ def finite_or_none(value: Any) -> float | None:
21
+ result = float(value)
22
+ return result if math.isfinite(result) else None
23
+
24
+ values.update(minimum=numeric.min(), maximum=numeric.max(), mean=finite_or_none(numeric.mean()), median=finite_or_none(numeric.median()), standard_deviation=finite_or_none(numeric.std()))
25
+ values["zero_count"] = int((numeric == 0).sum())
26
+ values["negative_count"] = int((numeric < 0).sum())
27
+ if include_quantiles:
28
+ qs = numeric.quantile([.25, .5, .75])
29
+ values["quantiles"] = {str(q): float(v) for q, v in qs.items()}
30
+ if len(numeric) >= 3 and numeric.nunique() > 1:
31
+ skew = float(numeric.skew())
32
+ values["skewness"] = skew if pd.notna(skew) else None
33
+ elif obs.logical_type == "temporal" and len(non_null):
34
+ values["minimum"] = non_null.min()
35
+ values["maximum"] = non_null.max()
36
+ values["date_range"] = f"{values['minimum']} to {values['maximum']}"
37
+ if len(non_null) and (obs.logical_type in {"text", "categorical", "boolean", "unknown"} or obs.distinct_count <= top_k * 4):
38
+ try:
39
+ counts = non_null.value_counts(dropna=True).head(top_k)
40
+ values["top_values"] = tuple((k, int(v)) for k, v in counts.items())
41
+ except (TypeError, ValueError):
42
+ pass
43
+ return StatisticsContext(
44
+ missing_count=obs.missing_count,
45
+ missing_rate=obs.missing_count / denom,
46
+ distinct_count=obs.distinct_count,
47
+ distinct_rate=obs.distinct_count / denom,
48
+ **values,
49
+ )