colsemantics 0.1.0__tar.gz → 0.3.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. colsemantics-0.3.1/PKG-INFO +193 -0
  2. colsemantics-0.3.1/README.md +160 -0
  3. {colsemantics-0.1.0 → colsemantics-0.3.1}/pyproject.toml +27 -10
  4. colsemantics-0.3.1/src/colsemantics/__init__.py +220 -0
  5. colsemantics-0.3.1/src/colsemantics/_taxonomy.py +378 -0
  6. colsemantics-0.3.1/src/colsemantics/benchmark.py +146 -0
  7. colsemantics-0.3.1/src/colsemantics/calibration.json +17 -0
  8. colsemantics-0.3.1/src/colsemantics/calibration.py +42 -0
  9. colsemantics-0.3.1/src/colsemantics/cli.py +64 -0
  10. colsemantics-0.3.1/src/colsemantics/context.py +73 -0
  11. colsemantics-0.3.1/src/colsemantics/csv_inference.py +69 -0
  12. colsemantics-0.3.1/src/colsemantics/detectors.py +318 -0
  13. colsemantics-0.3.1/src/colsemantics/evidence.py +54 -0
  14. colsemantics-0.3.1/src/colsemantics/profiles.py +29 -0
  15. colsemantics-0.3.1/src/colsemantics/sensitivity.py +30 -0
  16. colsemantics-0.3.1/src/colsemantics/tokens.py +80 -0
  17. colsemantics-0.3.1/src/colsemantics/vocabularies.py +89 -0
  18. colsemantics-0.3.1/src/colsemantics/vocabulary.py +473 -0
  19. colsemantics-0.3.1/src/colsemantics.egg-info/PKG-INFO +193 -0
  20. colsemantics-0.3.1/src/colsemantics.egg-info/SOURCES.txt +33 -0
  21. colsemantics-0.3.1/src/colsemantics.egg-info/entry_points.txt +2 -0
  22. colsemantics-0.3.1/src/colsemantics.egg-info/requires.txt +9 -0
  23. colsemantics-0.3.1/tests/test_benchmark.py +70 -0
  24. colsemantics-0.3.1/tests/test_calibration.py +29 -0
  25. colsemantics-0.3.1/tests/test_cli.py +39 -0
  26. colsemantics-0.3.1/tests/test_colsemantics.py +202 -0
  27. colsemantics-0.3.1/tests/test_csv_inference.py +49 -0
  28. colsemantics-0.3.1/tests/test_english_core.py +10 -0
  29. colsemantics-0.3.1/tests/test_profiles.py +24 -0
  30. colsemantics-0.3.1/tests/test_sensitivity.py +42 -0
  31. colsemantics-0.1.0/PKG-INFO +0 -162
  32. colsemantics-0.1.0/README.md +0 -135
  33. colsemantics-0.1.0/src/colsemantics/__init__.py +0 -248
  34. colsemantics-0.1.0/src/colsemantics/_taxonomy.py +0 -177
  35. colsemantics-0.1.0/src/colsemantics/detectors.py +0 -362
  36. colsemantics-0.1.0/src/colsemantics/evidence.py +0 -82
  37. colsemantics-0.1.0/src/colsemantics/tokens.py +0 -131
  38. colsemantics-0.1.0/src/colsemantics/vocabulary.py +0 -171
  39. colsemantics-0.1.0/src/colsemantics.egg-info/PKG-INFO +0 -162
  40. colsemantics-0.1.0/src/colsemantics.egg-info/SOURCES.txt +0 -16
  41. colsemantics-0.1.0/src/colsemantics.egg-info/requires.txt +0 -7
  42. colsemantics-0.1.0/tests/test_colsemantics.py +0 -384
  43. {colsemantics-0.1.0 → colsemantics-0.3.1}/LICENSE +0 -0
  44. {colsemantics-0.1.0 → colsemantics-0.3.1}/setup.cfg +0 -0
  45. {colsemantics-0.1.0 → colsemantics-0.3.1}/src/colsemantics/py.typed +0 -0
  46. {colsemantics-0.1.0 → colsemantics-0.3.1}/src/colsemantics.egg-info/dependency_links.txt +0 -0
  47. {colsemantics-0.1.0 → colsemantics-0.3.1}/src/colsemantics.egg-info/top_level.txt +0 -0
@@ -0,0 +1,193 @@
1
+ Metadata-Version: 2.4
2
+ Name: colsemantics
3
+ Version: 0.3.1
4
+ Summary: Infer semantic roles and domains for tabular columns from names, values, and table context.
5
+ Author: Caio
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/Caio-Analytics/colsemantics
8
+ Project-URL: Repository, https://github.com/Caio-Analytics/colsemantics
9
+ Project-URL: Issues, https://github.com/Caio-Analytics/colsemantics/issues
10
+ Project-URL: Changelog, https://github.com/Caio-Analytics/colsemantics/blob/main/CHANGELOG.md
11
+ Keywords: semantic-inference,data-profiling,schema-inference,column-classification,pii,etl
12
+ Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Programming Language :: Python :: 3.13
18
+ Classifier: Programming Language :: Python :: 3.14
19
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
20
+ Classifier: Typing :: Typed
21
+ Requires-Python: >=3.11
22
+ Description-Content-Type: text/markdown
23
+ License-File: LICENSE
24
+ Requires-Dist: rapidfuzz>=3.14.6
25
+ Requires-Dist: unidecode>=1.4.0
26
+ Requires-Dist: PyYAML>=6.0.3
27
+ Provides-Extra: dev
28
+ Requires-Dist: pytest>=9.2.0; extra == "dev"
29
+ Requires-Dist: pytest-cov>=7.1.0; extra == "dev"
30
+ Requires-Dist: mypy>=2.4.1; extra == "dev"
31
+ Requires-Dist: ruff>=0.17.4; extra == "dev"
32
+ Dynamic: license-file
33
+
34
+ # colsemantics
35
+
36
+ [![PyPI version](https://img.shields.io/pypi/v/colsemantics)](https://pypi.org/project/colsemantics/)
37
+ [![Python versions](https://img.shields.io/pypi/pyversions/colsemantics)](https://pypi.org/project/colsemantics/)
38
+ [![CI](https://github.com/Caio-Analytics/colsemantics/actions/workflows/ci.yml/badge.svg)](https://github.com/Caio-Analytics/colsemantics/actions/workflows/ci.yml)
39
+ [![License](https://img.shields.io/github/license/Caio-Analytics/colsemantics)](LICENSE)
40
+ [![PyPI downloads](https://img.shields.io/pypi/dm/colsemantics)](https://pypi.org/project/colsemantics/)
41
+
42
+ `colsemantics` infers the structural role and business domain of tabular columns from column names, sampled values, and neighboring columns.
43
+
44
+ It is useful when a dataset has unclear headers such as `cd_dpto_lot`, `f27`, or `SUPPLIER_CONTACT_CODE`.
45
+
46
+ ## Installation
47
+
48
+ ```bash
49
+ pip install colsemantics
50
+ ```
51
+
52
+ ## Quick start
53
+
54
+ ```python
55
+ from colsemantics import infer_column
56
+
57
+ result = infer_column("SUPPLIER_CONTACT_CODE")
58
+ print(result["role"])
59
+ ```
60
+
61
+ The English API returns English field names and category labels.
62
+
63
+ ## Analyze a CSV file
64
+
65
+ Install the command-line interface with the package, then create a JSON report from a bounded CSV sample:
66
+
67
+ ```bash
68
+ colsemantics infer employees.csv --profile pt-BR --sample 10000 --output report.json
69
+ ```
70
+
71
+ CSV is the supported file format in this release. The report records its source, profile, sample size, inferred columns, and a semantic summary.
72
+
73
+ ## Use sampled values
74
+
75
+ ```python
76
+ from colsemantics import ContentProfile, infer_column
77
+
78
+ profile = ContentProfile(
79
+ data_type="Text",
80
+ distinct_values=["SP", "RJ", "MG", "BA"],
81
+ distinct_count=4,
82
+ uniqueness_ratio=0.1,
83
+ )
84
+
85
+ result = infer_column("f27", profile=profile)
86
+ print(result["domain"])
87
+ ```
88
+
89
+ ## Analyze a table
90
+
91
+ ```python
92
+ from colsemantics import infer_table
93
+
94
+ results = infer_table(
95
+ [
96
+ {"column_name": "employee_id"},
97
+ {"column_name": "department_name"},
98
+ {"column_name": "start_date"},
99
+ ]
100
+ )
101
+ ```
102
+
103
+ `infer_table` uses high-confidence classifications as table context when it evaluates ambiguous columns.
104
+
105
+ ## Vocabulary profiles and extensions
106
+
107
+ `pt-BR` is the embedded profile. Use `available_profiles()` to list embedded profiles and `load_profile()` when an application needs an explicit context.
108
+
109
+ ```python
110
+ from colsemantics import infer_column, temporary_profile
111
+
112
+ with temporary_profile("pt-BR", "company-vocabulary.yaml"):
113
+ result = infer_column("cost_bucket")
114
+ ```
115
+
116
+ YAML extensions use the English schema and apply only within the selected context:
117
+
118
+ Provide one or more comma-separated YAML files to `load_vocabularies`, then use the returned context for the current operation.
119
+
120
+ ```yaml
121
+ strong_categories:
122
+ Project domain:
123
+ - workstream
124
+ column_overrides:
125
+ cost_bucket: Finance / Cost
126
+ ```
127
+
128
+ Custom vocabularies are scoped with `ContextVar`, so separate concurrent analyses do not share vocabulary changes.
129
+
130
+ ## Benchmark inference quality
131
+
132
+ Run the synthetic benchmark fixture included with the repository:
133
+
134
+ ```bash
135
+ colsemantics benchmark benchmarks/synthetic.json --output benchmark-report.json
136
+ ```
137
+
138
+ The report includes semantic, role, and domain accuracy, coverage, per-profile metrics, a confusion matrix, and calibration buckets with expected calibration error. Public corpus definitions are stored as source manifests; they are not bundled or downloaded automatically.
139
+
140
+ ## Confidence and review decisions
141
+
142
+ `raw_confidence` is the direct score assembled from independent evidence. `confidence` is the profile calibration for that score bucket, derived from the versioned calibration fixture. A result sets `review_required` when calibrated confidence is below `0.80`.
143
+
144
+ The embedded calibration is provisional: it is a controlled regression fixture, not evidence of production-grade accuracy. A held-out, versioned corpus is required before treating its precision as a production measurement.
145
+
146
+ ## Sensitive-data recommendations
147
+
148
+ Each result includes a descriptive `sensitivity` recommendation. It does not modify input data.
149
+
150
+ - Validated CPF patterns: `high` / `mask`
151
+ - Person identity semantics: `high` / `restrict`
152
+ - Contact semantics: `high` / `mask`
153
+ - Other identifiers: `medium` / `review`
154
+ - Everything else: `none` / `allow`
155
+
156
+ ## Result shape
157
+
158
+ ```python
159
+ {
160
+ "semantic": str,
161
+ "role": str | None,
162
+ "domain": str | None,
163
+ "raw_confidence": float,
164
+ "confidence": float,
165
+ "evidence": str,
166
+ "conclusive": bool,
167
+ "review_required": bool,
168
+ "sensitivity": {"level": str, "action": str, "reason": str},
169
+ "hypotheses": list[dict],
170
+ }
171
+ ```
172
+
173
+ ## Migrating from 0.2
174
+
175
+ `normalizar` and `tokenizar` were renamed to `normalize` and `tokenize`. The old names still work but emit a `DeprecationWarning` and will be removed in 1.0. `infer_table`, `assess_sensitivity`, `calibrate`, and `requires_review` are now listed in `__all__`.
176
+
177
+ Python 3.10 is no longer supported (it reached end of life, and current `rapidfuzz` requires 3.11).
178
+
179
+ ## Contributing
180
+
181
+ Contributions are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, checks, and the release process, and [CHANGELOG.md](CHANGELOG.md) for version history.
182
+
183
+ ```bash
184
+ python -m pip install -e '.[dev]'
185
+ pytest --cov
186
+ ruff check .
187
+ ruff format --check .
188
+ mypy
189
+ ```
190
+
191
+ ## License
192
+
193
+ MIT.
@@ -0,0 +1,160 @@
1
+ # colsemantics
2
+
3
+ [![PyPI version](https://img.shields.io/pypi/v/colsemantics)](https://pypi.org/project/colsemantics/)
4
+ [![Python versions](https://img.shields.io/pypi/pyversions/colsemantics)](https://pypi.org/project/colsemantics/)
5
+ [![CI](https://github.com/Caio-Analytics/colsemantics/actions/workflows/ci.yml/badge.svg)](https://github.com/Caio-Analytics/colsemantics/actions/workflows/ci.yml)
6
+ [![License](https://img.shields.io/github/license/Caio-Analytics/colsemantics)](LICENSE)
7
+ [![PyPI downloads](https://img.shields.io/pypi/dm/colsemantics)](https://pypi.org/project/colsemantics/)
8
+
9
+ `colsemantics` infers the structural role and business domain of tabular columns from column names, sampled values, and neighboring columns.
10
+
11
+ It is useful when a dataset has unclear headers such as `cd_dpto_lot`, `f27`, or `SUPPLIER_CONTACT_CODE`.
12
+
13
+ ## Installation
14
+
15
+ ```bash
16
+ pip install colsemantics
17
+ ```
18
+
19
+ ## Quick start
20
+
21
+ ```python
22
+ from colsemantics import infer_column
23
+
24
+ result = infer_column("SUPPLIER_CONTACT_CODE")
25
+ print(result["role"])
26
+ ```
27
+
28
+ The English API returns English field names and category labels.
29
+
30
+ ## Analyze a CSV file
31
+
32
+ Install the command-line interface with the package, then create a JSON report from a bounded CSV sample:
33
+
34
+ ```bash
35
+ colsemantics infer employees.csv --profile pt-BR --sample 10000 --output report.json
36
+ ```
37
+
38
+ CSV is the supported file format in this release. The report records its source, profile, sample size, inferred columns, and a semantic summary.
39
+
40
+ ## Use sampled values
41
+
42
+ ```python
43
+ from colsemantics import ContentProfile, infer_column
44
+
45
+ profile = ContentProfile(
46
+ data_type="Text",
47
+ distinct_values=["SP", "RJ", "MG", "BA"],
48
+ distinct_count=4,
49
+ uniqueness_ratio=0.1,
50
+ )
51
+
52
+ result = infer_column("f27", profile=profile)
53
+ print(result["domain"])
54
+ ```
55
+
56
+ ## Analyze a table
57
+
58
+ ```python
59
+ from colsemantics import infer_table
60
+
61
+ results = infer_table(
62
+ [
63
+ {"column_name": "employee_id"},
64
+ {"column_name": "department_name"},
65
+ {"column_name": "start_date"},
66
+ ]
67
+ )
68
+ ```
69
+
70
+ `infer_table` uses high-confidence classifications as table context when it evaluates ambiguous columns.
71
+
72
+ ## Vocabulary profiles and extensions
73
+
74
+ `pt-BR` is the embedded profile. Use `available_profiles()` to list embedded profiles and `load_profile()` when an application needs an explicit context.
75
+
76
+ ```python
77
+ from colsemantics import infer_column, temporary_profile
78
+
79
+ with temporary_profile("pt-BR", "company-vocabulary.yaml"):
80
+ result = infer_column("cost_bucket")
81
+ ```
82
+
83
+ YAML extensions use the English schema and apply only within the selected context:
84
+
85
+ Provide one or more comma-separated YAML files to `load_vocabularies`, then use the returned context for the current operation.
86
+
87
+ ```yaml
88
+ strong_categories:
89
+ Project domain:
90
+ - workstream
91
+ column_overrides:
92
+ cost_bucket: Finance / Cost
93
+ ```
94
+
95
+ Custom vocabularies are scoped with `ContextVar`, so separate concurrent analyses do not share vocabulary changes.
96
+
97
+ ## Benchmark inference quality
98
+
99
+ Run the synthetic benchmark fixture included with the repository:
100
+
101
+ ```bash
102
+ colsemantics benchmark benchmarks/synthetic.json --output benchmark-report.json
103
+ ```
104
+
105
+ The report includes semantic, role, and domain accuracy, coverage, per-profile metrics, a confusion matrix, and calibration buckets with expected calibration error. Public corpus definitions are stored as source manifests; they are not bundled or downloaded automatically.
106
+
107
+ ## Confidence and review decisions
108
+
109
+ `raw_confidence` is the direct score assembled from independent evidence. `confidence` is the profile calibration for that score bucket, derived from the versioned calibration fixture. A result sets `review_required` when calibrated confidence is below `0.80`.
110
+
111
+ The embedded calibration is provisional: it is a controlled regression fixture, not evidence of production-grade accuracy. A held-out, versioned corpus is required before treating its precision as a production measurement.
112
+
113
+ ## Sensitive-data recommendations
114
+
115
+ Each result includes a descriptive `sensitivity` recommendation. It does not modify input data.
116
+
117
+ - Validated CPF patterns: `high` / `mask`
118
+ - Person identity semantics: `high` / `restrict`
119
+ - Contact semantics: `high` / `mask`
120
+ - Other identifiers: `medium` / `review`
121
+ - Everything else: `none` / `allow`
122
+
123
+ ## Result shape
124
+
125
+ ```python
126
+ {
127
+ "semantic": str,
128
+ "role": str | None,
129
+ "domain": str | None,
130
+ "raw_confidence": float,
131
+ "confidence": float,
132
+ "evidence": str,
133
+ "conclusive": bool,
134
+ "review_required": bool,
135
+ "sensitivity": {"level": str, "action": str, "reason": str},
136
+ "hypotheses": list[dict],
137
+ }
138
+ ```
139
+
140
+ ## Migrating from 0.2
141
+
142
+ `normalizar` and `tokenizar` were renamed to `normalize` and `tokenize`. The old names still work but emit a `DeprecationWarning` and will be removed in 1.0. `infer_table`, `assess_sensitivity`, `calibrate`, and `requires_review` are now listed in `__all__`.
143
+
144
+ Python 3.10 is no longer supported (it reached end of life, and current `rapidfuzz` requires 3.11).
145
+
146
+ ## Contributing
147
+
148
+ Contributions are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, checks, and the release process, and [CHANGELOG.md](CHANGELOG.md) for version history.
149
+
150
+ ```bash
151
+ python -m pip install -e '.[dev]'
152
+ pytest --cov
153
+ ruff check .
154
+ ruff format --check .
155
+ mypy
156
+ ```
157
+
158
+ ## License
159
+
160
+ MIT.
@@ -1,32 +1,40 @@
1
1
  [project]
2
2
  name = "colsemantics"
3
- version = "0.1.0"
4
- description = "Inferência semântica de colunas: descobre o papel (o que a coluna é) e o domínio (do que ela fala) a partir do nome, de abreviações corporativas e do conteúdo."
3
+ version = "0.3.1"
4
+ description = "Infer semantic roles and domains for tabular columns from names, values, and table context."
5
5
  readme = "README.md"
6
6
  license = "MIT"
7
7
  authors = [{ name = "Caio" }]
8
- requires-python = ">=3.10"
9
- keywords = ["semantic", "data-profiling", "column-inference", "pii", "etl", "pandas"]
8
+ requires-python = ">=3.11"
9
+ keywords = ["semantic-inference", "data-profiling", "schema-inference", "column-classification", "pii", "etl"]
10
10
  classifiers = [
11
11
  "Development Status :: 3 - Alpha",
12
12
  "Intended Audience :: Developers",
13
13
  "Programming Language :: Python :: 3",
14
- "Programming Language :: Python :: 3.10",
15
14
  "Programming Language :: Python :: 3.11",
16
15
  "Programming Language :: Python :: 3.12",
16
+ "Programming Language :: Python :: 3.13",
17
+ "Programming Language :: Python :: 3.14",
17
18
  "Topic :: Software Development :: Libraries :: Python Modules",
18
19
  "Typing :: Typed",
19
20
  ]
20
21
  dependencies = [
21
- "rapidfuzz>=3.0",
22
- "unidecode>=1.3",
22
+ "rapidfuzz>=3.14.6",
23
+ "unidecode>=1.4.0",
24
+ "PyYAML>=6.0.3",
23
25
  ]
24
26
 
25
27
  [project.urls]
26
28
  Homepage = "https://github.com/Caio-Analytics/colsemantics"
29
+ Repository = "https://github.com/Caio-Analytics/colsemantics"
30
+ Issues = "https://github.com/Caio-Analytics/colsemantics/issues"
31
+ Changelog = "https://github.com/Caio-Analytics/colsemantics/blob/main/CHANGELOG.md"
32
+
33
+ [project.scripts]
34
+ colsemantics = "colsemantics.cli:main"
27
35
 
28
36
  [project.optional-dependencies]
29
- dev = ["pytest>=8", "mypy>=1.10", "ruff>=0.6"]
37
+ dev = ["pytest>=9.2.0", "pytest-cov>=7.1.0", "mypy>=2.4.1", "ruff>=0.17.4"]
30
38
 
31
39
  [build-system]
32
40
  requires = ["setuptools>=68"]
@@ -36,7 +44,7 @@ build-backend = "setuptools.build_meta"
36
44
  where = ["src"]
37
45
 
38
46
  [tool.setuptools.package-data]
39
- colsemantics = ["py.typed"]
47
+ colsemantics = ["py.typed", "calibration.json"]
40
48
 
41
49
  [tool.pytest.ini_options]
42
50
  testpaths = ["tests"]
@@ -55,7 +63,16 @@ ignore = [
55
63
  ]
56
64
 
57
65
  [tool.mypy]
58
- python_version = "3.10"
66
+ python_version = "3.11"
59
67
  files = ["src/colsemantics"]
60
68
  ignore_missing_imports = true
61
69
  warn_unused_ignores = true
70
+
71
+ [tool.coverage.run]
72
+ source = ["colsemantics"]
73
+ branch = true
74
+
75
+ [tool.coverage.report]
76
+ fail_under = 85
77
+ show_missing = true
78
+ skip_covered = true
@@ -0,0 +1,220 @@
1
+ import warnings
2
+ from typing import Any
3
+
4
+ from . import _taxonomy as config
5
+ from .calibration import calibrate, requires_review
6
+ from .context import SemanticContext, current_context
7
+ from .csv_inference import infer_csv
8
+ from .detectors import (
9
+ STRUCTURAL_ROLES,
10
+ ContentProfile,
11
+ by_content_pattern,
12
+ by_fuzzy,
13
+ by_gazetteer,
14
+ by_strong_token,
15
+ by_structural_signature,
16
+ by_table_context,
17
+ )
18
+ from .evidence import DOMAIN_AXIS, ROLE_AXIS, Evidence, choose, rank
19
+ from .profiles import available_profiles, load_profile, temporary_profile
20
+ from .sensitivity import assess_sensitivity
21
+ from .tokens import normalize, tokenize
22
+ from .vocabularies import export_overrides_template, load_vocabularies, temporary_vocabulary
23
+
24
+ __version__ = "0.3.1"
25
+
26
+ __all__ = [
27
+ "ContentProfile",
28
+ "SemanticContext",
29
+ "assess_sensitivity",
30
+ "available_profiles",
31
+ "calibrate",
32
+ "current_context",
33
+ "export_overrides_template",
34
+ "infer_column",
35
+ "infer_csv",
36
+ "infer_table",
37
+ "load_profile",
38
+ "load_vocabularies",
39
+ "normalize",
40
+ "requires_review",
41
+ "temporary_profile",
42
+ "temporary_vocabulary",
43
+ "tokenize",
44
+ ]
45
+
46
+ _DEPRECATED_ALIASES = {"normalizar": "normalize", "tokenizar": "tokenize"}
47
+
48
+
49
+ def __getattr__(name: str) -> Any:
50
+ if name in _DEPRECATED_ALIASES:
51
+ replacement = _DEPRECATED_ALIASES[name]
52
+ warnings.warn(
53
+ f"colsemantics.{name} is deprecated and will be removed in 1.0; use colsemantics.{replacement}",
54
+ DeprecationWarning,
55
+ stacklevel=2,
56
+ )
57
+ return globals()[replacement]
58
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
59
+
60
+
61
+ _CONTEXT_MINIMUM_CONFIDENCE = 0.7
62
+ _DOMAIN_MINIMUM_CONFIDENCE = 0.5
63
+ _MAX_HYPOTHESES = 4
64
+
65
+
66
+ def _collect_evidence(
67
+ column_name: str, detected_pattern: str, profile: ContentProfile | None
68
+ ) -> list[Evidence]:
69
+ tokens = tokenize(column_name)
70
+ normalized_name = normalize(column_name)
71
+ evidence_items = (
72
+ by_content_pattern(detected_pattern)
73
+ + by_strong_token(tokens)
74
+ + by_fuzzy(normalized_name, tokens)
75
+ )
76
+ if profile is not None:
77
+ evidence_items += by_gazetteer(profile) + by_structural_signature(profile)
78
+ return evidence_items
79
+
80
+
81
+ def _refine_role(
82
+ role: str | None, domain: str | None, profile: ContentProfile | None
83
+ ) -> str | None:
84
+ if (
85
+ role == config.PERSON_NAME_SEMANTIC
86
+ and domain is not None
87
+ and domain not in config.PERSON_DOMAINS
88
+ ):
89
+ return config.ENTITY_LABEL_SEMANTIC
90
+ if role == config.FREE_FORM_TEXT_SEMANTIC and profile is not None:
91
+ is_dimension_cardinality = (
92
+ 1 < profile.distinct_count <= config.MAX_CATEGORY_CARDINALITY
93
+ and profile.uniqueness_ratio < 0.5
94
+ )
95
+ if is_dimension_cardinality:
96
+ return config.CATEGORY_SEMANTIC
97
+ return role
98
+
99
+
100
+ def _build_result(
101
+ evidence_items: list[Evidence],
102
+ profile: ContentProfile | None = None,
103
+ detected_pattern: str = "None",
104
+ ) -> dict[str, Any]:
105
+ role_ranking = rank(evidence_items, ROLE_AXIS)
106
+ domain_ranking = rank(evidence_items, DOMAIN_AXIS)
107
+ role, role_confidence, role_source, role_conclusive = choose(role_ranking)
108
+ domain, domain_confidence, domain_source, _ = choose(domain_ranking)
109
+ uncertain_domain = domain is not None and domain_confidence < _DOMAIN_MINIMUM_CONFIDENCE
110
+ if uncertain_domain:
111
+ domain, domain_confidence, domain_source = None, 0.0, "No evidence"
112
+ role = _refine_role(role, domain, profile)
113
+ if role in STRUCTURAL_ROLES:
114
+ semantic, raw_confidence, source = role, role_confidence, role_source
115
+ elif domain is not None:
116
+ semantic, raw_confidence, source = domain, domain_confidence, domain_source
117
+ elif role is not None:
118
+ semantic, raw_confidence, source = role, role_confidence, role_source
119
+ else:
120
+ semantic, raw_confidence, source = config.GENERIC_SEMANTIC, 0.0, "Unmatched"
121
+ hypotheses = sorted(
122
+ [
123
+ {
124
+ "semantic": item["category"],
125
+ "axis": axis,
126
+ "confidence": item["confidence"],
127
+ "evidence": item["sources"][:3],
128
+ }
129
+ for axis, ranking in ((ROLE_AXIS, role_ranking), (DOMAIN_AXIS, domain_ranking))
130
+ for item in ranking
131
+ ],
132
+ key=lambda hypothesis: -hypothesis["confidence"],
133
+ )[:_MAX_HYPOTHESES]
134
+ confidence = calibrate(round(raw_confidence, 4), current_context().profile_name)
135
+ return {
136
+ "semantic": semantic,
137
+ "role": role,
138
+ "domain": domain,
139
+ "raw_confidence": round(raw_confidence, 4),
140
+ "confidence": confidence,
141
+ "evidence": source,
142
+ "conclusive": not (bool(role_ranking) and not role_conclusive) and not uncertain_domain,
143
+ "review_required": requires_review(confidence),
144
+ "sensitivity": assess_sensitivity(semantic, detected_pattern),
145
+ "hypotheses": hypotheses,
146
+ }
147
+
148
+
149
+ def infer_column(
150
+ column_name: str, detected_pattern: str = "None", profile: ContentProfile | None = None
151
+ ) -> dict[str, Any]:
152
+ override = current_context().column_overrides.get(column_name)
153
+ if override:
154
+ axis = ROLE_AXIS if override in STRUCTURAL_ROLES else DOMAIN_AXIS
155
+ result = _build_result(
156
+ [Evidence(override, axis, 1.0, "vocabulary override")], profile, detected_pattern
157
+ )
158
+ result["conclusive"] = True
159
+ return result
160
+ return _build_result(
161
+ _collect_evidence(column_name, detected_pattern, profile), profile, detected_pattern
162
+ )
163
+
164
+
165
+ def infer_table(columns: list[dict[str, Any]]) -> list[dict[str, Any]]:
166
+ evidence_by_column: list[list[Evidence]] = []
167
+ results: list[dict[str, Any]] = []
168
+ for column in columns:
169
+ name = str(column.get("column_name", column.get("name")))
170
+ profile = column.get("profile")
171
+ if profile is not None and not isinstance(profile, ContentProfile):
172
+ raise TypeError("profile must be a ContentProfile instance")
173
+ override = current_context().column_overrides.get(name)
174
+ detected_pattern = str(column.get("detected_pattern", column.get("pattern", "None")))
175
+ evidence_items = (
176
+ [
177
+ Evidence(
178
+ override,
179
+ ROLE_AXIS if override in STRUCTURAL_ROLES else DOMAIN_AXIS,
180
+ 1.0,
181
+ "vocabulary override",
182
+ )
183
+ ]
184
+ if override
185
+ else _collect_evidence(name, detected_pattern, profile)
186
+ )
187
+ evidence_by_column.append(evidence_items)
188
+ result = _build_result(evidence_items, profile, detected_pattern)
189
+ if override:
190
+ result["conclusive"] = True
191
+ results.append(result)
192
+ table_domains = _subject_profile(results)
193
+ if not table_domains:
194
+ return results
195
+ for index, (column, result) in enumerate(zip(columns, results, strict=True)):
196
+ if result["conclusive"]:
197
+ continue
198
+ name = str(column.get("column_name", column.get("name")))
199
+ extras = by_table_context(tokenize(name), table_domains)
200
+ if extras:
201
+ detected_pattern = str(column.get("detected_pattern", column.get("pattern", "None")))
202
+ results[index] = _build_result(
203
+ evidence_by_column[index] + extras, column.get("profile"), detected_pattern
204
+ )
205
+ return results
206
+
207
+
208
+ def _subject_profile(results: list[dict[str, Any]]) -> dict[str, float]:
209
+ strengths: dict[str, float] = {}
210
+ for result in results:
211
+ if not result["conclusive"]:
212
+ continue
213
+ for category in (result["role"], result["domain"]):
214
+ if (
215
+ category
216
+ and category != config.GENERIC_SEMANTIC
217
+ and result["raw_confidence"] >= _CONTEXT_MINIMUM_CONFIDENCE
218
+ ):
219
+ strengths[category] = max(strengths.get(category, 0.0), result["raw_confidence"])
220
+ return strengths