colsemantics 0.3.1__tar.gz → 0.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {colsemantics-0.3.1/src/colsemantics.egg-info → colsemantics-0.4.1}/PKG-INFO +43 -5
- {colsemantics-0.3.1 → colsemantics-0.4.1}/README.md +42 -4
- {colsemantics-0.3.1 → colsemantics-0.4.1}/pyproject.toml +5 -4
- {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/__init__.py +127 -66
- colsemantics-0.4.1/src/colsemantics/_data.py +13 -0
- colsemantics-0.4.1/src/colsemantics/_taxonomy.py +776 -0
- {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/benchmark.py +24 -10
- {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/calibration.py +2 -1
- {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/cli.py +23 -3
- colsemantics-0.4.1/src/colsemantics/csv_inference.py +155 -0
- colsemantics-0.4.1/src/colsemantics/data/SOURCES.md +18 -0
- colsemantics-0.4.1/src/colsemantics/data/br_first_names.txt +370 -0
- colsemantics-0.4.1/src/colsemantics/data/br_municipalities.txt +5290 -0
- colsemantics-0.4.1/src/colsemantics/data/countries.txt +373 -0
- colsemantics-0.4.1/src/colsemantics/data/modifiers.txt +763 -0
- colsemantics-0.4.1/src/colsemantics/data/us_cities.txt +2947 -0
- colsemantics-0.4.1/src/colsemantics/data/us_first_names.txt +1500 -0
- {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/detectors.py +96 -23
- colsemantics-0.4.1/src/colsemantics/patterns.py +430 -0
- {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/profiles.py +8 -3
- colsemantics-0.4.1/src/colsemantics/profiling.py +126 -0
- colsemantics-0.4.1/src/colsemantics/results.py +69 -0
- colsemantics-0.4.1/src/colsemantics/sensitivity.py +44 -0
- colsemantics-0.4.1/src/colsemantics/tokens.py +125 -0
- {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/vocabulary.py +77 -0
- {colsemantics-0.3.1 → colsemantics-0.4.1/src/colsemantics.egg-info}/PKG-INFO +43 -5
- {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics.egg-info/SOURCES.txt +18 -1
- {colsemantics-0.3.1 → colsemantics-0.4.1}/tests/test_colsemantics.py +7 -1
- colsemantics-0.4.1/tests/test_csv_inference.py +209 -0
- colsemantics-0.4.1/tests/test_pattern_results.py +135 -0
- colsemantics-0.4.1/tests/test_patterns.py +217 -0
- colsemantics-0.4.1/tests/test_profiling.py +87 -0
- colsemantics-0.4.1/tests/test_public_vocabulary.py +113 -0
- colsemantics-0.4.1/tests/test_validation_corpus.py +80 -0
- colsemantics-0.4.1/tests/test_vocabularies.py +110 -0
- colsemantics-0.3.1/src/colsemantics/_taxonomy.py +0 -378
- colsemantics-0.3.1/src/colsemantics/csv_inference.py +0 -69
- colsemantics-0.3.1/src/colsemantics/sensitivity.py +0 -30
- colsemantics-0.3.1/src/colsemantics/tokens.py +0 -80
- colsemantics-0.3.1/tests/test_csv_inference.py +0 -49
- {colsemantics-0.3.1 → colsemantics-0.4.1}/LICENSE +0 -0
- {colsemantics-0.3.1 → colsemantics-0.4.1}/setup.cfg +0 -0
- {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/calibration.json +0 -0
- {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/context.py +0 -0
- {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/evidence.py +0 -0
- {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/py.typed +0 -0
- {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/vocabularies.py +0 -0
- {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics.egg-info/dependency_links.txt +0 -0
- {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics.egg-info/entry_points.txt +0 -0
- {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics.egg-info/requires.txt +0 -0
- {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics.egg-info/top_level.txt +0 -0
- {colsemantics-0.3.1 → colsemantics-0.4.1}/tests/test_benchmark.py +0 -0
- {colsemantics-0.3.1 → colsemantics-0.4.1}/tests/test_calibration.py +0 -0
- {colsemantics-0.3.1 → colsemantics-0.4.1}/tests/test_cli.py +0 -0
- {colsemantics-0.3.1 → colsemantics-0.4.1}/tests/test_english_core.py +0 -0
- {colsemantics-0.3.1 → colsemantics-0.4.1}/tests/test_profiles.py +0 -0
- {colsemantics-0.3.1 → colsemantics-0.4.1}/tests/test_sensitivity.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: colsemantics
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.1
|
|
4
4
|
Summary: Infer semantic roles and domains for tabular columns from names, values, and table context.
|
|
5
5
|
Author: Caio
|
|
6
6
|
License-Expression: MIT
|
|
@@ -104,7 +104,7 @@ results = infer_table(
|
|
|
104
104
|
|
|
105
105
|
## Vocabulary profiles and extensions
|
|
106
106
|
|
|
107
|
-
`pt-BR` is the embedded profile. Use `available_profiles()` to list embedded profiles and `load_profile()` when an application needs an explicit context.
|
|
107
|
+
`pt-BR` is the embedded (and default) profile. A profile controls the column-name vocabulary only; value patterns such as e-mail, phone, and postal code are detected independently of it. Use `available_profiles()` to list embedded profiles and `load_profile()` when an application needs an explicit context.
|
|
108
108
|
|
|
109
109
|
```python
|
|
110
110
|
from colsemantics import infer_column, temporary_profile
|
|
@@ -141,16 +141,53 @@ The report includes semantic, role, and domain accuracy, coverage, per-profile m
|
|
|
141
141
|
|
|
142
142
|
`raw_confidence` is the direct score assembled from independent evidence. `confidence` is the profile calibration for that score bucket, derived from the versioned calibration fixture. A result sets `review_required` when calibrated confidence is below `0.80`.
|
|
143
143
|
|
|
144
|
+
See `benchmarks/README.md` for the held-out validation set; its first measurement was 71% semantic accuracy on 52 hand-written cases (Brazilian and US column names), which shows the engine is not yet production-grade for English column names.
|
|
145
|
+
|
|
144
146
|
The embedded calibration is provisional: it is a controlled regression fixture, not evidence of production-grade accuracy. A held-out, versioned corpus is required before treating its precision as a production measurement.
|
|
145
147
|
|
|
148
|
+
## Vocabulary and data sources
|
|
149
|
+
|
|
150
|
+
Beyond the hand-written terms, the vocabulary was extended from public data: column names of about 4,500 US open datasets (Socrata catalog) and 500 Brazilian CKAN resources, and word lists from IBGE (municipalities, countries, first names), the US SSA (first names), and `datasets/world-cities`. Only derived word lists are shipped. Sources, licenses, and the scripts that rebuild them are in `src/colsemantics/data/SOURCES.md` and `scripts/`.
|
|
151
|
+
|
|
152
|
+
With sampled values, columns of municipalities, US cities, countries, and first names (Brazilian or US) are recognized even when the column name says nothing. These lists are matched by exact (accent-insensitive) value, so misspelled or abbreviated values do not match.
|
|
153
|
+
|
|
154
|
+
## Value patterns and locales
|
|
155
|
+
|
|
156
|
+
When a column has sampled values, colsemantics also looks at how the values are written. Detection uses values only, so it works the same for Brazilian and US data and does not depend on the vocabulary profile.
|
|
157
|
+
|
|
158
|
+
| Type | Brazil (BR) | United States (US) |
|
|
159
|
+
|---|---|---|
|
|
160
|
+
| `phone` | `(11) 91234-5678`, `11 91234-5678`, `+55 11 91234-5678`, `(21) 3456-7890`, `0800 123 4567`; the area code (DDD) must exist, mobiles start with 9, landlines with 2-5 | `(415) 555-2671`, `415-555-2671`, `415.555.2671`, `+1 415 555 2671`; area code and exchange start with 2-9 and are not N11 |
|
|
161
|
+
| `postal_code` | CEP `01310-100` | `94105-1234` (ZIP+4) |
|
|
162
|
+
| `address` | `Rua das Flores, 123`, `Av. Paulista, 1578`, `Rodovia BR-101 km 5` | `123 Main St`, `500 N. Michigan Ave., Suite 300`, `PO Box 1234` |
|
|
163
|
+
| national id | `cpf`, `cnpj` (check digits verified) | `ssn` (area/group/serial ranges verified), `ein` (format only) |
|
|
164
|
+
| any | `email`, `uuid` | |
|
|
165
|
+
|
|
166
|
+
Bare digit strings such as `4155552671` or `11912345678`, and plain five-digit ZIP codes, are not reported as phones or postal codes because they are indistinguishable from ids and amounts. Addresses are matched heuristically and are never marked `validated`.
|
|
167
|
+
|
|
168
|
+
The result carries a `pattern` that says what was found and where it looks like it comes from:
|
|
169
|
+
|
|
170
|
+
```python
|
|
171
|
+
result = infer_column("f6", profile=profile_values(["(11) 91234-5678", "(21) 99876-5432"]))
|
|
172
|
+
result["pattern"]
|
|
173
|
+
# {"type": "phone", "locale": "BR", "variant": "mobile", "format": "(DD) 9NNNN-NNNN",
|
|
174
|
+
# "coverage": 1.0, "validated": True,
|
|
175
|
+
# "message": "100% of sampled values look like a Brazilian mobile phone number (e.g. (DD) 9NNNN-NNNN)"}
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
`pattern` is `None` when no format reaches its coverage threshold (80% of the sampled values, 60% for addresses). The same text is used as the `evidence` of the classification. You can still pass `detected_pattern="phone"` to `infer_column`; legacy names (`CPF`, `E-mail`, `Telefone`, `CEP`) keep working.
|
|
179
|
+
|
|
180
|
+
`infer_csv` reads each file with type detection (numbers, dates, booleans in Brazilian and US formats), picks the `,`, `;`, tab, or `|` delimiter from the header, samples rows uniformly with a seed, and reports duplicate headers and malformed rows in `warnings`.
|
|
181
|
+
|
|
146
182
|
## Sensitive-data recommendations
|
|
147
183
|
|
|
148
184
|
Each result includes a descriptive `sensitivity` recommendation. It does not modify input data.
|
|
149
185
|
|
|
150
|
-
- Validated CPF patterns: `high` / `mask`
|
|
186
|
+
- Validated CPF or SSN patterns: `high` / `mask`
|
|
187
|
+
- Detected street addresses: `high` / `mask`
|
|
151
188
|
- Person identity semantics: `high` / `restrict`
|
|
152
|
-
- Contact semantics: `high` / `mask`
|
|
153
|
-
-
|
|
189
|
+
- Contact semantics (e-mail, phone): `high` / `mask`
|
|
190
|
+
- CNPJ, EIN, postal codes, other identifiers: `medium` / `review`
|
|
154
191
|
- Everything else: `none` / `allow`
|
|
155
192
|
|
|
156
193
|
## Result shape
|
|
@@ -166,6 +203,7 @@ Each result includes a descriptive `sensitivity` recommendation. It does not mod
|
|
|
166
203
|
"conclusive": bool,
|
|
167
204
|
"review_required": bool,
|
|
168
205
|
"sensitivity": {"level": str, "action": str, "reason": str},
|
|
206
|
+
"pattern": dict | None, # see "Value patterns and locales"
|
|
169
207
|
"hypotheses": list[dict],
|
|
170
208
|
}
|
|
171
209
|
```
|
|
@@ -71,7 +71,7 @@ results = infer_table(
|
|
|
71
71
|
|
|
72
72
|
## Vocabulary profiles and extensions
|
|
73
73
|
|
|
74
|
-
`pt-BR` is the embedded profile. Use `available_profiles()` to list embedded profiles and `load_profile()` when an application needs an explicit context.
|
|
74
|
+
`pt-BR` is the embedded (and default) profile. A profile controls the column-name vocabulary only; value patterns such as e-mail, phone, and postal code are detected independently of it. Use `available_profiles()` to list embedded profiles and `load_profile()` when an application needs an explicit context.
|
|
75
75
|
|
|
76
76
|
```python
|
|
77
77
|
from colsemantics import infer_column, temporary_profile
|
|
@@ -108,16 +108,53 @@ The report includes semantic, role, and domain accuracy, coverage, per-profile m
|
|
|
108
108
|
|
|
109
109
|
`raw_confidence` is the direct score assembled from independent evidence. `confidence` is the profile calibration for that score bucket, derived from the versioned calibration fixture. A result sets `review_required` when calibrated confidence is below `0.80`.
|
|
110
110
|
|
|
111
|
+
See `benchmarks/README.md` for the held-out validation set; its first measurement was 71% semantic accuracy on 52 hand-written cases (Brazilian and US column names), which shows the engine is not yet production-grade for English column names.
|
|
112
|
+
|
|
111
113
|
The embedded calibration is provisional: it is a controlled regression fixture, not evidence of production-grade accuracy. A held-out, versioned corpus is required before treating its precision as a production measurement.
|
|
112
114
|
|
|
115
|
+
## Vocabulary and data sources
|
|
116
|
+
|
|
117
|
+
Beyond the hand-written terms, the vocabulary was extended from public data: column names of about 4,500 US open datasets (Socrata catalog) and 500 Brazilian CKAN resources, and word lists from IBGE (municipalities, countries, first names), the US SSA (first names), and `datasets/world-cities`. Only derived word lists are shipped. Sources, licenses, and the scripts that rebuild them are in `src/colsemantics/data/SOURCES.md` and `scripts/`.
|
|
118
|
+
|
|
119
|
+
With sampled values, columns of municipalities, US cities, countries, and first names (Brazilian or US) are recognized even when the column name says nothing. These lists are matched by exact (accent-insensitive) value, so misspelled or abbreviated values do not match.
|
|
120
|
+
|
|
121
|
+
## Value patterns and locales
|
|
122
|
+
|
|
123
|
+
When a column has sampled values, colsemantics also looks at how the values are written. Detection uses values only, so it works the same for Brazilian and US data and does not depend on the vocabulary profile.
|
|
124
|
+
|
|
125
|
+
| Type | Brazil (BR) | United States (US) |
|
|
126
|
+
|---|---|---|
|
|
127
|
+
| `phone` | `(11) 91234-5678`, `11 91234-5678`, `+55 11 91234-5678`, `(21) 3456-7890`, `0800 123 4567`; the area code (DDD) must exist, mobiles start with 9, landlines with 2-5 | `(415) 555-2671`, `415-555-2671`, `415.555.2671`, `+1 415 555 2671`; area code and exchange start with 2-9 and are not N11 |
|
|
128
|
+
| `postal_code` | CEP `01310-100` | `94105-1234` (ZIP+4) |
|
|
129
|
+
| `address` | `Rua das Flores, 123`, `Av. Paulista, 1578`, `Rodovia BR-101 km 5` | `123 Main St`, `500 N. Michigan Ave., Suite 300`, `PO Box 1234` |
|
|
130
|
+
| national id | `cpf`, `cnpj` (check digits verified) | `ssn` (area/group/serial ranges verified), `ein` (format only) |
|
|
131
|
+
| any | `email`, `uuid` | |
|
|
132
|
+
|
|
133
|
+
Bare digit strings such as `4155552671` or `11912345678`, and plain five-digit ZIP codes, are not reported as phones or postal codes because they are indistinguishable from ids and amounts. Addresses are matched heuristically and are never marked `validated`.
|
|
134
|
+
|
|
135
|
+
The result carries a `pattern` that says what was found and where it looks like it comes from:
|
|
136
|
+
|
|
137
|
+
```python
|
|
138
|
+
result = infer_column("f6", profile=profile_values(["(11) 91234-5678", "(21) 99876-5432"]))
|
|
139
|
+
result["pattern"]
|
|
140
|
+
# {"type": "phone", "locale": "BR", "variant": "mobile", "format": "(DD) 9NNNN-NNNN",
|
|
141
|
+
# "coverage": 1.0, "validated": True,
|
|
142
|
+
# "message": "100% of sampled values look like a Brazilian mobile phone number (e.g. (DD) 9NNNN-NNNN)"}
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
`pattern` is `None` when no format reaches its coverage threshold (80% of the sampled values, 60% for addresses). The same text is used as the `evidence` of the classification. You can still pass `detected_pattern="phone"` to `infer_column`; legacy names (`CPF`, `E-mail`, `Telefone`, `CEP`) keep working.
|
|
146
|
+
|
|
147
|
+
`infer_csv` reads each file with type detection (numbers, dates, booleans in Brazilian and US formats), picks the `,`, `;`, tab, or `|` delimiter from the header, samples rows uniformly with a seed, and reports duplicate headers and malformed rows in `warnings`.
|
|
148
|
+
|
|
113
149
|
## Sensitive-data recommendations
|
|
114
150
|
|
|
115
151
|
Each result includes a descriptive `sensitivity` recommendation. It does not modify input data.
|
|
116
152
|
|
|
117
|
-
- Validated CPF patterns: `high` / `mask`
|
|
153
|
+
- Validated CPF or SSN patterns: `high` / `mask`
|
|
154
|
+
- Detected street addresses: `high` / `mask`
|
|
118
155
|
- Person identity semantics: `high` / `restrict`
|
|
119
|
-
- Contact semantics: `high` / `mask`
|
|
120
|
-
-
|
|
156
|
+
- Contact semantics (e-mail, phone): `high` / `mask`
|
|
157
|
+
- CNPJ, EIN, postal codes, other identifiers: `medium` / `review`
|
|
121
158
|
- Everything else: `none` / `allow`
|
|
122
159
|
|
|
123
160
|
## Result shape
|
|
@@ -133,6 +170,7 @@ Each result includes a descriptive `sensitivity` recommendation. It does not mod
|
|
|
133
170
|
"conclusive": bool,
|
|
134
171
|
"review_required": bool,
|
|
135
172
|
"sensitivity": {"level": str, "action": str, "reason": str},
|
|
173
|
+
"pattern": dict | None, # see "Value patterns and locales"
|
|
136
174
|
"hypotheses": list[dict],
|
|
137
175
|
}
|
|
138
176
|
```
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "colsemantics"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.4.1"
|
|
4
4
|
description = "Infer semantic roles and domains for tabular columns from names, values, and table context."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
@@ -37,14 +37,14 @@ colsemantics = "colsemantics.cli:main"
|
|
|
37
37
|
dev = ["pytest>=9.2.0", "pytest-cov>=7.1.0", "mypy>=2.4.1", "ruff>=0.17.4"]
|
|
38
38
|
|
|
39
39
|
[build-system]
|
|
40
|
-
requires = ["setuptools>=
|
|
40
|
+
requires = ["setuptools>=84.2.1"]
|
|
41
41
|
build-backend = "setuptools.build_meta"
|
|
42
42
|
|
|
43
43
|
[tool.setuptools.packages.find]
|
|
44
44
|
where = ["src"]
|
|
45
45
|
|
|
46
46
|
[tool.setuptools.package-data]
|
|
47
|
-
colsemantics = ["py.typed", "calibration.json"]
|
|
47
|
+
colsemantics = ["py.typed", "calibration.json", "data/*.txt", "data/*.md"]
|
|
48
48
|
|
|
49
49
|
[tool.pytest.ini_options]
|
|
50
50
|
testpaths = ["tests"]
|
|
@@ -67,12 +67,13 @@ python_version = "3.11"
|
|
|
67
67
|
files = ["src/colsemantics"]
|
|
68
68
|
ignore_missing_imports = true
|
|
69
69
|
warn_unused_ignores = true
|
|
70
|
+
strict = true
|
|
70
71
|
|
|
71
72
|
[tool.coverage.run]
|
|
72
73
|
source = ["colsemantics"]
|
|
73
74
|
branch = true
|
|
74
75
|
|
|
75
76
|
[tool.coverage.report]
|
|
76
|
-
fail_under =
|
|
77
|
+
fail_under = 90
|
|
77
78
|
show_missing = true
|
|
78
79
|
skip_covered = true
|
|
@@ -16,16 +16,31 @@ from .detectors import (
|
|
|
16
16
|
by_table_context,
|
|
17
17
|
)
|
|
18
18
|
from .evidence import DOMAIN_AXIS, ROLE_AXIS, Evidence, choose, rank
|
|
19
|
+
from .patterns import NO_PATTERN, PATTERN_LABELS, analyze_values, canonical_pattern
|
|
19
20
|
from .profiles import available_profiles, load_profile, temporary_profile
|
|
21
|
+
from .results import (
|
|
22
|
+
CsvColumnResult,
|
|
23
|
+
CsvReport,
|
|
24
|
+
Hypothesis,
|
|
25
|
+
InferenceResult,
|
|
26
|
+
PatternInfo,
|
|
27
|
+
Sensitivity,
|
|
28
|
+
)
|
|
20
29
|
from .sensitivity import assess_sensitivity
|
|
21
30
|
from .tokens import normalize, tokenize
|
|
22
31
|
from .vocabularies import export_overrides_template, load_vocabularies, temporary_vocabulary
|
|
23
32
|
|
|
24
|
-
__version__ = "0.
|
|
33
|
+
__version__ = "0.4.1"
|
|
25
34
|
|
|
26
35
|
__all__ = [
|
|
27
36
|
"ContentProfile",
|
|
37
|
+
"CsvColumnResult",
|
|
38
|
+
"CsvReport",
|
|
39
|
+
"Hypothesis",
|
|
40
|
+
"InferenceResult",
|
|
41
|
+
"PatternInfo",
|
|
28
42
|
"SemanticContext",
|
|
43
|
+
"Sensitivity",
|
|
29
44
|
"assess_sensitivity",
|
|
30
45
|
"available_profiles",
|
|
31
46
|
"calibrate",
|
|
@@ -61,18 +76,47 @@ def __getattr__(name: str) -> Any:
|
|
|
61
76
|
_CONTEXT_MINIMUM_CONFIDENCE = 0.7
|
|
62
77
|
_DOMAIN_MINIMUM_CONFIDENCE = 0.5
|
|
63
78
|
_MAX_HYPOTHESES = 4
|
|
79
|
+
# A strong domain (for example an address pattern in the values) beats a weak structural
|
|
80
|
+
# role guessed from an uninformative column name.
|
|
81
|
+
_CONTENT_DOMAIN_CONFIDENCE = 0.85
|
|
82
|
+
_CONTENT_DOMAIN_MARGIN = 0.3
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _resolve_pattern(detected_pattern: str, profile: ContentProfile | None) -> PatternInfo | None:
|
|
86
|
+
"""Combine a caller-reported pattern with what the sampled values actually look like."""
|
|
87
|
+
reported = canonical_pattern(detected_pattern)
|
|
88
|
+
expected = None if reported == NO_PATTERN else reported
|
|
89
|
+
found = (
|
|
90
|
+
analyze_values(profile.distinct_values, expected_type=expected)
|
|
91
|
+
if profile is not None and profile.distinct_values
|
|
92
|
+
else None
|
|
93
|
+
)
|
|
94
|
+
if found is not None or expected is None:
|
|
95
|
+
return found
|
|
96
|
+
label = PATTERN_LABELS.get(expected, expected)
|
|
97
|
+
return {
|
|
98
|
+
"type": expected,
|
|
99
|
+
"locale": None,
|
|
100
|
+
"variant": None,
|
|
101
|
+
"format": None,
|
|
102
|
+
"coverage": None,
|
|
103
|
+
"validated": True,
|
|
104
|
+
"message": f"validated {label} pattern",
|
|
105
|
+
}
|
|
64
106
|
|
|
65
107
|
|
|
66
108
|
def _collect_evidence(
|
|
67
|
-
column_name: str,
|
|
109
|
+
column_name: str, pattern: PatternInfo | None, profile: ContentProfile | None
|
|
68
110
|
) -> list[Evidence]:
|
|
69
111
|
tokens = tokenize(column_name)
|
|
70
112
|
normalized_name = normalize(column_name)
|
|
71
113
|
evidence_items = (
|
|
72
|
-
by_content_pattern(
|
|
73
|
-
+ by_strong_token(tokens)
|
|
74
|
-
+ by_fuzzy(normalized_name, tokens)
|
|
114
|
+
by_content_pattern(pattern) + by_strong_token(tokens) + by_fuzzy(normalized_name, tokens)
|
|
75
115
|
)
|
|
116
|
+
if column_name.rstrip().endswith("?"):
|
|
117
|
+
evidence_items.append(
|
|
118
|
+
Evidence("Status / Indicator / Flag", ROLE_AXIS, 0.8, "question-style column name")
|
|
119
|
+
)
|
|
76
120
|
if profile is not None:
|
|
77
121
|
evidence_items += by_gazetteer(profile) + by_structural_signature(profile)
|
|
78
122
|
return evidence_items
|
|
@@ -100,8 +144,8 @@ def _refine_role(
|
|
|
100
144
|
def _build_result(
|
|
101
145
|
evidence_items: list[Evidence],
|
|
102
146
|
profile: ContentProfile | None = None,
|
|
103
|
-
|
|
104
|
-
) ->
|
|
147
|
+
pattern: PatternInfo | None = None,
|
|
148
|
+
) -> InferenceResult:
|
|
105
149
|
role_ranking = rank(evidence_items, ROLE_AXIS)
|
|
106
150
|
domain_ranking = rank(evidence_items, DOMAIN_AXIS)
|
|
107
151
|
role, role_confidence, role_source, role_conclusive = choose(role_ranking)
|
|
@@ -110,7 +154,15 @@ def _build_result(
|
|
|
110
154
|
if uncertain_domain:
|
|
111
155
|
domain, domain_confidence, domain_source = None, 0.0, "No evidence"
|
|
112
156
|
role = _refine_role(role, domain, profile)
|
|
113
|
-
|
|
157
|
+
geometry_found = pattern is not None and pattern["type"] in ("geometry", "coordinates")
|
|
158
|
+
content_domain_wins = domain is not None and (
|
|
159
|
+
geometry_found
|
|
160
|
+
or (
|
|
161
|
+
domain_confidence >= _CONTENT_DOMAIN_CONFIDENCE
|
|
162
|
+
and role_confidence < domain_confidence - _CONTENT_DOMAIN_MARGIN
|
|
163
|
+
)
|
|
164
|
+
)
|
|
165
|
+
if role in STRUCTURAL_ROLES and not content_domain_wins:
|
|
114
166
|
semantic, raw_confidence, source = role, role_confidence, role_source
|
|
115
167
|
elif domain is not None:
|
|
116
168
|
semantic, raw_confidence, source = domain, domain_confidence, domain_source
|
|
@@ -118,19 +170,19 @@ def _build_result(
|
|
|
118
170
|
semantic, raw_confidence, source = role, role_confidence, role_source
|
|
119
171
|
else:
|
|
120
172
|
semantic, raw_confidence, source = config.GENERIC_SEMANTIC, 0.0, "Unmatched"
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
173
|
+
candidates: list[Hypothesis] = [
|
|
174
|
+
{
|
|
175
|
+
"semantic": item["category"],
|
|
176
|
+
"axis": axis,
|
|
177
|
+
"confidence": item["confidence"],
|
|
178
|
+
"evidence": item["sources"][:3],
|
|
179
|
+
}
|
|
180
|
+
for axis, ranking in ((ROLE_AXIS, role_ranking), (DOMAIN_AXIS, domain_ranking))
|
|
181
|
+
for item in ranking
|
|
182
|
+
]
|
|
183
|
+
hypotheses = sorted(candidates, key=lambda hypothesis: -hypothesis["confidence"])[
|
|
184
|
+
:_MAX_HYPOTHESES
|
|
185
|
+
]
|
|
134
186
|
confidence = calibrate(round(raw_confidence, 4), current_context().profile_name)
|
|
135
187
|
return {
|
|
136
188
|
"semantic": semantic,
|
|
@@ -141,71 +193,80 @@ def _build_result(
|
|
|
141
193
|
"evidence": source,
|
|
142
194
|
"conclusive": not (bool(role_ranking) and not role_conclusive) and not uncertain_domain,
|
|
143
195
|
"review_required": requires_review(confidence),
|
|
144
|
-
"sensitivity": assess_sensitivity(semantic,
|
|
196
|
+
"sensitivity": assess_sensitivity(semantic, pattern["type"] if pattern else NO_PATTERN),
|
|
197
|
+
"pattern": pattern,
|
|
145
198
|
"hypotheses": hypotheses,
|
|
146
199
|
}
|
|
147
200
|
|
|
148
201
|
|
|
149
|
-
def
|
|
150
|
-
|
|
151
|
-
) ->
|
|
152
|
-
override
|
|
202
|
+
def _column_evidence(
|
|
203
|
+
name: str, pattern: PatternInfo | None, profile: ContentProfile | None
|
|
204
|
+
) -> tuple[list[Evidence], bool]:
|
|
205
|
+
"""Return the evidence for a column and whether a vocabulary override decided it."""
|
|
206
|
+
override = current_context().column_overrides.get(name)
|
|
153
207
|
if override:
|
|
154
208
|
axis = ROLE_AXIS if override in STRUCTURAL_ROLES else DOMAIN_AXIS
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
209
|
+
return [Evidence(override, axis, 1.0, "vocabulary override")], True
|
|
210
|
+
return _collect_evidence(name, pattern, profile), False
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _infer_one(
|
|
214
|
+
name: str, detected_pattern: str, profile: ContentProfile | None
|
|
215
|
+
) -> tuple[InferenceResult, list[Evidence]]:
|
|
216
|
+
pattern = _resolve_pattern(detected_pattern, profile)
|
|
217
|
+
evidence_items, overridden = _column_evidence(name, pattern, profile)
|
|
218
|
+
result = _build_result(evidence_items, profile, pattern)
|
|
219
|
+
if overridden:
|
|
158
220
|
result["conclusive"] = True
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
221
|
+
return result, evidence_items
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def infer_column(
|
|
225
|
+
column_name: str, detected_pattern: str = "None", profile: ContentProfile | None = None
|
|
226
|
+
) -> InferenceResult:
|
|
227
|
+
return _infer_one(column_name, detected_pattern, profile)[0]
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _column_name(column: dict[str, Any], position: int) -> str:
|
|
231
|
+
name = column.get("column_name", column.get("name"))
|
|
232
|
+
if not isinstance(name, str) or not name.strip():
|
|
233
|
+
raise ValueError(f"Column at position {position} requires a non-empty 'column_name'.")
|
|
234
|
+
return name
|
|
163
235
|
|
|
164
236
|
|
|
165
|
-
def
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
override,
|
|
179
|
-
ROLE_AXIS if override in STRUCTURAL_ROLES else DOMAIN_AXIS,
|
|
180
|
-
1.0,
|
|
181
|
-
"vocabulary override",
|
|
182
|
-
)
|
|
183
|
-
]
|
|
184
|
-
if override
|
|
185
|
-
else _collect_evidence(name, detected_pattern, profile)
|
|
237
|
+
def _column_profile(column: dict[str, Any]) -> ContentProfile | None:
|
|
238
|
+
profile = column.get("profile")
|
|
239
|
+
if profile is not None and not isinstance(profile, ContentProfile):
|
|
240
|
+
raise TypeError("profile must be a ContentProfile instance")
|
|
241
|
+
return profile
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def infer_table(columns: list[dict[str, Any]]) -> list[InferenceResult]:
|
|
245
|
+
parsed = [
|
|
246
|
+
(
|
|
247
|
+
_column_name(column, position),
|
|
248
|
+
str(column.get("detected_pattern", column.get("pattern", NO_PATTERN))),
|
|
249
|
+
_column_profile(column),
|
|
186
250
|
)
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
results.append(result)
|
|
251
|
+
for position, column in enumerate(columns)
|
|
252
|
+
]
|
|
253
|
+
first_pass = [_infer_one(name, pattern, profile) for name, pattern, profile in parsed]
|
|
254
|
+
results = [result for result, _ in first_pass]
|
|
192
255
|
table_domains = _subject_profile(results)
|
|
193
256
|
if not table_domains:
|
|
194
257
|
return results
|
|
195
|
-
for index, (
|
|
258
|
+
for index, ((name, _pattern, profile), (result, evidence_items)) in enumerate(
|
|
259
|
+
zip(parsed, first_pass, strict=True)
|
|
260
|
+
):
|
|
196
261
|
if result["conclusive"]:
|
|
197
262
|
continue
|
|
198
|
-
name = str(column.get("column_name", column.get("name")))
|
|
199
263
|
extras = by_table_context(tokenize(name), table_domains)
|
|
200
264
|
if extras:
|
|
201
|
-
|
|
202
|
-
results[index] = _build_result(
|
|
203
|
-
evidence_by_column[index] + extras, column.get("profile"), detected_pattern
|
|
204
|
-
)
|
|
265
|
+
results[index] = _build_result(evidence_items + extras, profile, result["pattern"])
|
|
205
266
|
return results
|
|
206
267
|
|
|
207
268
|
|
|
208
|
-
def _subject_profile(results: list[
|
|
269
|
+
def _subject_profile(results: list[InferenceResult]) -> dict[str, float]:
|
|
209
270
|
strengths: dict[str, float] = {}
|
|
210
271
|
for result in results:
|
|
211
272
|
if not result["conclusive"]:
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"""Load the word lists shipped in ``colsemantics/data``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from functools import cache
|
|
6
|
+
from importlib import resources
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@cache
|
|
10
|
+
def load_wordlist(name: str) -> frozenset[str]:
|
|
11
|
+
"""Return the lines of ``data/<name>`` (already normalized: lowercase, no accents)."""
|
|
12
|
+
text = resources.files("colsemantics").joinpath("data", name).read_text(encoding="utf-8")
|
|
13
|
+
return frozenset(line for line in text.splitlines() if line)
|