colsemantics 0.3.1__tar.gz → 0.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. {colsemantics-0.3.1/src/colsemantics.egg-info → colsemantics-0.4.1}/PKG-INFO +43 -5
  2. {colsemantics-0.3.1 → colsemantics-0.4.1}/README.md +42 -4
  3. {colsemantics-0.3.1 → colsemantics-0.4.1}/pyproject.toml +5 -4
  4. {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/__init__.py +127 -66
  5. colsemantics-0.4.1/src/colsemantics/_data.py +13 -0
  6. colsemantics-0.4.1/src/colsemantics/_taxonomy.py +776 -0
  7. {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/benchmark.py +24 -10
  8. {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/calibration.py +2 -1
  9. {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/cli.py +23 -3
  10. colsemantics-0.4.1/src/colsemantics/csv_inference.py +155 -0
  11. colsemantics-0.4.1/src/colsemantics/data/SOURCES.md +18 -0
  12. colsemantics-0.4.1/src/colsemantics/data/br_first_names.txt +370 -0
  13. colsemantics-0.4.1/src/colsemantics/data/br_municipalities.txt +5290 -0
  14. colsemantics-0.4.1/src/colsemantics/data/countries.txt +373 -0
  15. colsemantics-0.4.1/src/colsemantics/data/modifiers.txt +763 -0
  16. colsemantics-0.4.1/src/colsemantics/data/us_cities.txt +2947 -0
  17. colsemantics-0.4.1/src/colsemantics/data/us_first_names.txt +1500 -0
  18. {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/detectors.py +96 -23
  19. colsemantics-0.4.1/src/colsemantics/patterns.py +430 -0
  20. {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/profiles.py +8 -3
  21. colsemantics-0.4.1/src/colsemantics/profiling.py +126 -0
  22. colsemantics-0.4.1/src/colsemantics/results.py +69 -0
  23. colsemantics-0.4.1/src/colsemantics/sensitivity.py +44 -0
  24. colsemantics-0.4.1/src/colsemantics/tokens.py +125 -0
  25. {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/vocabulary.py +77 -0
  26. {colsemantics-0.3.1 → colsemantics-0.4.1/src/colsemantics.egg-info}/PKG-INFO +43 -5
  27. {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics.egg-info/SOURCES.txt +18 -1
  28. {colsemantics-0.3.1 → colsemantics-0.4.1}/tests/test_colsemantics.py +7 -1
  29. colsemantics-0.4.1/tests/test_csv_inference.py +209 -0
  30. colsemantics-0.4.1/tests/test_pattern_results.py +135 -0
  31. colsemantics-0.4.1/tests/test_patterns.py +217 -0
  32. colsemantics-0.4.1/tests/test_profiling.py +87 -0
  33. colsemantics-0.4.1/tests/test_public_vocabulary.py +113 -0
  34. colsemantics-0.4.1/tests/test_validation_corpus.py +80 -0
  35. colsemantics-0.4.1/tests/test_vocabularies.py +110 -0
  36. colsemantics-0.3.1/src/colsemantics/_taxonomy.py +0 -378
  37. colsemantics-0.3.1/src/colsemantics/csv_inference.py +0 -69
  38. colsemantics-0.3.1/src/colsemantics/sensitivity.py +0 -30
  39. colsemantics-0.3.1/src/colsemantics/tokens.py +0 -80
  40. colsemantics-0.3.1/tests/test_csv_inference.py +0 -49
  41. {colsemantics-0.3.1 → colsemantics-0.4.1}/LICENSE +0 -0
  42. {colsemantics-0.3.1 → colsemantics-0.4.1}/setup.cfg +0 -0
  43. {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/calibration.json +0 -0
  44. {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/context.py +0 -0
  45. {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/evidence.py +0 -0
  46. {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/py.typed +0 -0
  47. {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics/vocabularies.py +0 -0
  48. {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics.egg-info/dependency_links.txt +0 -0
  49. {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics.egg-info/entry_points.txt +0 -0
  50. {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics.egg-info/requires.txt +0 -0
  51. {colsemantics-0.3.1 → colsemantics-0.4.1}/src/colsemantics.egg-info/top_level.txt +0 -0
  52. {colsemantics-0.3.1 → colsemantics-0.4.1}/tests/test_benchmark.py +0 -0
  53. {colsemantics-0.3.1 → colsemantics-0.4.1}/tests/test_calibration.py +0 -0
  54. {colsemantics-0.3.1 → colsemantics-0.4.1}/tests/test_cli.py +0 -0
  55. {colsemantics-0.3.1 → colsemantics-0.4.1}/tests/test_english_core.py +0 -0
  56. {colsemantics-0.3.1 → colsemantics-0.4.1}/tests/test_profiles.py +0 -0
  57. {colsemantics-0.3.1 → colsemantics-0.4.1}/tests/test_sensitivity.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: colsemantics
3
- Version: 0.3.1
3
+ Version: 0.4.1
4
4
  Summary: Infer semantic roles and domains for tabular columns from names, values, and table context.
5
5
  Author: Caio
6
6
  License-Expression: MIT
@@ -104,7 +104,7 @@ results = infer_table(
104
104
 
105
105
  ## Vocabulary profiles and extensions
106
106
 
107
- `pt-BR` is the embedded profile. Use `available_profiles()` to list embedded profiles and `load_profile()` when an application needs an explicit context.
107
+ `pt-BR` is the embedded (and default) profile. A profile controls the column-name vocabulary only; value patterns such as e-mail, phone, and postal code are detected independently of it. Use `available_profiles()` to list embedded profiles and `load_profile()` when an application needs an explicit context.
108
108
 
109
109
  ```python
110
110
  from colsemantics import infer_column, temporary_profile
@@ -141,16 +141,53 @@ The report includes semantic, role, and domain accuracy, coverage, per-profile m
141
141
 
142
142
  `raw_confidence` is the direct score assembled from independent evidence. `confidence` is the profile calibration for that score bucket, derived from the versioned calibration fixture. A result sets `review_required` when calibrated confidence is below `0.80`.
143
143
 
144
+ See `benchmarks/README.md` for the held-out validation set; its first measurement was 71% semantic accuracy on 52 hand-written cases (Brazilian and US column names), which shows the engine is not yet production-grade for English column names.
145
+
144
146
  The embedded calibration is provisional: it is a controlled regression fixture, not evidence of production-grade accuracy. A held-out, versioned corpus is required before treating its precision as a production measurement.
145
147
 
148
+ ## Vocabulary and data sources
149
+
150
+ Beyond the hand-written terms, the vocabulary was extended from public data: column names of about 4,500 US open datasets (Socrata catalog) and 500 Brazilian CKAN resources, and word lists from IBGE (municipalities, countries, first names), the US SSA (first names), and `datasets/world-cities`. Only derived word lists are shipped. Sources, licenses, and the scripts that rebuild them are in `src/colsemantics/data/SOURCES.md` and `scripts/`.
151
+
152
+ With sampled values, columns of municipalities, US cities, countries, and first names (Brazilian or US) are recognized even when the column name says nothing. These lists are matched by exact (accent-insensitive) value, so misspelled or abbreviated values do not match.
153
+
154
+ ## Value patterns and locales
155
+
156
+ When a column has sampled values, colsemantics also looks at how the values are written. Detection uses values only, so it works the same for Brazilian and US data and does not depend on the vocabulary profile.
157
+
158
+ | Type | Brazil (BR) | United States (US) |
159
+ |---|---|---|
160
+ | `phone` | `(11) 91234-5678`, `11 91234-5678`, `+55 11 91234-5678`, `(21) 3456-7890`, `0800 123 4567`; the area code (DDD) must exist, mobiles start with 9, landlines with 2-5 | `(415) 555-2671`, `415-555-2671`, `415.555.2671`, `+1 415 555 2671`; area code and exchange start with 2-9 and are not N11 |
161
+ | `postal_code` | CEP `01310-100` | `94105-1234` (ZIP+4) |
162
+ | `address` | `Rua das Flores, 123`, `Av. Paulista, 1578`, `Rodovia BR-101 km 5` | `123 Main St`, `500 N. Michigan Ave., Suite 300`, `PO Box 1234` |
163
+ | national id | `cpf`, `cnpj` (check digits verified) | `ssn` (area/group/serial ranges verified), `ein` (format only) |
164
+ | any | `email`, `uuid` | |
165
+
166
+ Bare digit strings such as `4155552671` or `11912345678`, and plain five-digit ZIP codes, are not reported as phones or postal codes because they are indistinguishable from ids and amounts. Addresses are matched heuristically and are never marked `validated`.
167
+
168
+ The result carries a `pattern` that says what was found and where it looks like it comes from:
169
+
170
+ ```python
171
+ result = infer_column("f6", profile=profile_values(["(11) 91234-5678", "(21) 99876-5432"]))
172
+ result["pattern"]
173
+ # {"type": "phone", "locale": "BR", "variant": "mobile", "format": "(DD) 9NNNN-NNNN",
174
+ # "coverage": 1.0, "validated": True,
175
+ # "message": "100% of sampled values look like a Brazilian mobile phone number (e.g. (DD) 9NNNN-NNNN)"}
176
+ ```
177
+
178
+ `pattern` is `None` when no format reaches its coverage threshold (80% of the sampled values, 60% for addresses). The same text is used as the `evidence` of the classification. You can still pass `detected_pattern="phone"` to `infer_column`; legacy names (`CPF`, `E-mail`, `Telefone`, `CEP`) keep working.
179
+
180
+ `infer_csv` reads each file with type detection (numbers, dates, booleans in Brazilian and US formats), picks the `,`, `;`, tab, or `|` delimiter from the header, samples rows uniformly with a seed, and reports duplicate headers and malformed rows in `warnings`.
181
+
146
182
  ## Sensitive-data recommendations
147
183
 
148
184
  Each result includes a descriptive `sensitivity` recommendation. It does not modify input data.
149
185
 
150
- - Validated CPF patterns: `high` / `mask`
186
+ - Validated CPF or SSN patterns: `high` / `mask`
187
+ - Detected street addresses: `high` / `mask`
151
188
  - Person identity semantics: `high` / `restrict`
152
- - Contact semantics: `high` / `mask`
153
- - Other identifiers: `medium` / `review`
189
+ - Contact semantics (e-mail, phone): `high` / `mask`
190
+ - CNPJ, EIN, postal codes, other identifiers: `medium` / `review`
154
191
  - Everything else: `none` / `allow`
155
192
 
156
193
  ## Result shape
@@ -166,6 +203,7 @@ Each result includes a descriptive `sensitivity` recommendation. It does not mod
166
203
  "conclusive": bool,
167
204
  "review_required": bool,
168
205
  "sensitivity": {"level": str, "action": str, "reason": str},
206
+ "pattern": dict | None, # see "Value patterns and locales"
169
207
  "hypotheses": list[dict],
170
208
  }
171
209
  ```
@@ -71,7 +71,7 @@ results = infer_table(
71
71
 
72
72
  ## Vocabulary profiles and extensions
73
73
 
74
- `pt-BR` is the embedded profile. Use `available_profiles()` to list embedded profiles and `load_profile()` when an application needs an explicit context.
74
+ `pt-BR` is the embedded (and default) profile. A profile controls the column-name vocabulary only; value patterns such as e-mail, phone, and postal code are detected independently of it. Use `available_profiles()` to list embedded profiles and `load_profile()` when an application needs an explicit context.
75
75
 
76
76
  ```python
77
77
  from colsemantics import infer_column, temporary_profile
@@ -108,16 +108,53 @@ The report includes semantic, role, and domain accuracy, coverage, per-profile m
108
108
 
109
109
  `raw_confidence` is the direct score assembled from independent evidence. `confidence` is the profile calibration for that score bucket, derived from the versioned calibration fixture. A result sets `review_required` when calibrated confidence is below `0.80`.
110
110
 
111
+ See `benchmarks/README.md` for the held-out validation set; its first measurement was 71% semantic accuracy on 52 hand-written cases (Brazilian and US column names), which shows the engine is not yet production-grade for English column names.
112
+
111
113
  The embedded calibration is provisional: it is a controlled regression fixture, not evidence of production-grade accuracy. A held-out, versioned corpus is required before treating its precision as a production measurement.
112
114
 
115
+ ## Vocabulary and data sources
116
+
117
+ Beyond the hand-written terms, the vocabulary was extended from public data: column names of about 4,500 US open datasets (Socrata catalog) and 500 Brazilian CKAN resources, and word lists from IBGE (municipalities, countries, first names), the US SSA (first names), and `datasets/world-cities`. Only derived word lists are shipped. Sources, licenses, and the scripts that rebuild them are in `src/colsemantics/data/SOURCES.md` and `scripts/`.
118
+
119
+ With sampled values, columns of municipalities, US cities, countries, and first names (Brazilian or US) are recognized even when the column name says nothing. These lists are matched by exact (accent-insensitive) value, so misspelled or abbreviated values do not match.
120
+
121
+ ## Value patterns and locales
122
+
123
+ When a column has sampled values, colsemantics also looks at how the values are written. Detection uses values only, so it works the same for Brazilian and US data and does not depend on the vocabulary profile.
124
+
125
+ | Type | Brazil (BR) | United States (US) |
126
+ |---|---|---|
127
+ | `phone` | `(11) 91234-5678`, `11 91234-5678`, `+55 11 91234-5678`, `(21) 3456-7890`, `0800 123 4567`; the area code (DDD) must exist, mobiles start with 9, landlines with 2-5 | `(415) 555-2671`, `415-555-2671`, `415.555.2671`, `+1 415 555 2671`; area code and exchange start with 2-9 and are not N11 |
128
+ | `postal_code` | CEP `01310-100` | `94105-1234` (ZIP+4) |
129
+ | `address` | `Rua das Flores, 123`, `Av. Paulista, 1578`, `Rodovia BR-101 km 5` | `123 Main St`, `500 N. Michigan Ave., Suite 300`, `PO Box 1234` |
130
+ | national id | `cpf`, `cnpj` (check digits verified) | `ssn` (area/group/serial ranges verified), `ein` (format only) |
131
+ | any | `email`, `uuid` | |
132
+
133
+ Bare digit strings such as `4155552671` or `11912345678`, and plain five-digit ZIP codes, are not reported as phones or postal codes because they are indistinguishable from ids and amounts. Addresses are matched heuristically and are never marked `validated`.
134
+
135
+ The result carries a `pattern` that says what was found and where it looks like it comes from:
136
+
137
+ ```python
138
+ result = infer_column("f6", profile=profile_values(["(11) 91234-5678", "(21) 99876-5432"]))
139
+ result["pattern"]
140
+ # {"type": "phone", "locale": "BR", "variant": "mobile", "format": "(DD) 9NNNN-NNNN",
141
+ # "coverage": 1.0, "validated": True,
142
+ # "message": "100% of sampled values look like a Brazilian mobile phone number (e.g. (DD) 9NNNN-NNNN)"}
143
+ ```
144
+
145
+ `pattern` is `None` when no format reaches its coverage threshold (80% of the sampled values, 60% for addresses). The same text is used as the `evidence` of the classification. You can still pass `detected_pattern="phone"` to `infer_column`; legacy names (`CPF`, `E-mail`, `Telefone`, `CEP`) keep working.
146
+
147
+ `infer_csv` reads each file with type detection (numbers, dates, booleans in Brazilian and US formats), picks the `,`, `;`, tab, or `|` delimiter from the header, samples rows uniformly with a seed, and reports duplicate headers and malformed rows in `warnings`.
148
+
113
149
  ## Sensitive-data recommendations
114
150
 
115
151
  Each result includes a descriptive `sensitivity` recommendation. It does not modify input data.
116
152
 
117
- - Validated CPF patterns: `high` / `mask`
153
+ - Validated CPF or SSN patterns: `high` / `mask`
154
+ - Detected street addresses: `high` / `mask`
118
155
  - Person identity semantics: `high` / `restrict`
119
- - Contact semantics: `high` / `mask`
120
- - Other identifiers: `medium` / `review`
156
+ - Contact semantics (e-mail, phone): `high` / `mask`
157
+ - CNPJ, EIN, postal codes, other identifiers: `medium` / `review`
121
158
  - Everything else: `none` / `allow`
122
159
 
123
160
  ## Result shape
@@ -133,6 +170,7 @@ Each result includes a descriptive `sensitivity` recommendation. It does not mod
133
170
  "conclusive": bool,
134
171
  "review_required": bool,
135
172
  "sensitivity": {"level": str, "action": str, "reason": str},
173
+ "pattern": dict | None, # see "Value patterns and locales"
136
174
  "hypotheses": list[dict],
137
175
  }
138
176
  ```
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "colsemantics"
3
- version = "0.3.1"
3
+ version = "0.4.1"
4
4
  description = "Infer semantic roles and domains for tabular columns from names, values, and table context."
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -37,14 +37,14 @@ colsemantics = "colsemantics.cli:main"
37
37
  dev = ["pytest>=9.2.0", "pytest-cov>=7.1.0", "mypy>=2.4.1", "ruff>=0.17.4"]
38
38
 
39
39
  [build-system]
40
- requires = ["setuptools>=68"]
40
+ requires = ["setuptools>=84.2.1"]
41
41
  build-backend = "setuptools.build_meta"
42
42
 
43
43
  [tool.setuptools.packages.find]
44
44
  where = ["src"]
45
45
 
46
46
  [tool.setuptools.package-data]
47
- colsemantics = ["py.typed", "calibration.json"]
47
+ colsemantics = ["py.typed", "calibration.json", "data/*.txt", "data/*.md"]
48
48
 
49
49
  [tool.pytest.ini_options]
50
50
  testpaths = ["tests"]
@@ -67,12 +67,13 @@ python_version = "3.11"
67
67
  files = ["src/colsemantics"]
68
68
  ignore_missing_imports = true
69
69
  warn_unused_ignores = true
70
+ strict = true
70
71
 
71
72
  [tool.coverage.run]
72
73
  source = ["colsemantics"]
73
74
  branch = true
74
75
 
75
76
  [tool.coverage.report]
76
- fail_under = 85
77
+ fail_under = 90
77
78
  show_missing = true
78
79
  skip_covered = true
@@ -16,16 +16,31 @@ from .detectors import (
16
16
  by_table_context,
17
17
  )
18
18
  from .evidence import DOMAIN_AXIS, ROLE_AXIS, Evidence, choose, rank
19
+ from .patterns import NO_PATTERN, PATTERN_LABELS, analyze_values, canonical_pattern
19
20
  from .profiles import available_profiles, load_profile, temporary_profile
21
+ from .results import (
22
+ CsvColumnResult,
23
+ CsvReport,
24
+ Hypothesis,
25
+ InferenceResult,
26
+ PatternInfo,
27
+ Sensitivity,
28
+ )
20
29
  from .sensitivity import assess_sensitivity
21
30
  from .tokens import normalize, tokenize
22
31
  from .vocabularies import export_overrides_template, load_vocabularies, temporary_vocabulary
23
32
 
24
- __version__ = "0.3.1"
33
+ __version__ = "0.4.1"
25
34
 
26
35
  __all__ = [
27
36
  "ContentProfile",
37
+ "CsvColumnResult",
38
+ "CsvReport",
39
+ "Hypothesis",
40
+ "InferenceResult",
41
+ "PatternInfo",
28
42
  "SemanticContext",
43
+ "Sensitivity",
29
44
  "assess_sensitivity",
30
45
  "available_profiles",
31
46
  "calibrate",
@@ -61,18 +76,47 @@ def __getattr__(name: str) -> Any:
61
76
  _CONTEXT_MINIMUM_CONFIDENCE = 0.7
62
77
  _DOMAIN_MINIMUM_CONFIDENCE = 0.5
63
78
  _MAX_HYPOTHESES = 4
79
+ # A strong domain (for example an address pattern in the values) beats a weak structural
80
+ # role guessed from an uninformative column name.
81
+ _CONTENT_DOMAIN_CONFIDENCE = 0.85
82
+ _CONTENT_DOMAIN_MARGIN = 0.3
83
+
84
+
85
+ def _resolve_pattern(detected_pattern: str, profile: ContentProfile | None) -> PatternInfo | None:
86
+ """Combine a caller-reported pattern with what the sampled values actually look like."""
87
+ reported = canonical_pattern(detected_pattern)
88
+ expected = None if reported == NO_PATTERN else reported
89
+ found = (
90
+ analyze_values(profile.distinct_values, expected_type=expected)
91
+ if profile is not None and profile.distinct_values
92
+ else None
93
+ )
94
+ if found is not None or expected is None:
95
+ return found
96
+ label = PATTERN_LABELS.get(expected, expected)
97
+ return {
98
+ "type": expected,
99
+ "locale": None,
100
+ "variant": None,
101
+ "format": None,
102
+ "coverage": None,
103
+ "validated": True,
104
+ "message": f"validated {label} pattern",
105
+ }
64
106
 
65
107
 
66
108
  def _collect_evidence(
67
- column_name: str, detected_pattern: str, profile: ContentProfile | None
109
+ column_name: str, pattern: PatternInfo | None, profile: ContentProfile | None
68
110
  ) -> list[Evidence]:
69
111
  tokens = tokenize(column_name)
70
112
  normalized_name = normalize(column_name)
71
113
  evidence_items = (
72
- by_content_pattern(detected_pattern)
73
- + by_strong_token(tokens)
74
- + by_fuzzy(normalized_name, tokens)
114
+ by_content_pattern(pattern) + by_strong_token(tokens) + by_fuzzy(normalized_name, tokens)
75
115
  )
116
+ if column_name.rstrip().endswith("?"):
117
+ evidence_items.append(
118
+ Evidence("Status / Indicator / Flag", ROLE_AXIS, 0.8, "question-style column name")
119
+ )
76
120
  if profile is not None:
77
121
  evidence_items += by_gazetteer(profile) + by_structural_signature(profile)
78
122
  return evidence_items
@@ -100,8 +144,8 @@ def _refine_role(
100
144
  def _build_result(
101
145
  evidence_items: list[Evidence],
102
146
  profile: ContentProfile | None = None,
103
- detected_pattern: str = "None",
104
- ) -> dict[str, Any]:
147
+ pattern: PatternInfo | None = None,
148
+ ) -> InferenceResult:
105
149
  role_ranking = rank(evidence_items, ROLE_AXIS)
106
150
  domain_ranking = rank(evidence_items, DOMAIN_AXIS)
107
151
  role, role_confidence, role_source, role_conclusive = choose(role_ranking)
@@ -110,7 +154,15 @@ def _build_result(
110
154
  if uncertain_domain:
111
155
  domain, domain_confidence, domain_source = None, 0.0, "No evidence"
112
156
  role = _refine_role(role, domain, profile)
113
- if role in STRUCTURAL_ROLES:
157
+ geometry_found = pattern is not None and pattern["type"] in ("geometry", "coordinates")
158
+ content_domain_wins = domain is not None and (
159
+ geometry_found
160
+ or (
161
+ domain_confidence >= _CONTENT_DOMAIN_CONFIDENCE
162
+ and role_confidence < domain_confidence - _CONTENT_DOMAIN_MARGIN
163
+ )
164
+ )
165
+ if role in STRUCTURAL_ROLES and not content_domain_wins:
114
166
  semantic, raw_confidence, source = role, role_confidence, role_source
115
167
  elif domain is not None:
116
168
  semantic, raw_confidence, source = domain, domain_confidence, domain_source
@@ -118,19 +170,19 @@ def _build_result(
118
170
  semantic, raw_confidence, source = role, role_confidence, role_source
119
171
  else:
120
172
  semantic, raw_confidence, source = config.GENERIC_SEMANTIC, 0.0, "Unmatched"
121
- hypotheses = sorted(
122
- [
123
- {
124
- "semantic": item["category"],
125
- "axis": axis,
126
- "confidence": item["confidence"],
127
- "evidence": item["sources"][:3],
128
- }
129
- for axis, ranking in ((ROLE_AXIS, role_ranking), (DOMAIN_AXIS, domain_ranking))
130
- for item in ranking
131
- ],
132
- key=lambda hypothesis: -hypothesis["confidence"],
133
- )[:_MAX_HYPOTHESES]
173
+ candidates: list[Hypothesis] = [
174
+ {
175
+ "semantic": item["category"],
176
+ "axis": axis,
177
+ "confidence": item["confidence"],
178
+ "evidence": item["sources"][:3],
179
+ }
180
+ for axis, ranking in ((ROLE_AXIS, role_ranking), (DOMAIN_AXIS, domain_ranking))
181
+ for item in ranking
182
+ ]
183
+ hypotheses = sorted(candidates, key=lambda hypothesis: -hypothesis["confidence"])[
184
+ :_MAX_HYPOTHESES
185
+ ]
134
186
  confidence = calibrate(round(raw_confidence, 4), current_context().profile_name)
135
187
  return {
136
188
  "semantic": semantic,
@@ -141,71 +193,80 @@ def _build_result(
141
193
  "evidence": source,
142
194
  "conclusive": not (bool(role_ranking) and not role_conclusive) and not uncertain_domain,
143
195
  "review_required": requires_review(confidence),
144
- "sensitivity": assess_sensitivity(semantic, detected_pattern),
196
+ "sensitivity": assess_sensitivity(semantic, pattern["type"] if pattern else NO_PATTERN),
197
+ "pattern": pattern,
145
198
  "hypotheses": hypotheses,
146
199
  }
147
200
 
148
201
 
149
- def infer_column(
150
- column_name: str, detected_pattern: str = "None", profile: ContentProfile | None = None
151
- ) -> dict[str, Any]:
152
- override = current_context().column_overrides.get(column_name)
202
+ def _column_evidence(
203
+ name: str, pattern: PatternInfo | None, profile: ContentProfile | None
204
+ ) -> tuple[list[Evidence], bool]:
205
+ """Return the evidence for a column and whether a vocabulary override decided it."""
206
+ override = current_context().column_overrides.get(name)
153
207
  if override:
154
208
  axis = ROLE_AXIS if override in STRUCTURAL_ROLES else DOMAIN_AXIS
155
- result = _build_result(
156
- [Evidence(override, axis, 1.0, "vocabulary override")], profile, detected_pattern
157
- )
209
+ return [Evidence(override, axis, 1.0, "vocabulary override")], True
210
+ return _collect_evidence(name, pattern, profile), False
211
+
212
+
213
+ def _infer_one(
214
+ name: str, detected_pattern: str, profile: ContentProfile | None
215
+ ) -> tuple[InferenceResult, list[Evidence]]:
216
+ pattern = _resolve_pattern(detected_pattern, profile)
217
+ evidence_items, overridden = _column_evidence(name, pattern, profile)
218
+ result = _build_result(evidence_items, profile, pattern)
219
+ if overridden:
158
220
  result["conclusive"] = True
159
- return result
160
- return _build_result(
161
- _collect_evidence(column_name, detected_pattern, profile), profile, detected_pattern
162
- )
221
+ return result, evidence_items
222
+
223
+
224
+ def infer_column(
225
+ column_name: str, detected_pattern: str = "None", profile: ContentProfile | None = None
226
+ ) -> InferenceResult:
227
+ return _infer_one(column_name, detected_pattern, profile)[0]
228
+
229
+
230
+ def _column_name(column: dict[str, Any], position: int) -> str:
231
+ name = column.get("column_name", column.get("name"))
232
+ if not isinstance(name, str) or not name.strip():
233
+ raise ValueError(f"Column at position {position} requires a non-empty 'column_name'.")
234
+ return name
163
235
 
164
236
 
165
- def infer_table(columns: list[dict[str, Any]]) -> list[dict[str, Any]]:
166
- evidence_by_column: list[list[Evidence]] = []
167
- results: list[dict[str, Any]] = []
168
- for column in columns:
169
- name = str(column.get("column_name", column.get("name")))
170
- profile = column.get("profile")
171
- if profile is not None and not isinstance(profile, ContentProfile):
172
- raise TypeError("profile must be a ContentProfile instance")
173
- override = current_context().column_overrides.get(name)
174
- detected_pattern = str(column.get("detected_pattern", column.get("pattern", "None")))
175
- evidence_items = (
176
- [
177
- Evidence(
178
- override,
179
- ROLE_AXIS if override in STRUCTURAL_ROLES else DOMAIN_AXIS,
180
- 1.0,
181
- "vocabulary override",
182
- )
183
- ]
184
- if override
185
- else _collect_evidence(name, detected_pattern, profile)
237
+ def _column_profile(column: dict[str, Any]) -> ContentProfile | None:
238
+ profile = column.get("profile")
239
+ if profile is not None and not isinstance(profile, ContentProfile):
240
+ raise TypeError("profile must be a ContentProfile instance")
241
+ return profile
242
+
243
+
244
+ def infer_table(columns: list[dict[str, Any]]) -> list[InferenceResult]:
245
+ parsed = [
246
+ (
247
+ _column_name(column, position),
248
+ str(column.get("detected_pattern", column.get("pattern", NO_PATTERN))),
249
+ _column_profile(column),
186
250
  )
187
- evidence_by_column.append(evidence_items)
188
- result = _build_result(evidence_items, profile, detected_pattern)
189
- if override:
190
- result["conclusive"] = True
191
- results.append(result)
251
+ for position, column in enumerate(columns)
252
+ ]
253
+ first_pass = [_infer_one(name, pattern, profile) for name, pattern, profile in parsed]
254
+ results = [result for result, _ in first_pass]
192
255
  table_domains = _subject_profile(results)
193
256
  if not table_domains:
194
257
  return results
195
- for index, (column, result) in enumerate(zip(columns, results, strict=True)):
258
+ for index, ((name, _pattern, profile), (result, evidence_items)) in enumerate(
259
+ zip(parsed, first_pass, strict=True)
260
+ ):
196
261
  if result["conclusive"]:
197
262
  continue
198
- name = str(column.get("column_name", column.get("name")))
199
263
  extras = by_table_context(tokenize(name), table_domains)
200
264
  if extras:
201
- detected_pattern = str(column.get("detected_pattern", column.get("pattern", "None")))
202
- results[index] = _build_result(
203
- evidence_by_column[index] + extras, column.get("profile"), detected_pattern
204
- )
265
+ results[index] = _build_result(evidence_items + extras, profile, result["pattern"])
205
266
  return results
206
267
 
207
268
 
208
- def _subject_profile(results: list[dict[str, Any]]) -> dict[str, float]:
269
+ def _subject_profile(results: list[InferenceResult]) -> dict[str, float]:
209
270
  strengths: dict[str, float] = {}
210
271
  for result in results:
211
272
  if not result["conclusive"]:
@@ -0,0 +1,13 @@
1
+ """Load the word lists shipped in ``colsemantics/data``."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from functools import cache
6
+ from importlib import resources
7
+
8
+
9
+ @cache
10
+ def load_wordlist(name: str) -> frozenset[str]:
11
+ """Return the lines of ``data/<name>`` (already normalized: lowercase, no accents)."""
12
+ text = resources.files("colsemantics").joinpath("data", name).read_text(encoding="utf-8")
13
+ return frozenset(line for line in text.splitlines() if line)