colsemantics 0.1.0__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- colsemantics-0.3.1/PKG-INFO +193 -0
- colsemantics-0.3.1/README.md +160 -0
- {colsemantics-0.1.0 → colsemantics-0.3.1}/pyproject.toml +27 -10
- colsemantics-0.3.1/src/colsemantics/__init__.py +220 -0
- colsemantics-0.3.1/src/colsemantics/_taxonomy.py +378 -0
- colsemantics-0.3.1/src/colsemantics/benchmark.py +146 -0
- colsemantics-0.3.1/src/colsemantics/calibration.json +17 -0
- colsemantics-0.3.1/src/colsemantics/calibration.py +42 -0
- colsemantics-0.3.1/src/colsemantics/cli.py +64 -0
- colsemantics-0.3.1/src/colsemantics/context.py +73 -0
- colsemantics-0.3.1/src/colsemantics/csv_inference.py +69 -0
- colsemantics-0.3.1/src/colsemantics/detectors.py +318 -0
- colsemantics-0.3.1/src/colsemantics/evidence.py +54 -0
- colsemantics-0.3.1/src/colsemantics/profiles.py +29 -0
- colsemantics-0.3.1/src/colsemantics/sensitivity.py +30 -0
- colsemantics-0.3.1/src/colsemantics/tokens.py +80 -0
- colsemantics-0.3.1/src/colsemantics/vocabularies.py +89 -0
- colsemantics-0.3.1/src/colsemantics/vocabulary.py +473 -0
- colsemantics-0.3.1/src/colsemantics.egg-info/PKG-INFO +193 -0
- colsemantics-0.3.1/src/colsemantics.egg-info/SOURCES.txt +33 -0
- colsemantics-0.3.1/src/colsemantics.egg-info/entry_points.txt +2 -0
- colsemantics-0.3.1/src/colsemantics.egg-info/requires.txt +9 -0
- colsemantics-0.3.1/tests/test_benchmark.py +70 -0
- colsemantics-0.3.1/tests/test_calibration.py +29 -0
- colsemantics-0.3.1/tests/test_cli.py +39 -0
- colsemantics-0.3.1/tests/test_colsemantics.py +202 -0
- colsemantics-0.3.1/tests/test_csv_inference.py +49 -0
- colsemantics-0.3.1/tests/test_english_core.py +10 -0
- colsemantics-0.3.1/tests/test_profiles.py +24 -0
- colsemantics-0.3.1/tests/test_sensitivity.py +42 -0
- colsemantics-0.1.0/PKG-INFO +0 -162
- colsemantics-0.1.0/README.md +0 -135
- colsemantics-0.1.0/src/colsemantics/__init__.py +0 -248
- colsemantics-0.1.0/src/colsemantics/_taxonomy.py +0 -177
- colsemantics-0.1.0/src/colsemantics/detectors.py +0 -362
- colsemantics-0.1.0/src/colsemantics/evidence.py +0 -82
- colsemantics-0.1.0/src/colsemantics/tokens.py +0 -131
- colsemantics-0.1.0/src/colsemantics/vocabulary.py +0 -171
- colsemantics-0.1.0/src/colsemantics.egg-info/PKG-INFO +0 -162
- colsemantics-0.1.0/src/colsemantics.egg-info/SOURCES.txt +0 -16
- colsemantics-0.1.0/src/colsemantics.egg-info/requires.txt +0 -7
- colsemantics-0.1.0/tests/test_colsemantics.py +0 -384
- {colsemantics-0.1.0 → colsemantics-0.3.1}/LICENSE +0 -0
- {colsemantics-0.1.0 → colsemantics-0.3.1}/setup.cfg +0 -0
- {colsemantics-0.1.0 → colsemantics-0.3.1}/src/colsemantics/py.typed +0 -0
- {colsemantics-0.1.0 → colsemantics-0.3.1}/src/colsemantics.egg-info/dependency_links.txt +0 -0
- {colsemantics-0.1.0 → colsemantics-0.3.1}/src/colsemantics.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: colsemantics
|
|
3
|
+
Version: 0.3.1
|
|
4
|
+
Summary: Infer semantic roles and domains for tabular columns from names, values, and table context.
|
|
5
|
+
Author: Caio
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Caio-Analytics/colsemantics
|
|
8
|
+
Project-URL: Repository, https://github.com/Caio-Analytics/colsemantics
|
|
9
|
+
Project-URL: Issues, https://github.com/Caio-Analytics/colsemantics/issues
|
|
10
|
+
Project-URL: Changelog, https://github.com/Caio-Analytics/colsemantics/blob/main/CHANGELOG.md
|
|
11
|
+
Keywords: semantic-inference,data-profiling,schema-inference,column-classification,pii,etl
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
20
|
+
Classifier: Typing :: Typed
|
|
21
|
+
Requires-Python: >=3.11
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
License-File: LICENSE
|
|
24
|
+
Requires-Dist: rapidfuzz>=3.14.6
|
|
25
|
+
Requires-Dist: unidecode>=1.4.0
|
|
26
|
+
Requires-Dist: PyYAML>=6.0.3
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: pytest>=9.2.0; extra == "dev"
|
|
29
|
+
Requires-Dist: pytest-cov>=7.1.0; extra == "dev"
|
|
30
|
+
Requires-Dist: mypy>=2.4.1; extra == "dev"
|
|
31
|
+
Requires-Dist: ruff>=0.17.4; extra == "dev"
|
|
32
|
+
Dynamic: license-file
|
|
33
|
+
|
|
34
|
+
# colsemantics
|
|
35
|
+
|
|
36
|
+
[](https://pypi.org/project/colsemantics/)
|
|
37
|
+
[](https://pypi.org/project/colsemantics/)
|
|
38
|
+
[](https://github.com/Caio-Analytics/colsemantics/actions/workflows/ci.yml)
|
|
39
|
+
[](LICENSE)
|
|
40
|
+
[](https://pypi.org/project/colsemantics/)
|
|
41
|
+
|
|
42
|
+
`colsemantics` infers the structural role and business domain of tabular columns from column names, sampled values, and neighboring columns.
|
|
43
|
+
|
|
44
|
+
It is useful when a dataset has unclear headers such as `cd_dpto_lot`, `f27`, or `SUPPLIER_CONTACT_CODE`.
|
|
45
|
+
|
|
46
|
+
## Installation
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
pip install colsemantics
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Quick start
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from colsemantics import infer_column
|
|
56
|
+
|
|
57
|
+
result = infer_column("SUPPLIER_CONTACT_CODE")
|
|
58
|
+
print(result["role"])
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
The English API returns English field names and category labels.
|
|
62
|
+
|
|
63
|
+
## Analyze a CSV file
|
|
64
|
+
|
|
65
|
+
Install the command-line interface with the package, then create a JSON report from a bounded CSV sample:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
colsemantics infer employees.csv --profile pt-BR --sample 10000 --output report.json
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
CSV is the supported file format in this release. The report records its source, profile, sample size, inferred columns, and a semantic summary.
|
|
72
|
+
|
|
73
|
+
## Use sampled values
|
|
74
|
+
|
|
75
|
+
```python
|
|
76
|
+
from colsemantics import ContentProfile, infer_column
|
|
77
|
+
|
|
78
|
+
profile = ContentProfile(
|
|
79
|
+
data_type="Text",
|
|
80
|
+
distinct_values=["SP", "RJ", "MG", "BA"],
|
|
81
|
+
distinct_count=4,
|
|
82
|
+
uniqueness_ratio=0.1,
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
result = infer_column("f27", profile=profile)
|
|
86
|
+
print(result["domain"])
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## Analyze a table
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
from colsemantics import infer_table
|
|
93
|
+
|
|
94
|
+
results = infer_table(
|
|
95
|
+
[
|
|
96
|
+
{"column_name": "employee_id"},
|
|
97
|
+
{"column_name": "department_name"},
|
|
98
|
+
{"column_name": "start_date"},
|
|
99
|
+
]
|
|
100
|
+
)
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
`infer_table` uses high-confidence classifications as table context when it evaluates ambiguous columns.
|
|
104
|
+
|
|
105
|
+
## Vocabulary profiles and extensions
|
|
106
|
+
|
|
107
|
+
`pt-BR` is the embedded profile. Use `available_profiles()` to list embedded profiles and `load_profile()` when an application needs an explicit context.
|
|
108
|
+
|
|
109
|
+
```python
|
|
110
|
+
from colsemantics import infer_column, temporary_profile
|
|
111
|
+
|
|
112
|
+
with temporary_profile("pt-BR", "company-vocabulary.yaml"):
|
|
113
|
+
result = infer_column("cost_bucket")
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
YAML extensions use the English schema and apply only within the selected context:
|
|
117
|
+
|
|
118
|
+
Provide one or more comma-separated YAML files to `load_vocabularies`, then use the returned context for the current operation.
|
|
119
|
+
|
|
120
|
+
```yaml
|
|
121
|
+
strong_categories:
|
|
122
|
+
Project domain:
|
|
123
|
+
- workstream
|
|
124
|
+
column_overrides:
|
|
125
|
+
cost_bucket: Finance / Cost
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Custom vocabularies are scoped with `ContextVar`, so separate concurrent analyses do not share vocabulary changes.
|
|
129
|
+
|
|
130
|
+
## Benchmark inference quality
|
|
131
|
+
|
|
132
|
+
Run the synthetic benchmark fixture included with the repository:
|
|
133
|
+
|
|
134
|
+
```bash
|
|
135
|
+
colsemantics benchmark benchmarks/synthetic.json --output benchmark-report.json
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
The report includes semantic, role, and domain accuracy, coverage, per-profile metrics, a confusion matrix, and calibration buckets with expected calibration error. Public corpus definitions are stored as source manifests; they are not bundled or downloaded automatically.
|
|
139
|
+
|
|
140
|
+
## Confidence and review decisions
|
|
141
|
+
|
|
142
|
+
`raw_confidence` is the direct score assembled from independent evidence. `confidence` is the profile calibration for that score bucket, derived from the versioned calibration fixture. A result sets `review_required` when calibrated confidence is below `0.80`.
|
|
143
|
+
|
|
144
|
+
The embedded calibration is provisional: it is a controlled regression fixture, not evidence of production-grade accuracy. A held-out, versioned corpus is required before treating its precision as a production measurement.
|
|
145
|
+
|
|
146
|
+
## Sensitive-data recommendations
|
|
147
|
+
|
|
148
|
+
Each result includes a descriptive `sensitivity` recommendation. It does not modify input data.
|
|
149
|
+
|
|
150
|
+
- Validated CPF patterns: `high` / `mask`
|
|
151
|
+
- Person identity semantics: `high` / `restrict`
|
|
152
|
+
- Contact semantics: `high` / `mask`
|
|
153
|
+
- Other identifiers: `medium` / `review`
|
|
154
|
+
- Everything else: `none` / `allow`
|
|
155
|
+
|
|
156
|
+
## Result shape
|
|
157
|
+
|
|
158
|
+
```python
|
|
159
|
+
{
|
|
160
|
+
"semantic": str,
|
|
161
|
+
"role": str | None,
|
|
162
|
+
"domain": str | None,
|
|
163
|
+
"raw_confidence": float,
|
|
164
|
+
"confidence": float,
|
|
165
|
+
"evidence": str,
|
|
166
|
+
"conclusive": bool,
|
|
167
|
+
"review_required": bool,
|
|
168
|
+
"sensitivity": {"level": str, "action": str, "reason": str},
|
|
169
|
+
"hypotheses": list[dict],
|
|
170
|
+
}
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
## Migrating from 0.2
|
|
174
|
+
|
|
175
|
+
`normalizar` and `tokenizar` were renamed to `normalize` and `tokenize`. The old names still work but emit a `DeprecationWarning` and will be removed in 1.0. `infer_table`, `assess_sensitivity`, `calibrate`, and `requires_review` are now listed in `__all__`.
|
|
176
|
+
|
|
177
|
+
Python 3.10 is no longer supported (it reached end of life, and current `rapidfuzz` requires 3.11).
|
|
178
|
+
|
|
179
|
+
## Contributing
|
|
180
|
+
|
|
181
|
+
Contributions are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, checks, and the release process, and [CHANGELOG.md](CHANGELOG.md) for version history.
|
|
182
|
+
|
|
183
|
+
```bash
|
|
184
|
+
python -m pip install -e '.[dev]'
|
|
185
|
+
pytest --cov
|
|
186
|
+
ruff check .
|
|
187
|
+
ruff format --check .
|
|
188
|
+
mypy
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
## License
|
|
192
|
+
|
|
193
|
+
MIT.
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
# colsemantics
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/colsemantics/)
|
|
4
|
+
[](https://pypi.org/project/colsemantics/)
|
|
5
|
+
[](https://github.com/Caio-Analytics/colsemantics/actions/workflows/ci.yml)
|
|
6
|
+
[](LICENSE)
|
|
7
|
+
[](https://pypi.org/project/colsemantics/)
|
|
8
|
+
|
|
9
|
+
`colsemantics` infers the structural role and business domain of tabular columns from column names, sampled values, and neighboring columns.
|
|
10
|
+
|
|
11
|
+
It is useful when a dataset has unclear headers such as `cd_dpto_lot`, `f27`, or `SUPPLIER_CONTACT_CODE`.
|
|
12
|
+
|
|
13
|
+
## Installation
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
pip install colsemantics
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## Quick start
|
|
20
|
+
|
|
21
|
+
```python
|
|
22
|
+
from colsemantics import infer_column
|
|
23
|
+
|
|
24
|
+
result = infer_column("SUPPLIER_CONTACT_CODE")
|
|
25
|
+
print(result["role"])
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
The English API returns English field names and category labels.
|
|
29
|
+
|
|
30
|
+
## Analyze a CSV file
|
|
31
|
+
|
|
32
|
+
Install the command-line interface with the package, then create a JSON report from a bounded CSV sample:
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
colsemantics infer employees.csv --profile pt-BR --sample 10000 --output report.json
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
CSV is the supported file format in this release. The report records its source, profile, sample size, inferred columns, and a semantic summary.
|
|
39
|
+
|
|
40
|
+
## Use sampled values
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
from colsemantics import ContentProfile, infer_column
|
|
44
|
+
|
|
45
|
+
profile = ContentProfile(
|
|
46
|
+
data_type="Text",
|
|
47
|
+
distinct_values=["SP", "RJ", "MG", "BA"],
|
|
48
|
+
distinct_count=4,
|
|
49
|
+
uniqueness_ratio=0.1,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
result = infer_column("f27", profile=profile)
|
|
53
|
+
print(result["domain"])
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
## Analyze a table
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
from colsemantics import infer_table
|
|
60
|
+
|
|
61
|
+
results = infer_table(
|
|
62
|
+
[
|
|
63
|
+
{"column_name": "employee_id"},
|
|
64
|
+
{"column_name": "department_name"},
|
|
65
|
+
{"column_name": "start_date"},
|
|
66
|
+
]
|
|
67
|
+
)
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
`infer_table` uses high-confidence classifications as table context when it evaluates ambiguous columns.
|
|
71
|
+
|
|
72
|
+
## Vocabulary profiles and extensions
|
|
73
|
+
|
|
74
|
+
`pt-BR` is the embedded profile. Use `available_profiles()` to list embedded profiles and `load_profile()` when an application needs an explicit context.
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
from colsemantics import infer_column, temporary_profile
|
|
78
|
+
|
|
79
|
+
with temporary_profile("pt-BR", "company-vocabulary.yaml"):
|
|
80
|
+
result = infer_column("cost_bucket")
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
YAML extensions use the English schema and apply only within the selected context:
|
|
84
|
+
|
|
85
|
+
Provide one or more comma-separated YAML files to `load_vocabularies`, then use the returned context for the current operation.
|
|
86
|
+
|
|
87
|
+
```yaml
|
|
88
|
+
strong_categories:
|
|
89
|
+
Project domain:
|
|
90
|
+
- workstream
|
|
91
|
+
column_overrides:
|
|
92
|
+
cost_bucket: Finance / Cost
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Custom vocabularies are scoped with `ContextVar`, so separate concurrent analyses do not share vocabulary changes.
|
|
96
|
+
|
|
97
|
+
## Benchmark inference quality
|
|
98
|
+
|
|
99
|
+
Run the synthetic benchmark fixture included with the repository:
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
colsemantics benchmark benchmarks/synthetic.json --output benchmark-report.json
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
The report includes semantic, role, and domain accuracy, coverage, per-profile metrics, a confusion matrix, and calibration buckets with expected calibration error. Public corpus definitions are stored as source manifests; they are not bundled or downloaded automatically.
|
|
106
|
+
|
|
107
|
+
## Confidence and review decisions
|
|
108
|
+
|
|
109
|
+
`raw_confidence` is the direct score assembled from independent evidence. `confidence` is the profile calibration for that score bucket, derived from the versioned calibration fixture. A result sets `review_required` when calibrated confidence is below `0.80`.
|
|
110
|
+
|
|
111
|
+
The embedded calibration is provisional: it is a controlled regression fixture, not evidence of production-grade accuracy. A held-out, versioned corpus is required before treating its precision as a production measurement.
|
|
112
|
+
|
|
113
|
+
## Sensitive-data recommendations
|
|
114
|
+
|
|
115
|
+
Each result includes a descriptive `sensitivity` recommendation. It does not modify input data.
|
|
116
|
+
|
|
117
|
+
- Validated CPF patterns: `high` / `mask`
|
|
118
|
+
- Person identity semantics: `high` / `restrict`
|
|
119
|
+
- Contact semantics: `high` / `mask`
|
|
120
|
+
- Other identifiers: `medium` / `review`
|
|
121
|
+
- Everything else: `none` / `allow`
|
|
122
|
+
|
|
123
|
+
## Result shape
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
{
|
|
127
|
+
"semantic": str,
|
|
128
|
+
"role": str | None,
|
|
129
|
+
"domain": str | None,
|
|
130
|
+
"raw_confidence": float,
|
|
131
|
+
"confidence": float,
|
|
132
|
+
"evidence": str,
|
|
133
|
+
"conclusive": bool,
|
|
134
|
+
"review_required": bool,
|
|
135
|
+
"sensitivity": {"level": str, "action": str, "reason": str},
|
|
136
|
+
"hypotheses": list[dict],
|
|
137
|
+
}
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
## Migrating from 0.2
|
|
141
|
+
|
|
142
|
+
`normalizar` and `tokenizar` were renamed to `normalize` and `tokenize`. The old names still work but emit a `DeprecationWarning` and will be removed in 1.0. `infer_table`, `assess_sensitivity`, `calibrate`, and `requires_review` are now listed in `__all__`.
|
|
143
|
+
|
|
144
|
+
Python 3.10 is no longer supported (it reached end of life, and current `rapidfuzz` requires 3.11).
|
|
145
|
+
|
|
146
|
+
## Contributing
|
|
147
|
+
|
|
148
|
+
Contributions are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, checks, and the release process, and [CHANGELOG.md](CHANGELOG.md) for version history.
|
|
149
|
+
|
|
150
|
+
```bash
|
|
151
|
+
python -m pip install -e '.[dev]'
|
|
152
|
+
pytest --cov
|
|
153
|
+
ruff check .
|
|
154
|
+
ruff format --check .
|
|
155
|
+
mypy
|
|
156
|
+
```
|
|
157
|
+
|
|
158
|
+
## License
|
|
159
|
+
|
|
160
|
+
MIT.
|
|
@@ -1,32 +1,40 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "colsemantics"
|
|
3
|
-
version = "0.1
|
|
4
|
-
description = "
|
|
3
|
+
version = "0.3.1"
|
|
4
|
+
description = "Infer semantic roles and domains for tabular columns from names, values, and table context."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "MIT"
|
|
7
7
|
authors = [{ name = "Caio" }]
|
|
8
|
-
requires-python = ">=3.
|
|
9
|
-
keywords = ["semantic", "data-profiling", "
|
|
8
|
+
requires-python = ">=3.11"
|
|
9
|
+
keywords = ["semantic-inference", "data-profiling", "schema-inference", "column-classification", "pii", "etl"]
|
|
10
10
|
classifiers = [
|
|
11
11
|
"Development Status :: 3 - Alpha",
|
|
12
12
|
"Intended Audience :: Developers",
|
|
13
13
|
"Programming Language :: Python :: 3",
|
|
14
|
-
"Programming Language :: Python :: 3.10",
|
|
15
14
|
"Programming Language :: Python :: 3.11",
|
|
16
15
|
"Programming Language :: Python :: 3.12",
|
|
16
|
+
"Programming Language :: Python :: 3.13",
|
|
17
|
+
"Programming Language :: Python :: 3.14",
|
|
17
18
|
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
18
19
|
"Typing :: Typed",
|
|
19
20
|
]
|
|
20
21
|
dependencies = [
|
|
21
|
-
"rapidfuzz>=3.
|
|
22
|
-
"unidecode>=1.
|
|
22
|
+
"rapidfuzz>=3.14.6",
|
|
23
|
+
"unidecode>=1.4.0",
|
|
24
|
+
"PyYAML>=6.0.3",
|
|
23
25
|
]
|
|
24
26
|
|
|
25
27
|
[project.urls]
|
|
26
28
|
Homepage = "https://github.com/Caio-Analytics/colsemantics"
|
|
29
|
+
Repository = "https://github.com/Caio-Analytics/colsemantics"
|
|
30
|
+
Issues = "https://github.com/Caio-Analytics/colsemantics/issues"
|
|
31
|
+
Changelog = "https://github.com/Caio-Analytics/colsemantics/blob/main/CHANGELOG.md"
|
|
32
|
+
|
|
33
|
+
[project.scripts]
|
|
34
|
+
colsemantics = "colsemantics.cli:main"
|
|
27
35
|
|
|
28
36
|
[project.optional-dependencies]
|
|
29
|
-
dev = ["pytest>=
|
|
37
|
+
dev = ["pytest>=9.2.0", "pytest-cov>=7.1.0", "mypy>=2.4.1", "ruff>=0.17.4"]
|
|
30
38
|
|
|
31
39
|
[build-system]
|
|
32
40
|
requires = ["setuptools>=68"]
|
|
@@ -36,7 +44,7 @@ build-backend = "setuptools.build_meta"
|
|
|
36
44
|
where = ["src"]
|
|
37
45
|
|
|
38
46
|
[tool.setuptools.package-data]
|
|
39
|
-
colsemantics = ["py.typed"]
|
|
47
|
+
colsemantics = ["py.typed", "calibration.json"]
|
|
40
48
|
|
|
41
49
|
[tool.pytest.ini_options]
|
|
42
50
|
testpaths = ["tests"]
|
|
@@ -55,7 +63,16 @@ ignore = [
|
|
|
55
63
|
]
|
|
56
64
|
|
|
57
65
|
[tool.mypy]
|
|
58
|
-
python_version = "3.
|
|
66
|
+
python_version = "3.11"
|
|
59
67
|
files = ["src/colsemantics"]
|
|
60
68
|
ignore_missing_imports = true
|
|
61
69
|
warn_unused_ignores = true
|
|
70
|
+
|
|
71
|
+
[tool.coverage.run]
|
|
72
|
+
source = ["colsemantics"]
|
|
73
|
+
branch = true
|
|
74
|
+
|
|
75
|
+
[tool.coverage.report]
|
|
76
|
+
fail_under = 85
|
|
77
|
+
show_missing = true
|
|
78
|
+
skip_covered = true
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
import warnings
|
|
2
|
+
from typing import Any
|
|
3
|
+
|
|
4
|
+
from . import _taxonomy as config
|
|
5
|
+
from .calibration import calibrate, requires_review
|
|
6
|
+
from .context import SemanticContext, current_context
|
|
7
|
+
from .csv_inference import infer_csv
|
|
8
|
+
from .detectors import (
|
|
9
|
+
STRUCTURAL_ROLES,
|
|
10
|
+
ContentProfile,
|
|
11
|
+
by_content_pattern,
|
|
12
|
+
by_fuzzy,
|
|
13
|
+
by_gazetteer,
|
|
14
|
+
by_strong_token,
|
|
15
|
+
by_structural_signature,
|
|
16
|
+
by_table_context,
|
|
17
|
+
)
|
|
18
|
+
from .evidence import DOMAIN_AXIS, ROLE_AXIS, Evidence, choose, rank
|
|
19
|
+
from .profiles import available_profiles, load_profile, temporary_profile
|
|
20
|
+
from .sensitivity import assess_sensitivity
|
|
21
|
+
from .tokens import normalize, tokenize
|
|
22
|
+
from .vocabularies import export_overrides_template, load_vocabularies, temporary_vocabulary
|
|
23
|
+
|
|
24
|
+
__version__ = "0.3.1"
|
|
25
|
+
|
|
26
|
+
__all__ = [
|
|
27
|
+
"ContentProfile",
|
|
28
|
+
"SemanticContext",
|
|
29
|
+
"assess_sensitivity",
|
|
30
|
+
"available_profiles",
|
|
31
|
+
"calibrate",
|
|
32
|
+
"current_context",
|
|
33
|
+
"export_overrides_template",
|
|
34
|
+
"infer_column",
|
|
35
|
+
"infer_csv",
|
|
36
|
+
"infer_table",
|
|
37
|
+
"load_profile",
|
|
38
|
+
"load_vocabularies",
|
|
39
|
+
"normalize",
|
|
40
|
+
"requires_review",
|
|
41
|
+
"temporary_profile",
|
|
42
|
+
"temporary_vocabulary",
|
|
43
|
+
"tokenize",
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
_DEPRECATED_ALIASES = {"normalizar": "normalize", "tokenizar": "tokenize"}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def __getattr__(name: str) -> Any:
|
|
50
|
+
if name in _DEPRECATED_ALIASES:
|
|
51
|
+
replacement = _DEPRECATED_ALIASES[name]
|
|
52
|
+
warnings.warn(
|
|
53
|
+
f"colsemantics.{name} is deprecated and will be removed in 1.0; use colsemantics.{replacement}",
|
|
54
|
+
DeprecationWarning,
|
|
55
|
+
stacklevel=2,
|
|
56
|
+
)
|
|
57
|
+
return globals()[replacement]
|
|
58
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
_CONTEXT_MINIMUM_CONFIDENCE = 0.7
|
|
62
|
+
_DOMAIN_MINIMUM_CONFIDENCE = 0.5
|
|
63
|
+
_MAX_HYPOTHESES = 4
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _collect_evidence(
|
|
67
|
+
column_name: str, detected_pattern: str, profile: ContentProfile | None
|
|
68
|
+
) -> list[Evidence]:
|
|
69
|
+
tokens = tokenize(column_name)
|
|
70
|
+
normalized_name = normalize(column_name)
|
|
71
|
+
evidence_items = (
|
|
72
|
+
by_content_pattern(detected_pattern)
|
|
73
|
+
+ by_strong_token(tokens)
|
|
74
|
+
+ by_fuzzy(normalized_name, tokens)
|
|
75
|
+
)
|
|
76
|
+
if profile is not None:
|
|
77
|
+
evidence_items += by_gazetteer(profile) + by_structural_signature(profile)
|
|
78
|
+
return evidence_items
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _refine_role(
|
|
82
|
+
role: str | None, domain: str | None, profile: ContentProfile | None
|
|
83
|
+
) -> str | None:
|
|
84
|
+
if (
|
|
85
|
+
role == config.PERSON_NAME_SEMANTIC
|
|
86
|
+
and domain is not None
|
|
87
|
+
and domain not in config.PERSON_DOMAINS
|
|
88
|
+
):
|
|
89
|
+
return config.ENTITY_LABEL_SEMANTIC
|
|
90
|
+
if role == config.FREE_FORM_TEXT_SEMANTIC and profile is not None:
|
|
91
|
+
is_dimension_cardinality = (
|
|
92
|
+
1 < profile.distinct_count <= config.MAX_CATEGORY_CARDINALITY
|
|
93
|
+
and profile.uniqueness_ratio < 0.5
|
|
94
|
+
)
|
|
95
|
+
if is_dimension_cardinality:
|
|
96
|
+
return config.CATEGORY_SEMANTIC
|
|
97
|
+
return role
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _build_result(
|
|
101
|
+
evidence_items: list[Evidence],
|
|
102
|
+
profile: ContentProfile | None = None,
|
|
103
|
+
detected_pattern: str = "None",
|
|
104
|
+
) -> dict[str, Any]:
|
|
105
|
+
role_ranking = rank(evidence_items, ROLE_AXIS)
|
|
106
|
+
domain_ranking = rank(evidence_items, DOMAIN_AXIS)
|
|
107
|
+
role, role_confidence, role_source, role_conclusive = choose(role_ranking)
|
|
108
|
+
domain, domain_confidence, domain_source, _ = choose(domain_ranking)
|
|
109
|
+
uncertain_domain = domain is not None and domain_confidence < _DOMAIN_MINIMUM_CONFIDENCE
|
|
110
|
+
if uncertain_domain:
|
|
111
|
+
domain, domain_confidence, domain_source = None, 0.0, "No evidence"
|
|
112
|
+
role = _refine_role(role, domain, profile)
|
|
113
|
+
if role in STRUCTURAL_ROLES:
|
|
114
|
+
semantic, raw_confidence, source = role, role_confidence, role_source
|
|
115
|
+
elif domain is not None:
|
|
116
|
+
semantic, raw_confidence, source = domain, domain_confidence, domain_source
|
|
117
|
+
elif role is not None:
|
|
118
|
+
semantic, raw_confidence, source = role, role_confidence, role_source
|
|
119
|
+
else:
|
|
120
|
+
semantic, raw_confidence, source = config.GENERIC_SEMANTIC, 0.0, "Unmatched"
|
|
121
|
+
hypotheses = sorted(
|
|
122
|
+
[
|
|
123
|
+
{
|
|
124
|
+
"semantic": item["category"],
|
|
125
|
+
"axis": axis,
|
|
126
|
+
"confidence": item["confidence"],
|
|
127
|
+
"evidence": item["sources"][:3],
|
|
128
|
+
}
|
|
129
|
+
for axis, ranking in ((ROLE_AXIS, role_ranking), (DOMAIN_AXIS, domain_ranking))
|
|
130
|
+
for item in ranking
|
|
131
|
+
],
|
|
132
|
+
key=lambda hypothesis: -hypothesis["confidence"],
|
|
133
|
+
)[:_MAX_HYPOTHESES]
|
|
134
|
+
confidence = calibrate(round(raw_confidence, 4), current_context().profile_name)
|
|
135
|
+
return {
|
|
136
|
+
"semantic": semantic,
|
|
137
|
+
"role": role,
|
|
138
|
+
"domain": domain,
|
|
139
|
+
"raw_confidence": round(raw_confidence, 4),
|
|
140
|
+
"confidence": confidence,
|
|
141
|
+
"evidence": source,
|
|
142
|
+
"conclusive": not (bool(role_ranking) and not role_conclusive) and not uncertain_domain,
|
|
143
|
+
"review_required": requires_review(confidence),
|
|
144
|
+
"sensitivity": assess_sensitivity(semantic, detected_pattern),
|
|
145
|
+
"hypotheses": hypotheses,
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def infer_column(
|
|
150
|
+
column_name: str, detected_pattern: str = "None", profile: ContentProfile | None = None
|
|
151
|
+
) -> dict[str, Any]:
|
|
152
|
+
override = current_context().column_overrides.get(column_name)
|
|
153
|
+
if override:
|
|
154
|
+
axis = ROLE_AXIS if override in STRUCTURAL_ROLES else DOMAIN_AXIS
|
|
155
|
+
result = _build_result(
|
|
156
|
+
[Evidence(override, axis, 1.0, "vocabulary override")], profile, detected_pattern
|
|
157
|
+
)
|
|
158
|
+
result["conclusive"] = True
|
|
159
|
+
return result
|
|
160
|
+
return _build_result(
|
|
161
|
+
_collect_evidence(column_name, detected_pattern, profile), profile, detected_pattern
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def infer_table(columns: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
166
|
+
evidence_by_column: list[list[Evidence]] = []
|
|
167
|
+
results: list[dict[str, Any]] = []
|
|
168
|
+
for column in columns:
|
|
169
|
+
name = str(column.get("column_name", column.get("name")))
|
|
170
|
+
profile = column.get("profile")
|
|
171
|
+
if profile is not None and not isinstance(profile, ContentProfile):
|
|
172
|
+
raise TypeError("profile must be a ContentProfile instance")
|
|
173
|
+
override = current_context().column_overrides.get(name)
|
|
174
|
+
detected_pattern = str(column.get("detected_pattern", column.get("pattern", "None")))
|
|
175
|
+
evidence_items = (
|
|
176
|
+
[
|
|
177
|
+
Evidence(
|
|
178
|
+
override,
|
|
179
|
+
ROLE_AXIS if override in STRUCTURAL_ROLES else DOMAIN_AXIS,
|
|
180
|
+
1.0,
|
|
181
|
+
"vocabulary override",
|
|
182
|
+
)
|
|
183
|
+
]
|
|
184
|
+
if override
|
|
185
|
+
else _collect_evidence(name, detected_pattern, profile)
|
|
186
|
+
)
|
|
187
|
+
evidence_by_column.append(evidence_items)
|
|
188
|
+
result = _build_result(evidence_items, profile, detected_pattern)
|
|
189
|
+
if override:
|
|
190
|
+
result["conclusive"] = True
|
|
191
|
+
results.append(result)
|
|
192
|
+
table_domains = _subject_profile(results)
|
|
193
|
+
if not table_domains:
|
|
194
|
+
return results
|
|
195
|
+
for index, (column, result) in enumerate(zip(columns, results, strict=True)):
|
|
196
|
+
if result["conclusive"]:
|
|
197
|
+
continue
|
|
198
|
+
name = str(column.get("column_name", column.get("name")))
|
|
199
|
+
extras = by_table_context(tokenize(name), table_domains)
|
|
200
|
+
if extras:
|
|
201
|
+
detected_pattern = str(column.get("detected_pattern", column.get("pattern", "None")))
|
|
202
|
+
results[index] = _build_result(
|
|
203
|
+
evidence_by_column[index] + extras, column.get("profile"), detected_pattern
|
|
204
|
+
)
|
|
205
|
+
return results
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _subject_profile(results: list[dict[str, Any]]) -> dict[str, float]:
|
|
209
|
+
strengths: dict[str, float] = {}
|
|
210
|
+
for result in results:
|
|
211
|
+
if not result["conclusive"]:
|
|
212
|
+
continue
|
|
213
|
+
for category in (result["role"], result["domain"]):
|
|
214
|
+
if (
|
|
215
|
+
category
|
|
216
|
+
and category != config.GENERIC_SEMANTIC
|
|
217
|
+
and result["raw_confidence"] >= _CONTEXT_MINIMUM_CONFIDENCE
|
|
218
|
+
):
|
|
219
|
+
strengths[category] = max(strengths.get(category, 0.0), result["raw_confidence"])
|
|
220
|
+
return strengths
|