tabalyst 0.4.0__tar.gz → 0.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tabalyst-0.4.0/src/tabalyst.egg-info → tabalyst-0.4.2}/PKG-INFO +5 -2
- {tabalyst-0.4.0 → tabalyst-0.4.2}/README.md +4 -1
- {tabalyst-0.4.0 → tabalyst-0.4.2}/pyproject.toml +1 -1
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/cli/report.py +12 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/cli/scan.py +12 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/models.py +1 -1
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/report_service.py +8 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/reporting.py +14 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scan_service.py +6 -2
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/api.py +35 -4
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/config.py +9 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/base.py +30 -4
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/boolean.py +10 -1
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/currency.py +11 -1
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/email.py +6 -1
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/enumeration.py +12 -2
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/ip.py +13 -1
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/number.py +23 -1
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/pattern.py +7 -1
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/percentage.py +5 -1
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/phone.py +13 -1
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/postal.py +6 -1
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/quantity.py +6 -1
- tabalyst-0.4.2/src/tabalyst/scanner/detectors/registry.py +832 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/temporal.py +19 -1
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/url.py +9 -1
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/uuid.py +6 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/engine.py +22 -1
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/exposure.py +18 -1
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/field.py +53 -13
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/measures.py +139 -104
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/models.py +15 -1
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/normalization.py +43 -9
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/observations.py +41 -6
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/readers/csv_reader.py +24 -14
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/readers/json_reader.py +32 -4
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/records.py +167 -8
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/structure.py +106 -11
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/technical.py +15 -22
- tabalyst-0.4.2/src/tabalyst/scanner/values.py +1022 -0
- tabalyst-0.4.2/src/tabalyst/scanner/workers.py +391 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/service.py +12 -5
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/static/theme.css +4 -3
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/templates/report.html +2 -2
- {tabalyst-0.4.0 → tabalyst-0.4.2/src/tabalyst.egg-info}/PKG-INFO +5 -2
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst.egg-info/SOURCES.txt +2 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_cli.py +2 -2
- {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_reporting.py +19 -1
- {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_scan_command.py +55 -0
- tabalyst-0.4.2/tests/test_scanner_batches.py +248 -0
- tabalyst-0.4.0/src/tabalyst/scanner/detectors/registry.py +0 -445
- tabalyst-0.4.0/src/tabalyst/scanner/values.py +0 -558
- {tabalyst-0.4.0 → tabalyst-0.4.2}/LICENSE +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/MANIFEST.in +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/examples/README.md +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/examples/config.json +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/examples/input/basic.csv +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/examples/input/insurance-customers.csv +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/setup.cfg +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/__init__.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/__main__.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/_version.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/batch.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/cli/__init__.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/cli/app.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/cli/sample.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/cli/terminal.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/config.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/errors.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/execution_log.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/progress.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/report_config.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/report_profile.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/sampling.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/sampling_service.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scan_reuse.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/__init__.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/__init__.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/amount.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/names.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/shape.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/diagnostics.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/paths.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/readers/__init__.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/readers/base.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/static/report.js +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst.egg-info/dependency_links.txt +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst.egg-info/entry_points.txt +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst.egg-info/requires.txt +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst.egg-info/top_level.txt +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_public_api.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_sampling.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_scanner_amounts.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_scanner_config.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_scanner_csv.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_scanner_email_url.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_scanner_exposure.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_scanner_paths.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_scanner_phone.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_scanner_postal.py +0 -0
- {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_scanner_uuid_ip.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tabalyst
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.2
|
|
4
4
|
Summary: Open-source toolkit for understanding and working with unfamiliar data.
|
|
5
5
|
Author: Gregory Borelli
|
|
6
6
|
License-Expression: MIT
|
|
@@ -181,6 +181,7 @@ tabalyst report data.csv --delimiter ";"
|
|
|
181
181
|
tabalyst report data.csv --encoding cp1252
|
|
182
182
|
tabalyst report data.csv --config tabalyst.json
|
|
183
183
|
tabalyst report data.csv --verbose
|
|
184
|
+
tabalyst report data.csv --workers 4
|
|
184
185
|
tabalyst report --help
|
|
185
186
|
tabalyst --version
|
|
186
187
|
```
|
|
@@ -233,7 +234,9 @@ detector: numbers with decimal commas, dates, booleans, enumerations, email
|
|
|
233
234
|
addresses, URLs, phone numbers, postal codes, currency amounts, percentages,
|
|
234
235
|
quantities, UUIDs and IP addresses. Values of sensitive fields, such as email
|
|
235
236
|
addresses, are masked by default. Results are written atomically: an
|
|
236
|
-
interrupted scan never leaves a partial file.
|
|
237
|
+
interrupted scan never leaves a partial file. Files of 16 MiB or more are
|
|
238
|
+
analyzed by several worker processes, with the same result; `--workers`
|
|
239
|
+
chooses their number, `--workers 1` keeps one process.
|
|
237
240
|
|
|
238
241
|
Build the report from a scan document without reading the source again:
|
|
239
242
|
|
|
@@ -147,6 +147,7 @@ tabalyst report data.csv --delimiter ";"
|
|
|
147
147
|
tabalyst report data.csv --encoding cp1252
|
|
148
148
|
tabalyst report data.csv --config tabalyst.json
|
|
149
149
|
tabalyst report data.csv --verbose
|
|
150
|
+
tabalyst report data.csv --workers 4
|
|
150
151
|
tabalyst report --help
|
|
151
152
|
tabalyst --version
|
|
152
153
|
```
|
|
@@ -199,7 +200,9 @@ detector: numbers with decimal commas, dates, booleans, enumerations, email
|
|
|
199
200
|
addresses, URLs, phone numbers, postal codes, currency amounts, percentages,
|
|
200
201
|
quantities, UUIDs and IP addresses. Values of sensitive fields, such as email
|
|
201
202
|
addresses, are masked by default. Results are written atomically: an
|
|
202
|
-
interrupted scan never leaves a partial file.
|
|
203
|
+
interrupted scan never leaves a partial file. Files of 16 MiB or more are
|
|
204
|
+
analyzed by several worker processes, with the same result; `--workers`
|
|
205
|
+
chooses their number, `--workers 1` keeps one process.
|
|
203
206
|
|
|
204
207
|
Build the report from a scan document without reading the source again:
|
|
205
208
|
|
|
@@ -143,6 +143,17 @@ def report_command(
|
|
|
143
143
|
help="Disable interactive file and phase progress.",
|
|
144
144
|
),
|
|
145
145
|
] = False,
|
|
146
|
+
workers: Annotated[
|
|
147
|
+
int | None,
|
|
148
|
+
typer.Option(
|
|
149
|
+
"--workers",
|
|
150
|
+
min=1,
|
|
151
|
+
help=(
|
|
152
|
+
"Processes that analyze values: 1 for one process. By default, "
|
|
153
|
+
"files of 16 MiB or more use one per spare processor."
|
|
154
|
+
),
|
|
155
|
+
),
|
|
156
|
+
] = None,
|
|
146
157
|
) -> None:
|
|
147
158
|
"""Analyze and profile datasets with Tabalyst Report."""
|
|
148
159
|
if quiet and verbose:
|
|
@@ -162,6 +173,7 @@ def report_command(
|
|
|
162
173
|
force=force,
|
|
163
174
|
on_progress=progress,
|
|
164
175
|
from_scan=from_scan,
|
|
176
|
+
workers=workers,
|
|
165
177
|
)
|
|
166
178
|
except TabalystError as exc:
|
|
167
179
|
typer.echo(f"Error: {exc}", err=True)
|
|
@@ -149,6 +149,17 @@ def scan_command(
|
|
|
149
149
|
bool,
|
|
150
150
|
typer.Option("--no-progress", help="Disable interactive progress."),
|
|
151
151
|
] = False,
|
|
152
|
+
workers: Annotated[
|
|
153
|
+
int | None,
|
|
154
|
+
typer.Option(
|
|
155
|
+
"--workers",
|
|
156
|
+
min=1,
|
|
157
|
+
help=(
|
|
158
|
+
"Processes that analyze values: 1 for one process. By default, "
|
|
159
|
+
"files of 16 MiB or more use one per spare processor."
|
|
160
|
+
),
|
|
161
|
+
),
|
|
162
|
+
] = None,
|
|
152
163
|
) -> None:
|
|
153
164
|
"""Scan CSV and JSON files into a complete JSON description of their data."""
|
|
154
165
|
if quiet and verbose:
|
|
@@ -168,6 +179,7 @@ def scan_command(
|
|
|
168
179
|
collections=collection,
|
|
169
180
|
force=force,
|
|
170
181
|
on_progress=progress,
|
|
182
|
+
workers=workers,
|
|
171
183
|
)
|
|
172
184
|
except TabalystError as exc:
|
|
173
185
|
typer.echo(f"Error: {exc}", err=True)
|
|
@@ -378,7 +378,7 @@ class DatasetProfile(ResultModel):
|
|
|
378
378
|
|
|
379
379
|
class ReportProfile(ResultModel):
|
|
380
380
|
format_version: Literal["0.1.0a"] = "0.1.0a"
|
|
381
|
-
format_revision: Literal[
|
|
381
|
+
format_revision: Literal[8] = 8
|
|
382
382
|
generated_at: datetime
|
|
383
383
|
processing_seconds: FiniteFloat
|
|
384
384
|
source: SourceInfo
|
|
@@ -175,18 +175,25 @@ def generate_reports(
|
|
|
175
175
|
force: bool = False,
|
|
176
176
|
on_progress: ProgressCallback | None = None,
|
|
177
177
|
from_scan: bool = False,
|
|
178
|
+
workers: int | None = None,
|
|
178
179
|
) -> BatchReportResult:
|
|
179
180
|
"""Plan and execute one or more reports without depending on the CLI.
|
|
180
181
|
|
|
181
182
|
With ``from_scan``, the inputs are scan documents written by
|
|
182
183
|
``tabalyst scan``: each report is built from its document without reading
|
|
183
184
|
the source again, after the staleness checks of ``tabalyst.scan_reuse``.
|
|
185
|
+
Otherwise ``workers`` is passed to ``tabalyst.scan()``.
|
|
184
186
|
"""
|
|
185
187
|
if from_scan and (separator is not None or encoding is not None):
|
|
186
188
|
raise ConfigurationError(
|
|
187
189
|
"--delimiter and --encoding cannot be used with --scan: the scan "
|
|
188
190
|
"document records how its source was read."
|
|
189
191
|
)
|
|
192
|
+
if from_scan and workers is not None:
|
|
193
|
+
raise ConfigurationError(
|
|
194
|
+
"--workers cannot be used with --scan: the report reads the scan "
|
|
195
|
+
"document, not its source."
|
|
196
|
+
)
|
|
190
197
|
plan = build_report_plan(
|
|
191
198
|
input_specs,
|
|
192
199
|
output=output,
|
|
@@ -245,6 +252,7 @@ def generate_reports(
|
|
|
245
252
|
config_paths,
|
|
246
253
|
force=force,
|
|
247
254
|
on_progress=forward,
|
|
255
|
+
workers=workers,
|
|
248
256
|
)
|
|
249
257
|
except TabalystError as exc:
|
|
250
258
|
failures.append(ReportFailure(job=job, error=exc))
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
"""Render self-contained analysis results as an interactive HTML report."""
|
|
2
2
|
|
|
3
|
+
import re
|
|
3
4
|
from collections import defaultdict
|
|
4
5
|
from importlib.resources import files
|
|
5
6
|
from pathlib import Path
|
|
6
7
|
|
|
7
8
|
from jinja2 import Environment, StrictUndefined
|
|
9
|
+
from markupsafe import Markup, escape
|
|
8
10
|
|
|
9
11
|
from tabalyst._version import get_version
|
|
10
12
|
from tabalyst.models import ReportProfile
|
|
@@ -55,6 +57,17 @@ def format_size(size_bytes: int) -> str:
|
|
|
55
57
|
return f"{size_bytes / 1024:.1f} KB"
|
|
56
58
|
|
|
57
59
|
|
|
60
|
+
# Separators after which a long name may break: snake case, paths and dates.
|
|
61
|
+
_BREAKS = re.compile(r"(?<=[_./-])")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def breakable(name: str) -> Markup:
|
|
65
|
+
"""``name`` escaped, with a line break opportunity after each separator,
|
|
66
|
+
so that a table squeezed for room narrows its column instead of
|
|
67
|
+
scrolling."""
|
|
68
|
+
return Markup("<wbr>").join(escape(part) for part in _BREAKS.split(name))
|
|
69
|
+
|
|
70
|
+
|
|
58
71
|
def load_profile(path: str | Path) -> ReportProfile:
|
|
59
72
|
"""Validate and load the experimental JSON profile format."""
|
|
60
73
|
return ReportProfile.model_validate_json(Path(path).read_text(encoding="utf-8"))
|
|
@@ -68,6 +81,7 @@ def render_report(profile: ReportProfile) -> str:
|
|
|
68
81
|
environment.filters["number"] = format_number
|
|
69
82
|
environment.filters["seconds"] = format_seconds
|
|
70
83
|
environment.filters["size"] = format_size
|
|
84
|
+
environment.filters["breakable"] = breakable
|
|
71
85
|
template = environment.from_string(
|
|
72
86
|
resources.joinpath("templates/report.html").read_text(encoding="utf-8")
|
|
73
87
|
)
|
|
@@ -106,6 +106,7 @@ def generate_scans(
|
|
|
106
106
|
collections: Sequence[str] | None = None,
|
|
107
107
|
force: bool = False,
|
|
108
108
|
on_progress: ProgressCallback | None = None,
|
|
109
|
+
workers: int | None = None,
|
|
109
110
|
) -> BatchScanResult:
|
|
110
111
|
"""Scan one or more CSV or JSON files and write one scan document each.
|
|
111
112
|
|
|
@@ -113,7 +114,8 @@ def generate_scans(
|
|
|
113
114
|
the ``scan`` section of each ``config_path`` file in order, then
|
|
114
115
|
``delimiter``, ``encoding`` and ``collections``. The whole batch is
|
|
115
116
|
rejected before any scan when an output is unsafe; afterwards, a failed
|
|
116
|
-
source does not stop the others.
|
|
117
|
+
source does not stop the others. ``workers`` is passed to
|
|
118
|
+
``tabalyst.scan()``.
|
|
117
119
|
"""
|
|
118
120
|
config_paths = _config_paths(config_path)
|
|
119
121
|
config = resolve_scan_config(
|
|
@@ -151,7 +153,9 @@ def generate_scans(
|
|
|
151
153
|
)
|
|
152
154
|
|
|
153
155
|
try:
|
|
154
|
-
result = scan(
|
|
156
|
+
result = scan(
|
|
157
|
+
job.source, config=config, on_progress=forward, workers=workers
|
|
158
|
+
)
|
|
155
159
|
emit_progress(
|
|
156
160
|
on_progress, job.source, ProgressPhase.WRITING, index=index, total=total
|
|
157
161
|
)
|
|
@@ -8,7 +8,7 @@ from pathlib import Path
|
|
|
8
8
|
from time import perf_counter
|
|
9
9
|
|
|
10
10
|
from tabalyst._version import __version__
|
|
11
|
-
from tabalyst.errors import InputError
|
|
11
|
+
from tabalyst.errors import ConfigurationError, InputError
|
|
12
12
|
from tabalyst.progress import ProgressCallback, ProgressPhase, emit_progress
|
|
13
13
|
from tabalyst.scanner.config import ScanConfig, config_sha256
|
|
14
14
|
from tabalyst.scanner.detectors.registry import (
|
|
@@ -25,11 +25,17 @@ from tabalyst.scanner.models import (
|
|
|
25
25
|
Scope,
|
|
26
26
|
SourceInfo,
|
|
27
27
|
)
|
|
28
|
-
from tabalyst.scanner.observations import
|
|
28
|
+
from tabalyst.scanner.observations import (
|
|
29
|
+
DatasetOpened,
|
|
30
|
+
Record,
|
|
31
|
+
RecordBatch,
|
|
32
|
+
StreamItem,
|
|
33
|
+
)
|
|
29
34
|
from tabalyst.scanner.paths import format_absolute, parse_path
|
|
30
35
|
from tabalyst.scanner.readers.base import Reader
|
|
31
36
|
from tabalyst.scanner.readers.csv_reader import CsvReader
|
|
32
37
|
from tabalyst.scanner.readers.json_reader import JsonReader
|
|
38
|
+
from tabalyst.scanner.workers import WorkerPool, worker_count
|
|
33
39
|
|
|
34
40
|
NORMALIZATION_VERSION = 1
|
|
35
41
|
# Reading progress is reported at most once per percent of the source, and
|
|
@@ -86,9 +92,17 @@ def _collection_scope(source_format: str, config: ScanConfig) -> CollectionScope
|
|
|
86
92
|
def _observed(
|
|
87
93
|
items: Iterable[StreamItem], on_record: Callable[[Record], None]
|
|
88
94
|
) -> Iterator[StreamItem]:
|
|
95
|
+
"""Items as read, after passing each analyzed record to ``on_record``;
|
|
96
|
+
records of a batch are passed one by one, as ``Record`` items."""
|
|
97
|
+
declared = {}
|
|
89
98
|
for item in items:
|
|
90
99
|
if type(item) is Record:
|
|
91
100
|
on_record(item)
|
|
101
|
+
elif type(item) is RecordBatch:
|
|
102
|
+
for record in item.records(declared[item.dataset]):
|
|
103
|
+
on_record(record)
|
|
104
|
+
elif type(item) is DatasetOpened:
|
|
105
|
+
declared[item.dataset] = [field.path for field in item.fields]
|
|
92
106
|
yield item
|
|
93
107
|
|
|
94
108
|
|
|
@@ -99,6 +113,7 @@ def scan(
|
|
|
99
113
|
registry: DetectorRegistry | None = None,
|
|
100
114
|
on_progress: ProgressCallback | None = None,
|
|
101
115
|
on_record: Callable[[Record], None] | None = None,
|
|
116
|
+
workers: int | None = None,
|
|
102
117
|
) -> ScanResult:
|
|
103
118
|
"""Read one source completely and return its finalized scan result.
|
|
104
119
|
|
|
@@ -107,7 +122,14 @@ def scan(
|
|
|
107
122
|
order, before the engine: consumers that need record-level facts, such
|
|
108
123
|
as the report's preview and duplicate rows, get them in the same pass. It
|
|
109
124
|
must not modify the record.
|
|
125
|
+
|
|
126
|
+
``workers`` is the number of processes that analyze values: ``1`` scans
|
|
127
|
+
in this process; by default, large sources use one worker per spare
|
|
128
|
+
processor (``workers.worker_count``). The result does not depend on it.
|
|
129
|
+
A custom ``registry`` always scans in this process.
|
|
110
130
|
"""
|
|
131
|
+
if workers is not None and workers < 1:
|
|
132
|
+
raise ConfigurationError("workers must be at least 1")
|
|
111
133
|
path = Path(source)
|
|
112
134
|
# A private copy: the finalized result must not share state with the caller.
|
|
113
135
|
config = ScanConfig() if config is None else config.model_copy(deep=True)
|
|
@@ -130,8 +152,17 @@ def scan(
|
|
|
130
152
|
default_registry() if registry is None else registry, config
|
|
131
153
|
)
|
|
132
154
|
engine = ScanEngine(config, detectors)
|
|
133
|
-
|
|
134
|
-
|
|
155
|
+
count = 1 if registry is not None else worker_count(workers, stat.st_size)
|
|
156
|
+
if count > 1:
|
|
157
|
+
pool = WorkerPool(config.model_dump_json(by_alias=True), count)
|
|
158
|
+
if pool.start():
|
|
159
|
+
engine.values.pool = pool
|
|
160
|
+
try:
|
|
161
|
+
engine.consume(reader if on_record is None else _observed(reader, on_record))
|
|
162
|
+
datasets = engine.finalize()
|
|
163
|
+
finally:
|
|
164
|
+
if engine.values.pool is not None:
|
|
165
|
+
engine.values.pool.close()
|
|
135
166
|
summary = reader.summary()
|
|
136
167
|
|
|
137
168
|
excluded = engine.records_excluded
|
|
@@ -52,6 +52,9 @@ MAX_PREVIEW_RECORDS = 1_000
|
|
|
52
52
|
MAX_PATTERNS = 200
|
|
53
53
|
MAX_PATTERN_LENGTH = 1_000
|
|
54
54
|
MAX_PATTERN_INPUT_LENGTH = 10_000
|
|
55
|
+
# A streamed field keeps its warm-up values: the warm-up is bounded.
|
|
56
|
+
MAX_WARMUP_VALUES = 1_000_000
|
|
57
|
+
MAX_PROBE_INTERVAL = 1_000_000
|
|
55
58
|
|
|
56
59
|
MissingCategory = Literal["absent", "null", "empty", "blank", "marker"]
|
|
57
60
|
ScalarType = Literal["string", "integer", "number", "boolean"]
|
|
@@ -176,6 +179,12 @@ class TypeSettings(_Settings):
|
|
|
176
179
|
|
|
177
180
|
class DetectionSettings(_Settings):
|
|
178
181
|
minimum_share: float = Field(default=0.95, gt=0, le=1)
|
|
182
|
+
# Adaptive detection (design 13): 0 keeps detection exhaustive.
|
|
183
|
+
warmup_values: int = Field(default=10_000, ge=0, le=MAX_WARMUP_VALUES)
|
|
184
|
+
probe_interval: int = Field(default=100, ge=0, le=MAX_PROBE_INTERVAL)
|
|
185
|
+
# Detectors that reacted to at most this share of the warm-up values are
|
|
186
|
+
# skipped too; 0 skips only those that reacted to none.
|
|
187
|
+
rare_share: float = Field(default=0.001, ge=0, le=1)
|
|
179
188
|
|
|
180
189
|
|
|
181
190
|
NumberConvention = Literal["dot", "comma"]
|
|
@@ -10,9 +10,9 @@ accumulator adds its own ``details``.
|
|
|
10
10
|
from __future__ import annotations
|
|
11
11
|
|
|
12
12
|
import re
|
|
13
|
-
from collections.abc import Mapping
|
|
13
|
+
from collections.abc import Mapping, Sequence
|
|
14
14
|
from dataclasses import dataclass
|
|
15
|
-
from typing import TYPE_CHECKING, ClassVar, Literal
|
|
15
|
+
from typing import TYPE_CHECKING, ClassVar, Literal, NamedTuple
|
|
16
16
|
|
|
17
17
|
if TYPE_CHECKING:
|
|
18
18
|
from tabalyst.scanner.exposure import ExposureGate
|
|
@@ -20,8 +20,7 @@ if TYPE_CHECKING:
|
|
|
20
20
|
State = Literal["matched", "ambiguous", "invalid"]
|
|
21
21
|
|
|
22
22
|
|
|
23
|
-
|
|
24
|
-
class Classification:
|
|
23
|
+
class Classification(NamedTuple):
|
|
25
24
|
"""Result of a detector for one value; ``None`` stands for not matched.
|
|
26
25
|
|
|
27
26
|
- ``matched`` values may carry a ``format`` and a parsed ``value`` for
|
|
@@ -34,6 +33,9 @@ class Classification:
|
|
|
34
33
|
- ``convention`` names the reading of an unambiguous value when several
|
|
35
34
|
readings exist (date order, decimal convention); it feeds the ambiguity
|
|
36
35
|
evidence of the field (design 12.6).
|
|
36
|
+
|
|
37
|
+
An immutable named tuple: detectors build one per matched value, and a
|
|
38
|
+
tuple is built about twice as fast as a frozen dataclass.
|
|
37
39
|
"""
|
|
38
40
|
|
|
39
41
|
state: State
|
|
@@ -69,9 +71,15 @@ class DetectorAccumulator:
|
|
|
69
71
|
``add`` receives every tested value with its classification (``None`` when
|
|
70
72
|
not matched) and its count. In both execution modes it sees the same
|
|
71
73
|
distinct values, first seen first, with the same total counts.
|
|
74
|
+
|
|
75
|
+
A class whose ``add`` does nothing for unmatched values (``None``) sets
|
|
76
|
+
``ignores_unmatched`` in its own body: the engine then skips those calls.
|
|
77
|
+
The flag applies only to the class that defines ``add``, so a subclass
|
|
78
|
+
that overrides ``add`` receives every value unless it sets it again.
|
|
72
79
|
"""
|
|
73
80
|
|
|
74
81
|
__slots__ = ()
|
|
82
|
+
ignores_unmatched: ClassVar[bool] = True
|
|
75
83
|
|
|
76
84
|
def add(
|
|
77
85
|
self, value: str, classification: Classification | None, count: int
|
|
@@ -90,6 +98,14 @@ class DetectorAccumulator:
|
|
|
90
98
|
"""Field-level detectors decide at the end whether their values match."""
|
|
91
99
|
return True
|
|
92
100
|
|
|
101
|
+
def rejects_field(self) -> bool:
|
|
102
|
+
"""Whether a field-level detector already knows that the field cannot
|
|
103
|
+
match, whatever values follow, and that ``add`` no longer changes
|
|
104
|
+
``details``. The engine then counts later values as not matched
|
|
105
|
+
without classifying them, which is exact only for detectors whose
|
|
106
|
+
``classify`` never returns ``ambiguous`` or ``invalid``."""
|
|
107
|
+
return False
|
|
108
|
+
|
|
93
109
|
|
|
94
110
|
class Detector:
|
|
95
111
|
"""Base class of detectors; subclasses set the class attributes and
|
|
@@ -104,6 +120,11 @@ class Detector:
|
|
|
104
120
|
- ``max_input_length``: longer values are ``not_tested``.
|
|
105
121
|
- ``scope``: ``field`` detectors, such as enumeration candidates, decide
|
|
106
122
|
at the end whether every value of the field matches or none does.
|
|
123
|
+
|
|
124
|
+
The engine classifies distinct values in batches through
|
|
125
|
+
``classify_many``, which maps ``classify`` by default. A detector may
|
|
126
|
+
override it to reject most values without a call per value, provided it
|
|
127
|
+
returns exactly what ``classify`` returns for each value.
|
|
107
128
|
"""
|
|
108
129
|
|
|
109
130
|
id: ClassVar[str]
|
|
@@ -121,6 +142,10 @@ class Detector:
|
|
|
121
142
|
def classify(self, value: str) -> Classification | None:
|
|
122
143
|
raise NotImplementedError
|
|
123
144
|
|
|
145
|
+
def classify_many(self, values: Sequence[str]) -> list[Classification | None]:
|
|
146
|
+
"""``classify`` of each value, in order."""
|
|
147
|
+
return list(map(self.classify, values))
|
|
148
|
+
|
|
124
149
|
def accumulator(self) -> DetectorAccumulator:
|
|
125
150
|
return DetectorAccumulator()
|
|
126
151
|
|
|
@@ -135,6 +160,7 @@ class AmbiguityAccumulator(DetectorAccumulator):
|
|
|
135
160
|
"""
|
|
136
161
|
|
|
137
162
|
__slots__ = ("candidates", "count", "evidence", "resolution")
|
|
163
|
+
ignores_unmatched = True
|
|
138
164
|
|
|
139
165
|
def __init__(self, readings: list[str], resolution: dict[str, str] | None) -> None:
|
|
140
166
|
self.count = 0
|
|
@@ -7,7 +7,7 @@ booleans stay in the ``booleans`` block.
|
|
|
7
7
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
|
-
from collections.abc import Mapping
|
|
10
|
+
from collections.abc import Mapping, Sequence
|
|
11
11
|
|
|
12
12
|
from tabalyst.scanner.detectors.base import (
|
|
13
13
|
Classification,
|
|
@@ -45,12 +45,21 @@ class BooleanDetector(Detector):
|
|
|
45
45
|
return None
|
|
46
46
|
return self._words.get(value.casefold())
|
|
47
47
|
|
|
48
|
+
def classify_many(self, values: Sequence[str]) -> list[Classification | None]:
|
|
49
|
+
words = self._words
|
|
50
|
+
longest = self._longest
|
|
51
|
+
return [
|
|
52
|
+
words.get(value.casefold()) if len(value) <= longest else None
|
|
53
|
+
for value in values
|
|
54
|
+
]
|
|
55
|
+
|
|
48
56
|
def accumulator(self) -> BooleanAccumulator:
|
|
49
57
|
return BooleanAccumulator()
|
|
50
58
|
|
|
51
59
|
|
|
52
60
|
class BooleanAccumulator(DetectorAccumulator):
|
|
53
61
|
__slots__ = ("false", "true")
|
|
62
|
+
ignores_unmatched = True
|
|
54
63
|
|
|
55
64
|
def __init__(self) -> None:
|
|
56
65
|
self.true = 0
|
|
@@ -8,7 +8,7 @@ of money. The number part follows the number detector (``amount.py``).
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
10
|
import re
|
|
11
|
-
from collections.abc import Mapping
|
|
11
|
+
from collections.abc import Mapping, Sequence
|
|
12
12
|
|
|
13
13
|
from tabalyst.scanner.detectors.amount import (
|
|
14
14
|
NUMBER_PART,
|
|
@@ -107,12 +107,22 @@ class CurrencyDetector(Detector):
|
|
|
107
107
|
sign, _, body, prefix, suffix = parts
|
|
108
108
|
return self.reader.read(sign, body, prefix, suffix)
|
|
109
109
|
|
|
110
|
+
def classify_many(self, values: Sequence[str]) -> list[Classification | None]:
|
|
111
|
+
classify = self.classify
|
|
112
|
+
return [
|
|
113
|
+
classify(value)
|
|
114
|
+
if value[:1] in _FIRST or value[-1:] in _LAST
|
|
115
|
+
else None
|
|
116
|
+
for value in values
|
|
117
|
+
]
|
|
118
|
+
|
|
110
119
|
def accumulator(self) -> CurrencyAccumulator:
|
|
111
120
|
return CurrencyAccumulator(self.reader)
|
|
112
121
|
|
|
113
122
|
|
|
114
123
|
class CurrencyAccumulator(AmountAccumulator):
|
|
115
124
|
__slots__ = ("currencies",)
|
|
125
|
+
ignores_unmatched = True
|
|
116
126
|
|
|
117
127
|
def __init__(self, reader: AmountReader) -> None:
|
|
118
128
|
super().__init__(reader)
|
|
@@ -8,7 +8,7 @@ masked by default (design 12.8).
|
|
|
8
8
|
from __future__ import annotations
|
|
9
9
|
|
|
10
10
|
import re
|
|
11
|
-
from collections.abc import Mapping
|
|
11
|
+
from collections.abc import Mapping, Sequence
|
|
12
12
|
|
|
13
13
|
from tabalyst.scanner.detectors.base import (
|
|
14
14
|
Classification,
|
|
@@ -66,12 +66,17 @@ class EmailDetector(Detector):
|
|
|
66
66
|
return _BAD_DOMAIN
|
|
67
67
|
return Classification("matched", value=domain.lower())
|
|
68
68
|
|
|
69
|
+
def classify_many(self, values: Sequence[str]) -> list[Classification | None]:
|
|
70
|
+
classify = self.classify
|
|
71
|
+
return [classify(value) if "@" in value else None for value in values]
|
|
72
|
+
|
|
69
73
|
def accumulator(self) -> EmailAccumulator:
|
|
70
74
|
return EmailAccumulator(NameCounts(self.max_tracked, self.max_listed))
|
|
71
75
|
|
|
72
76
|
|
|
73
77
|
class EmailAccumulator(DetectorAccumulator):
|
|
74
78
|
__slots__ = ("domains",)
|
|
79
|
+
ignores_unmatched = True
|
|
75
80
|
|
|
76
81
|
def __init__(self, domains: NameCounts) -> None:
|
|
77
82
|
self.domains = domains
|
|
@@ -5,12 +5,14 @@ A field-level detector: its values match together when the field has at least
|
|
|
5
5
|
``minimum_values`` eligible values and at most ``maximum_distinct`` distinct
|
|
6
6
|
analytical values (ignoring case when ``case_sensitive`` is false); otherwise
|
|
7
7
|
none matches. It keeps at most ``maximum_distinct + 1`` values, so its state is
|
|
8
|
-
bounded and its result does not depend on frequency tables.
|
|
8
|
+
bounded and its result does not depend on frequency tables. Once it holds
|
|
9
|
+
``maximum_distinct + 1`` values, the field is rejected and later values are
|
|
10
|
+
only counted (``rejects_field``).
|
|
9
11
|
"""
|
|
10
12
|
|
|
11
13
|
from __future__ import annotations
|
|
12
14
|
|
|
13
|
-
from collections.abc import Mapping
|
|
15
|
+
from collections.abc import Mapping, Sequence
|
|
14
16
|
|
|
15
17
|
from tabalyst.scanner.detectors.base import (
|
|
16
18
|
Classification,
|
|
@@ -36,6 +38,9 @@ class EnumerationDetector(Detector):
|
|
|
36
38
|
def classify(self, value: str) -> Classification:
|
|
37
39
|
return _MEMBER
|
|
38
40
|
|
|
41
|
+
def classify_many(self, values: Sequence[str]) -> list[Classification]:
|
|
42
|
+
return [_MEMBER] * len(values)
|
|
43
|
+
|
|
39
44
|
def accumulator(self) -> EnumerationAccumulator:
|
|
40
45
|
return EnumerationAccumulator(
|
|
41
46
|
self.minimum_values, self.maximum_distinct, self.case_sensitive
|
|
@@ -62,6 +67,11 @@ class EnumerationAccumulator(DetectorAccumulator):
|
|
|
62
67
|
def field_matches(self) -> bool:
|
|
63
68
|
return self.values >= self.minimum and len(self.keys) <= self.maximum
|
|
64
69
|
|
|
70
|
+
def rejects_field(self) -> bool:
|
|
71
|
+
# Keys stop growing past the maximum: the field can no longer match,
|
|
72
|
+
# and ``details`` no longer changes.
|
|
73
|
+
return len(self.keys) > self.maximum
|
|
74
|
+
|
|
65
75
|
def details(self, gate: ExposureGate) -> dict[str, object]:
|
|
66
76
|
distinct = len(self.keys)
|
|
67
77
|
if distinct <= self.maximum:
|
|
@@ -9,7 +9,7 @@ so the detector is sensitive: its fields are masked by default (design 12.8).
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
11
|
import re
|
|
12
|
-
from collections.abc import Mapping
|
|
12
|
+
from collections.abc import Mapping, Sequence
|
|
13
13
|
|
|
14
14
|
from tabalyst.scanner.detectors.base import (
|
|
15
15
|
Classification,
|
|
@@ -108,12 +108,24 @@ class IpAddressDetector(Detector):
|
|
|
108
108
|
return _ipv6(value) if self._ipv6 else None
|
|
109
109
|
return _ipv4(value) if self._ipv4 else None
|
|
110
110
|
|
|
111
|
+
def classify_many(self, values: Sequence[str]) -> list[Classification | None]:
|
|
112
|
+
classify = self.classify
|
|
113
|
+
return [
|
|
114
|
+
classify(value)
|
|
115
|
+
if MIN_LENGTH <= len(value) <= MAX_LENGTH
|
|
116
|
+
and value[0] in _EDGE
|
|
117
|
+
and value[-1] in _EDGE
|
|
118
|
+
else None
|
|
119
|
+
for value in values
|
|
120
|
+
]
|
|
121
|
+
|
|
111
122
|
def accumulator(self) -> IpAddressAccumulator:
|
|
112
123
|
return IpAddressAccumulator(self.versions)
|
|
113
124
|
|
|
114
125
|
|
|
115
126
|
class IpAddressAccumulator(DetectorAccumulator):
|
|
116
127
|
__slots__ = ("versions",)
|
|
128
|
+
ignores_unmatched = True
|
|
117
129
|
|
|
118
130
|
def __init__(self, versions: tuple[str, ...]) -> None:
|
|
119
131
|
self.versions = dict.fromkeys(sorted(versions), 0)
|
|
@@ -18,7 +18,7 @@ spreadsheet number formats: ``0``, ``0.0``, ``0E0``, ``#,##0.0``, ``0,0``.
|
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
20
|
import re
|
|
21
|
-
from collections.abc import Mapping
|
|
21
|
+
from collections.abc import Mapping, Sequence
|
|
22
22
|
from decimal import Decimal
|
|
23
23
|
|
|
24
24
|
from tabalyst.scanner.detectors.base import (
|
|
@@ -37,6 +37,8 @@ _CONVENTIONS = {"dot": (".", "," + _SPACES), "comma": (",", "." + _SPACES)}
|
|
|
37
37
|
|
|
38
38
|
# Characters of every accepted value: an exact, cheap rejection.
|
|
39
39
|
_CHARACTERS = re.compile("[-+.]?[0-9.][0-9.,eE+ -]*")
|
|
40
|
+
# The first character of every value that ``_CHARACTERS`` accepts.
|
|
41
|
+
_FIRST = frozenset("+-.0123456789")
|
|
40
42
|
|
|
41
43
|
INTEGER_FORMATS = frozenset(
|
|
42
44
|
{"0"} | {f"#{separator}##0" for separator in ",." + _SPACES}
|
|
@@ -150,6 +152,26 @@ class NumberDetector(Detector):
|
|
|
150
152
|
)
|
|
151
153
|
return Classification("ambiguous", candidates=candidates)
|
|
152
154
|
|
|
155
|
+
def classify_many(self, values: Sequence[str]) -> list[Classification | None]:
|
|
156
|
+
"""``classify`` of each value: values that cannot start a number are
|
|
157
|
+
rejected, and strict integers of ASCII digits read, without a call."""
|
|
158
|
+
classify = self.classify
|
|
159
|
+
first = _FIRST
|
|
160
|
+
results: list[Classification | None] = []
|
|
161
|
+
append = results.append
|
|
162
|
+
for value in values:
|
|
163
|
+
if value[:1] not in first:
|
|
164
|
+
append(None)
|
|
165
|
+
elif (
|
|
166
|
+
value.isdigit()
|
|
167
|
+
and value.isascii()
|
|
168
|
+
and (value[0] != "0" or len(value) == 1)
|
|
169
|
+
):
|
|
170
|
+
append(Classification("matched", "0", (), None, _integer(value)))
|
|
171
|
+
else:
|
|
172
|
+
append(classify(value))
|
|
173
|
+
return results
|
|
174
|
+
|
|
153
175
|
def accumulator(self) -> AmbiguityAccumulator:
|
|
154
176
|
resolution = (
|
|
155
177
|
None
|
|
@@ -9,7 +9,7 @@ value. Patterns are trusted configuration: Python ``re`` has no timeout.
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
11
|
import re
|
|
12
|
-
from collections.abc import Mapping
|
|
12
|
+
from collections.abc import Mapping, Sequence
|
|
13
13
|
|
|
14
14
|
from tabalyst.scanner.config import PatternSettings
|
|
15
15
|
from tabalyst.scanner.detectors.base import Classification, Detector
|
|
@@ -31,6 +31,12 @@ class PatternDetector(Detector):
|
|
|
31
31
|
def classify(self, value: str) -> Classification | None:
|
|
32
32
|
return _MATCHED if self._fullmatch(value) is not None else None
|
|
33
33
|
|
|
34
|
+
def classify_many(self, values: Sequence[str]) -> list[Classification | None]:
|
|
35
|
+
fullmatch = self._fullmatch
|
|
36
|
+
return [
|
|
37
|
+
_MATCHED if fullmatch(value) is not None else None for value in values
|
|
38
|
+
]
|
|
39
|
+
|
|
34
40
|
|
|
35
41
|
def pattern_detector(settings: PatternSettings) -> type[PatternDetector]:
|
|
36
42
|
"""A detector class for one validated ``patterns`` entry."""
|