tabalyst 0.4.0__tar.gz → 0.4.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. {tabalyst-0.4.0/src/tabalyst.egg-info → tabalyst-0.4.2}/PKG-INFO +5 -2
  2. {tabalyst-0.4.0 → tabalyst-0.4.2}/README.md +4 -1
  3. {tabalyst-0.4.0 → tabalyst-0.4.2}/pyproject.toml +1 -1
  4. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/cli/report.py +12 -0
  5. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/cli/scan.py +12 -0
  6. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/models.py +1 -1
  7. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/report_service.py +8 -0
  8. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/reporting.py +14 -0
  9. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scan_service.py +6 -2
  10. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/api.py +35 -4
  11. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/config.py +9 -0
  12. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/base.py +30 -4
  13. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/boolean.py +10 -1
  14. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/currency.py +11 -1
  15. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/email.py +6 -1
  16. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/enumeration.py +12 -2
  17. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/ip.py +13 -1
  18. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/number.py +23 -1
  19. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/pattern.py +7 -1
  20. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/percentage.py +5 -1
  21. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/phone.py +13 -1
  22. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/postal.py +6 -1
  23. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/quantity.py +6 -1
  24. tabalyst-0.4.2/src/tabalyst/scanner/detectors/registry.py +832 -0
  25. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/temporal.py +19 -1
  26. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/url.py +9 -1
  27. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/uuid.py +6 -0
  28. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/engine.py +22 -1
  29. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/exposure.py +18 -1
  30. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/field.py +53 -13
  31. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/measures.py +139 -104
  32. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/models.py +15 -1
  33. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/normalization.py +43 -9
  34. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/observations.py +41 -6
  35. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/readers/csv_reader.py +24 -14
  36. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/readers/json_reader.py +32 -4
  37. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/records.py +167 -8
  38. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/structure.py +106 -11
  39. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/technical.py +15 -22
  40. tabalyst-0.4.2/src/tabalyst/scanner/values.py +1022 -0
  41. tabalyst-0.4.2/src/tabalyst/scanner/workers.py +391 -0
  42. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/service.py +12 -5
  43. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/static/theme.css +4 -3
  44. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/templates/report.html +2 -2
  45. {tabalyst-0.4.0 → tabalyst-0.4.2/src/tabalyst.egg-info}/PKG-INFO +5 -2
  46. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst.egg-info/SOURCES.txt +2 -0
  47. {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_cli.py +2 -2
  48. {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_reporting.py +19 -1
  49. {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_scan_command.py +55 -0
  50. tabalyst-0.4.2/tests/test_scanner_batches.py +248 -0
  51. tabalyst-0.4.0/src/tabalyst/scanner/detectors/registry.py +0 -445
  52. tabalyst-0.4.0/src/tabalyst/scanner/values.py +0 -558
  53. {tabalyst-0.4.0 → tabalyst-0.4.2}/LICENSE +0 -0
  54. {tabalyst-0.4.0 → tabalyst-0.4.2}/MANIFEST.in +0 -0
  55. {tabalyst-0.4.0 → tabalyst-0.4.2}/examples/README.md +0 -0
  56. {tabalyst-0.4.0 → tabalyst-0.4.2}/examples/config.json +0 -0
  57. {tabalyst-0.4.0 → tabalyst-0.4.2}/examples/input/basic.csv +0 -0
  58. {tabalyst-0.4.0 → tabalyst-0.4.2}/examples/input/insurance-customers.csv +0 -0
  59. {tabalyst-0.4.0 → tabalyst-0.4.2}/setup.cfg +0 -0
  60. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/__init__.py +0 -0
  61. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/__main__.py +0 -0
  62. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/_version.py +0 -0
  63. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/batch.py +0 -0
  64. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/cli/__init__.py +0 -0
  65. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/cli/app.py +0 -0
  66. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/cli/sample.py +0 -0
  67. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/cli/terminal.py +0 -0
  68. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/config.py +0 -0
  69. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/errors.py +0 -0
  70. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/execution_log.py +0 -0
  71. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/progress.py +0 -0
  72. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/report_config.py +0 -0
  73. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/report_profile.py +0 -0
  74. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/sampling.py +0 -0
  75. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/sampling_service.py +0 -0
  76. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scan_reuse.py +0 -0
  77. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/__init__.py +0 -0
  78. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/__init__.py +0 -0
  79. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/amount.py +0 -0
  80. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/names.py +0 -0
  81. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/detectors/shape.py +0 -0
  82. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/diagnostics.py +0 -0
  83. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/paths.py +0 -0
  84. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/readers/__init__.py +0 -0
  85. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/scanner/readers/base.py +0 -0
  86. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst/static/report.js +0 -0
  87. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst.egg-info/dependency_links.txt +0 -0
  88. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst.egg-info/entry_points.txt +0 -0
  89. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst.egg-info/requires.txt +0 -0
  90. {tabalyst-0.4.0 → tabalyst-0.4.2}/src/tabalyst.egg-info/top_level.txt +0 -0
  91. {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_public_api.py +0 -0
  92. {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_sampling.py +0 -0
  93. {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_scanner_amounts.py +0 -0
  94. {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_scanner_config.py +0 -0
  95. {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_scanner_csv.py +0 -0
  96. {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_scanner_email_url.py +0 -0
  97. {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_scanner_exposure.py +0 -0
  98. {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_scanner_paths.py +0 -0
  99. {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_scanner_phone.py +0 -0
  100. {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_scanner_postal.py +0 -0
  101. {tabalyst-0.4.0 → tabalyst-0.4.2}/tests/test_scanner_uuid_ip.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tabalyst
3
- Version: 0.4.0
3
+ Version: 0.4.2
4
4
  Summary: Open-source toolkit for understanding and working with unfamiliar data.
5
5
  Author: Gregory Borelli
6
6
  License-Expression: MIT
@@ -181,6 +181,7 @@ tabalyst report data.csv --delimiter ";"
181
181
  tabalyst report data.csv --encoding cp1252
182
182
  tabalyst report data.csv --config tabalyst.json
183
183
  tabalyst report data.csv --verbose
184
+ tabalyst report data.csv --workers 4
184
185
  tabalyst report --help
185
186
  tabalyst --version
186
187
  ```
@@ -233,7 +234,9 @@ detector: numbers with decimal commas, dates, booleans, enumerations, email
233
234
  addresses, URLs, phone numbers, postal codes, currency amounts, percentages,
234
235
  quantities, UUIDs and IP addresses. Values of sensitive fields, such as email
235
236
  addresses, are masked by default. Results are written atomically: an
236
- interrupted scan never leaves a partial file.
237
+ interrupted scan never leaves a partial file. Files of 16 MiB or more are
238
+ analyzed by several worker processes, with the same result; `--workers`
239
+ chooses their number, `--workers 1` keeps one process.
237
240
 
238
241
  Build the report from a scan document without reading the source again:
239
242
 
@@ -147,6 +147,7 @@ tabalyst report data.csv --delimiter ";"
147
147
  tabalyst report data.csv --encoding cp1252
148
148
  tabalyst report data.csv --config tabalyst.json
149
149
  tabalyst report data.csv --verbose
150
+ tabalyst report data.csv --workers 4
150
151
  tabalyst report --help
151
152
  tabalyst --version
152
153
  ```
@@ -199,7 +200,9 @@ detector: numbers with decimal commas, dates, booleans, enumerations, email
199
200
  addresses, URLs, phone numbers, postal codes, currency amounts, percentages,
200
201
  quantities, UUIDs and IP addresses. Values of sensitive fields, such as email
201
202
  addresses, are masked by default. Results are written atomically: an
202
- interrupted scan never leaves a partial file.
203
+ interrupted scan never leaves a partial file. Files of 16 MiB or more are
204
+ analyzed by several worker processes, with the same result; `--workers`
205
+ chooses their number, `--workers 1` keeps one process.
203
206
 
204
207
  Build the report from a scan document without reading the source again:
205
208
 
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "tabalyst"
7
- version = "0.4.0"
7
+ version = "0.4.2"
8
8
  description = "Open-source toolkit for understanding and working with unfamiliar data."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -143,6 +143,17 @@ def report_command(
143
143
  help="Disable interactive file and phase progress.",
144
144
  ),
145
145
  ] = False,
146
+ workers: Annotated[
147
+ int | None,
148
+ typer.Option(
149
+ "--workers",
150
+ min=1,
151
+ help=(
152
+ "Processes that analyze values: 1 for one process. By default, "
153
+ "files of 16 MiB or more use one per spare processor."
154
+ ),
155
+ ),
156
+ ] = None,
146
157
  ) -> None:
147
158
  """Analyze and profile datasets with Tabalyst Report."""
148
159
  if quiet and verbose:
@@ -162,6 +173,7 @@ def report_command(
162
173
  force=force,
163
174
  on_progress=progress,
164
175
  from_scan=from_scan,
176
+ workers=workers,
165
177
  )
166
178
  except TabalystError as exc:
167
179
  typer.echo(f"Error: {exc}", err=True)
@@ -149,6 +149,17 @@ def scan_command(
149
149
  bool,
150
150
  typer.Option("--no-progress", help="Disable interactive progress."),
151
151
  ] = False,
152
+ workers: Annotated[
153
+ int | None,
154
+ typer.Option(
155
+ "--workers",
156
+ min=1,
157
+ help=(
158
+ "Processes that analyze values: 1 for one process. By default, "
159
+ "files of 16 MiB or more use one per spare processor."
160
+ ),
161
+ ),
162
+ ] = None,
152
163
  ) -> None:
153
164
  """Scan CSV and JSON files into a complete JSON description of their data."""
154
165
  if quiet and verbose:
@@ -168,6 +179,7 @@ def scan_command(
168
179
  collections=collection,
169
180
  force=force,
170
181
  on_progress=progress,
182
+ workers=workers,
171
183
  )
172
184
  except TabalystError as exc:
173
185
  typer.echo(f"Error: {exc}", err=True)
@@ -378,7 +378,7 @@ class DatasetProfile(ResultModel):
378
378
 
379
379
  class ReportProfile(ResultModel):
380
380
  format_version: Literal["0.1.0a"] = "0.1.0a"
381
- format_revision: Literal[6] = 6
381
+ format_revision: Literal[8] = 8
382
382
  generated_at: datetime
383
383
  processing_seconds: FiniteFloat
384
384
  source: SourceInfo
@@ -175,18 +175,25 @@ def generate_reports(
175
175
  force: bool = False,
176
176
  on_progress: ProgressCallback | None = None,
177
177
  from_scan: bool = False,
178
+ workers: int | None = None,
178
179
  ) -> BatchReportResult:
179
180
  """Plan and execute one or more reports without depending on the CLI.
180
181
 
181
182
  With ``from_scan``, the inputs are scan documents written by
182
183
  ``tabalyst scan``: each report is built from its document without reading
183
184
  the source again, after the staleness checks of ``tabalyst.scan_reuse``.
185
+ Otherwise ``workers`` is passed to ``tabalyst.scan()``.
184
186
  """
185
187
  if from_scan and (separator is not None or encoding is not None):
186
188
  raise ConfigurationError(
187
189
  "--delimiter and --encoding cannot be used with --scan: the scan "
188
190
  "document records how its source was read."
189
191
  )
192
+ if from_scan and workers is not None:
193
+ raise ConfigurationError(
194
+ "--workers cannot be used with --scan: the report reads the scan "
195
+ "document, not its source."
196
+ )
190
197
  plan = build_report_plan(
191
198
  input_specs,
192
199
  output=output,
@@ -245,6 +252,7 @@ def generate_reports(
245
252
  config_paths,
246
253
  force=force,
247
254
  on_progress=forward,
255
+ workers=workers,
248
256
  )
249
257
  except TabalystError as exc:
250
258
  failures.append(ReportFailure(job=job, error=exc))
@@ -1,10 +1,12 @@
1
1
  """Render self-contained analysis results as an interactive HTML report."""
2
2
 
3
+ import re
3
4
  from collections import defaultdict
4
5
  from importlib.resources import files
5
6
  from pathlib import Path
6
7
 
7
8
  from jinja2 import Environment, StrictUndefined
9
+ from markupsafe import Markup, escape
8
10
 
9
11
  from tabalyst._version import get_version
10
12
  from tabalyst.models import ReportProfile
@@ -55,6 +57,17 @@ def format_size(size_bytes: int) -> str:
55
57
  return f"{size_bytes / 1024:.1f} KB"
56
58
 
57
59
 
60
+ # Separators after which a long name may break: snake case, paths and dates.
61
+ _BREAKS = re.compile(r"(?<=[_./-])")
62
+
63
+
64
+ def breakable(name: str) -> Markup:
65
+ """``name`` escaped, with a line break opportunity after each separator,
66
+ so that a table squeezed for room narrows its column instead of
67
+ scrolling."""
68
+ return Markup("<wbr>").join(escape(part) for part in _BREAKS.split(name))
69
+
70
+
58
71
  def load_profile(path: str | Path) -> ReportProfile:
59
72
  """Validate and load the experimental JSON profile format."""
60
73
  return ReportProfile.model_validate_json(Path(path).read_text(encoding="utf-8"))
@@ -68,6 +81,7 @@ def render_report(profile: ReportProfile) -> str:
68
81
  environment.filters["number"] = format_number
69
82
  environment.filters["seconds"] = format_seconds
70
83
  environment.filters["size"] = format_size
84
+ environment.filters["breakable"] = breakable
71
85
  template = environment.from_string(
72
86
  resources.joinpath("templates/report.html").read_text(encoding="utf-8")
73
87
  )
@@ -106,6 +106,7 @@ def generate_scans(
106
106
  collections: Sequence[str] | None = None,
107
107
  force: bool = False,
108
108
  on_progress: ProgressCallback | None = None,
109
+ workers: int | None = None,
109
110
  ) -> BatchScanResult:
110
111
  """Scan one or more CSV or JSON files and write one scan document each.
111
112
 
@@ -113,7 +114,8 @@ def generate_scans(
113
114
  the ``scan`` section of each ``config_path`` file in order, then
114
115
  ``delimiter``, ``encoding`` and ``collections``. The whole batch is
115
116
  rejected before any scan when an output is unsafe; afterwards, a failed
116
- source does not stop the others.
117
+ source does not stop the others. ``workers`` is passed to
118
+ ``tabalyst.scan()``.
117
119
  """
118
120
  config_paths = _config_paths(config_path)
119
121
  config = resolve_scan_config(
@@ -151,7 +153,9 @@ def generate_scans(
151
153
  )
152
154
 
153
155
  try:
154
- result = scan(job.source, config=config, on_progress=forward)
156
+ result = scan(
157
+ job.source, config=config, on_progress=forward, workers=workers
158
+ )
155
159
  emit_progress(
156
160
  on_progress, job.source, ProgressPhase.WRITING, index=index, total=total
157
161
  )
@@ -8,7 +8,7 @@ from pathlib import Path
8
8
  from time import perf_counter
9
9
 
10
10
  from tabalyst._version import __version__
11
- from tabalyst.errors import InputError
11
+ from tabalyst.errors import ConfigurationError, InputError
12
12
  from tabalyst.progress import ProgressCallback, ProgressPhase, emit_progress
13
13
  from tabalyst.scanner.config import ScanConfig, config_sha256
14
14
  from tabalyst.scanner.detectors.registry import (
@@ -25,11 +25,17 @@ from tabalyst.scanner.models import (
25
25
  Scope,
26
26
  SourceInfo,
27
27
  )
28
- from tabalyst.scanner.observations import Record, StreamItem
28
+ from tabalyst.scanner.observations import (
29
+ DatasetOpened,
30
+ Record,
31
+ RecordBatch,
32
+ StreamItem,
33
+ )
29
34
  from tabalyst.scanner.paths import format_absolute, parse_path
30
35
  from tabalyst.scanner.readers.base import Reader
31
36
  from tabalyst.scanner.readers.csv_reader import CsvReader
32
37
  from tabalyst.scanner.readers.json_reader import JsonReader
38
+ from tabalyst.scanner.workers import WorkerPool, worker_count
33
39
 
34
40
  NORMALIZATION_VERSION = 1
35
41
  # Reading progress is reported at most once per percent of the source, and
@@ -86,9 +92,17 @@ def _collection_scope(source_format: str, config: ScanConfig) -> CollectionScope
86
92
  def _observed(
87
93
  items: Iterable[StreamItem], on_record: Callable[[Record], None]
88
94
  ) -> Iterator[StreamItem]:
95
+ """Items as read, after passing each analyzed record to ``on_record``;
96
+ records of a batch are passed one by one, as ``Record`` items."""
97
+ declared = {}
89
98
  for item in items:
90
99
  if type(item) is Record:
91
100
  on_record(item)
101
+ elif type(item) is RecordBatch:
102
+ for record in item.records(declared[item.dataset]):
103
+ on_record(record)
104
+ elif type(item) is DatasetOpened:
105
+ declared[item.dataset] = [field.path for field in item.fields]
92
106
  yield item
93
107
 
94
108
 
@@ -99,6 +113,7 @@ def scan(
99
113
  registry: DetectorRegistry | None = None,
100
114
  on_progress: ProgressCallback | None = None,
101
115
  on_record: Callable[[Record], None] | None = None,
116
+ workers: int | None = None,
102
117
  ) -> ScanResult:
103
118
  """Read one source completely and return its finalized scan result.
104
119
 
@@ -107,7 +122,14 @@ def scan(
107
122
  order, before the engine: consumers that need record-level facts, such
108
123
  as the report's preview and duplicate rows, get them in the same pass. It
109
124
  must not modify the record.
125
+
126
+ ``workers`` is the number of processes that analyze values: ``1`` scans
127
+ in this process; by default, large sources use one worker per spare
128
+ processor (``workers.worker_count``). The result does not depend on it.
129
+ A custom ``registry`` always scans in this process.
110
130
  """
131
+ if workers is not None and workers < 1:
132
+ raise ConfigurationError("workers must be at least 1")
111
133
  path = Path(source)
112
134
  # A private copy: the finalized result must not share state with the caller.
113
135
  config = ScanConfig() if config is None else config.model_copy(deep=True)
@@ -130,8 +152,17 @@ def scan(
130
152
  default_registry() if registry is None else registry, config
131
153
  )
132
154
  engine = ScanEngine(config, detectors)
133
- engine.consume(reader if on_record is None else _observed(reader, on_record))
134
- datasets = engine.finalize()
155
+ count = 1 if registry is not None else worker_count(workers, stat.st_size)
156
+ if count > 1:
157
+ pool = WorkerPool(config.model_dump_json(by_alias=True), count)
158
+ if pool.start():
159
+ engine.values.pool = pool
160
+ try:
161
+ engine.consume(reader if on_record is None else _observed(reader, on_record))
162
+ datasets = engine.finalize()
163
+ finally:
164
+ if engine.values.pool is not None:
165
+ engine.values.pool.close()
135
166
  summary = reader.summary()
136
167
 
137
168
  excluded = engine.records_excluded
@@ -52,6 +52,9 @@ MAX_PREVIEW_RECORDS = 1_000
52
52
  MAX_PATTERNS = 200
53
53
  MAX_PATTERN_LENGTH = 1_000
54
54
  MAX_PATTERN_INPUT_LENGTH = 10_000
55
+ # A streamed field keeps its warm-up values: the warm-up is bounded.
56
+ MAX_WARMUP_VALUES = 1_000_000
57
+ MAX_PROBE_INTERVAL = 1_000_000
55
58
 
56
59
  MissingCategory = Literal["absent", "null", "empty", "blank", "marker"]
57
60
  ScalarType = Literal["string", "integer", "number", "boolean"]
@@ -176,6 +179,12 @@ class TypeSettings(_Settings):
176
179
 
177
180
  class DetectionSettings(_Settings):
178
181
  minimum_share: float = Field(default=0.95, gt=0, le=1)
182
+ # Adaptive detection (design 13): 0 keeps detection exhaustive.
183
+ warmup_values: int = Field(default=10_000, ge=0, le=MAX_WARMUP_VALUES)
184
+ probe_interval: int = Field(default=100, ge=0, le=MAX_PROBE_INTERVAL)
185
+ # Detectors that reacted to at most this share of the warm-up values are
186
+ # skipped too; 0 skips only those that reacted to none.
187
+ rare_share: float = Field(default=0.001, ge=0, le=1)
179
188
 
180
189
 
181
190
  NumberConvention = Literal["dot", "comma"]
@@ -10,9 +10,9 @@ accumulator adds its own ``details``.
10
10
  from __future__ import annotations
11
11
 
12
12
  import re
13
- from collections.abc import Mapping
13
+ from collections.abc import Mapping, Sequence
14
14
  from dataclasses import dataclass
15
- from typing import TYPE_CHECKING, ClassVar, Literal
15
+ from typing import TYPE_CHECKING, ClassVar, Literal, NamedTuple
16
16
 
17
17
  if TYPE_CHECKING:
18
18
  from tabalyst.scanner.exposure import ExposureGate
@@ -20,8 +20,7 @@ if TYPE_CHECKING:
20
20
  State = Literal["matched", "ambiguous", "invalid"]
21
21
 
22
22
 
23
- @dataclass(frozen=True, slots=True)
24
- class Classification:
23
+ class Classification(NamedTuple):
25
24
  """Result of a detector for one value; ``None`` stands for not matched.
26
25
 
27
26
  - ``matched`` values may carry a ``format`` and a parsed ``value`` for
@@ -34,6 +33,9 @@ class Classification:
34
33
  - ``convention`` names the reading of an unambiguous value when several
35
34
  readings exist (date order, decimal convention); it feeds the ambiguity
36
35
  evidence of the field (design 12.6).
36
+
37
+ An immutable named tuple: detectors build one per matched value, and a
38
+ tuple is built about twice as fast as a frozen dataclass.
37
39
  """
38
40
 
39
41
  state: State
@@ -69,9 +71,15 @@ class DetectorAccumulator:
69
71
  ``add`` receives every tested value with its classification (``None`` when
70
72
  not matched) and its count. In both execution modes it sees the same
71
73
  distinct values, first seen first, with the same total counts.
74
+
75
+ A class whose ``add`` does nothing for unmatched values (``None``) sets
76
+ ``ignores_unmatched`` in its own body: the engine then skips those calls.
77
+ The flag applies only to the class that defines ``add``, so a subclass
78
+ that overrides ``add`` receives every value unless it sets it again.
72
79
  """
73
80
 
74
81
  __slots__ = ()
82
+ ignores_unmatched: ClassVar[bool] = True
75
83
 
76
84
  def add(
77
85
  self, value: str, classification: Classification | None, count: int
@@ -90,6 +98,14 @@ class DetectorAccumulator:
90
98
  """Field-level detectors decide at the end whether their values match."""
91
99
  return True
92
100
 
101
+ def rejects_field(self) -> bool:
102
+ """Whether a field-level detector already knows that the field cannot
103
+ match, whatever values follow, and that ``add`` no longer changes
104
+ ``details``. The engine then counts later values as not matched
105
+ without classifying them, which is exact only for detectors whose
106
+ ``classify`` never returns ``ambiguous`` or ``invalid``."""
107
+ return False
108
+
93
109
 
94
110
  class Detector:
95
111
  """Base class of detectors; subclasses set the class attributes and
@@ -104,6 +120,11 @@ class Detector:
104
120
  - ``max_input_length``: longer values are ``not_tested``.
105
121
  - ``scope``: ``field`` detectors, such as enumeration candidates, decide
106
122
  at the end whether every value of the field matches or none does.
123
+
124
+ The engine classifies distinct values in batches through
125
+ ``classify_many``, which maps ``classify`` by default. A detector may
126
+ override it to reject most values without a call per value, provided it
127
+ returns exactly what ``classify`` returns for each value.
107
128
  """
108
129
 
109
130
  id: ClassVar[str]
@@ -121,6 +142,10 @@ class Detector:
121
142
  def classify(self, value: str) -> Classification | None:
122
143
  raise NotImplementedError
123
144
 
145
+ def classify_many(self, values: Sequence[str]) -> list[Classification | None]:
146
+ """``classify`` of each value, in order."""
147
+ return list(map(self.classify, values))
148
+
124
149
  def accumulator(self) -> DetectorAccumulator:
125
150
  return DetectorAccumulator()
126
151
 
@@ -135,6 +160,7 @@ class AmbiguityAccumulator(DetectorAccumulator):
135
160
  """
136
161
 
137
162
  __slots__ = ("candidates", "count", "evidence", "resolution")
163
+ ignores_unmatched = True
138
164
 
139
165
  def __init__(self, readings: list[str], resolution: dict[str, str] | None) -> None:
140
166
  self.count = 0
@@ -7,7 +7,7 @@ booleans stay in the ``booleans`` block.
7
7
 
8
8
  from __future__ import annotations
9
9
 
10
- from collections.abc import Mapping
10
+ from collections.abc import Mapping, Sequence
11
11
 
12
12
  from tabalyst.scanner.detectors.base import (
13
13
  Classification,
@@ -45,12 +45,21 @@ class BooleanDetector(Detector):
45
45
  return None
46
46
  return self._words.get(value.casefold())
47
47
 
48
+ def classify_many(self, values: Sequence[str]) -> list[Classification | None]:
49
+ words = self._words
50
+ longest = self._longest
51
+ return [
52
+ words.get(value.casefold()) if len(value) <= longest else None
53
+ for value in values
54
+ ]
55
+
48
56
  def accumulator(self) -> BooleanAccumulator:
49
57
  return BooleanAccumulator()
50
58
 
51
59
 
52
60
  class BooleanAccumulator(DetectorAccumulator):
53
61
  __slots__ = ("false", "true")
62
+ ignores_unmatched = True
54
63
 
55
64
  def __init__(self) -> None:
56
65
  self.true = 0
@@ -8,7 +8,7 @@ of money. The number part follows the number detector (``amount.py``).
8
8
  from __future__ import annotations
9
9
 
10
10
  import re
11
- from collections.abc import Mapping
11
+ from collections.abc import Mapping, Sequence
12
12
 
13
13
  from tabalyst.scanner.detectors.amount import (
14
14
  NUMBER_PART,
@@ -107,12 +107,22 @@ class CurrencyDetector(Detector):
107
107
  sign, _, body, prefix, suffix = parts
108
108
  return self.reader.read(sign, body, prefix, suffix)
109
109
 
110
+ def classify_many(self, values: Sequence[str]) -> list[Classification | None]:
111
+ classify = self.classify
112
+ return [
113
+ classify(value)
114
+ if value[:1] in _FIRST or value[-1:] in _LAST
115
+ else None
116
+ for value in values
117
+ ]
118
+
110
119
  def accumulator(self) -> CurrencyAccumulator:
111
120
  return CurrencyAccumulator(self.reader)
112
121
 
113
122
 
114
123
  class CurrencyAccumulator(AmountAccumulator):
115
124
  __slots__ = ("currencies",)
125
+ ignores_unmatched = True
116
126
 
117
127
  def __init__(self, reader: AmountReader) -> None:
118
128
  super().__init__(reader)
@@ -8,7 +8,7 @@ masked by default (design 12.8).
8
8
  from __future__ import annotations
9
9
 
10
10
  import re
11
- from collections.abc import Mapping
11
+ from collections.abc import Mapping, Sequence
12
12
 
13
13
  from tabalyst.scanner.detectors.base import (
14
14
  Classification,
@@ -66,12 +66,17 @@ class EmailDetector(Detector):
66
66
  return _BAD_DOMAIN
67
67
  return Classification("matched", value=domain.lower())
68
68
 
69
+ def classify_many(self, values: Sequence[str]) -> list[Classification | None]:
70
+ classify = self.classify
71
+ return [classify(value) if "@" in value else None for value in values]
72
+
69
73
  def accumulator(self) -> EmailAccumulator:
70
74
  return EmailAccumulator(NameCounts(self.max_tracked, self.max_listed))
71
75
 
72
76
 
73
77
  class EmailAccumulator(DetectorAccumulator):
74
78
  __slots__ = ("domains",)
79
+ ignores_unmatched = True
75
80
 
76
81
  def __init__(self, domains: NameCounts) -> None:
77
82
  self.domains = domains
@@ -5,12 +5,14 @@ A field-level detector: its values match together when the field has at least
5
5
  ``minimum_values`` eligible values and at most ``maximum_distinct`` distinct
6
6
  analytical values (ignoring case when ``case_sensitive`` is false); otherwise
7
7
  none matches. It keeps at most ``maximum_distinct + 1`` values, so its state is
8
- bounded and its result does not depend on frequency tables.
8
+ bounded and its result does not depend on frequency tables. Once it holds
9
+ ``maximum_distinct + 1`` values, the field is rejected and later values are
10
+ only counted (``rejects_field``).
9
11
  """
10
12
 
11
13
  from __future__ import annotations
12
14
 
13
- from collections.abc import Mapping
15
+ from collections.abc import Mapping, Sequence
14
16
 
15
17
  from tabalyst.scanner.detectors.base import (
16
18
  Classification,
@@ -36,6 +38,9 @@ class EnumerationDetector(Detector):
36
38
  def classify(self, value: str) -> Classification:
37
39
  return _MEMBER
38
40
 
41
+ def classify_many(self, values: Sequence[str]) -> list[Classification]:
42
+ return [_MEMBER] * len(values)
43
+
39
44
  def accumulator(self) -> EnumerationAccumulator:
40
45
  return EnumerationAccumulator(
41
46
  self.minimum_values, self.maximum_distinct, self.case_sensitive
@@ -62,6 +67,11 @@ class EnumerationAccumulator(DetectorAccumulator):
62
67
  def field_matches(self) -> bool:
63
68
  return self.values >= self.minimum and len(self.keys) <= self.maximum
64
69
 
70
+ def rejects_field(self) -> bool:
71
+ # Keys stop growing past the maximum: the field can no longer match,
72
+ # and ``details`` no longer changes.
73
+ return len(self.keys) > self.maximum
74
+
65
75
  def details(self, gate: ExposureGate) -> dict[str, object]:
66
76
  distinct = len(self.keys)
67
77
  if distinct <= self.maximum:
@@ -9,7 +9,7 @@ so the detector is sensitive: its fields are masked by default (design 12.8).
9
9
  from __future__ import annotations
10
10
 
11
11
  import re
12
- from collections.abc import Mapping
12
+ from collections.abc import Mapping, Sequence
13
13
 
14
14
  from tabalyst.scanner.detectors.base import (
15
15
  Classification,
@@ -108,12 +108,24 @@ class IpAddressDetector(Detector):
108
108
  return _ipv6(value) if self._ipv6 else None
109
109
  return _ipv4(value) if self._ipv4 else None
110
110
 
111
+ def classify_many(self, values: Sequence[str]) -> list[Classification | None]:
112
+ classify = self.classify
113
+ return [
114
+ classify(value)
115
+ if MIN_LENGTH <= len(value) <= MAX_LENGTH
116
+ and value[0] in _EDGE
117
+ and value[-1] in _EDGE
118
+ else None
119
+ for value in values
120
+ ]
121
+
111
122
  def accumulator(self) -> IpAddressAccumulator:
112
123
  return IpAddressAccumulator(self.versions)
113
124
 
114
125
 
115
126
  class IpAddressAccumulator(DetectorAccumulator):
116
127
  __slots__ = ("versions",)
128
+ ignores_unmatched = True
117
129
 
118
130
  def __init__(self, versions: tuple[str, ...]) -> None:
119
131
  self.versions = dict.fromkeys(sorted(versions), 0)
@@ -18,7 +18,7 @@ spreadsheet number formats: ``0``, ``0.0``, ``0E0``, ``#,##0.0``, ``0,0``.
18
18
  from __future__ import annotations
19
19
 
20
20
  import re
21
- from collections.abc import Mapping
21
+ from collections.abc import Mapping, Sequence
22
22
  from decimal import Decimal
23
23
 
24
24
  from tabalyst.scanner.detectors.base import (
@@ -37,6 +37,8 @@ _CONVENTIONS = {"dot": (".", "," + _SPACES), "comma": (",", "." + _SPACES)}
37
37
 
38
38
  # Characters of every accepted value: an exact, cheap rejection.
39
39
  _CHARACTERS = re.compile("[-+.]?[0-9.][0-9.,eE+   -]*")
40
+ # The first character of every value that ``_CHARACTERS`` accepts.
41
+ _FIRST = frozenset("+-.0123456789")
40
42
 
41
43
  INTEGER_FORMATS = frozenset(
42
44
  {"0"} | {f"#{separator}##0" for separator in ",." + _SPACES}
@@ -150,6 +152,26 @@ class NumberDetector(Detector):
150
152
  )
151
153
  return Classification("ambiguous", candidates=candidates)
152
154
 
155
+ def classify_many(self, values: Sequence[str]) -> list[Classification | None]:
156
+ """``classify`` of each value: values that cannot start a number are
157
+ rejected, and strict integers of ASCII digits read, without a call."""
158
+ classify = self.classify
159
+ first = _FIRST
160
+ results: list[Classification | None] = []
161
+ append = results.append
162
+ for value in values:
163
+ if value[:1] not in first:
164
+ append(None)
165
+ elif (
166
+ value.isdigit()
167
+ and value.isascii()
168
+ and (value[0] != "0" or len(value) == 1)
169
+ ):
170
+ append(Classification("matched", "0", (), None, _integer(value)))
171
+ else:
172
+ append(classify(value))
173
+ return results
174
+
153
175
  def accumulator(self) -> AmbiguityAccumulator:
154
176
  resolution = (
155
177
  None
@@ -9,7 +9,7 @@ value. Patterns are trusted configuration: Python ``re`` has no timeout.
9
9
  from __future__ import annotations
10
10
 
11
11
  import re
12
- from collections.abc import Mapping
12
+ from collections.abc import Mapping, Sequence
13
13
 
14
14
  from tabalyst.scanner.config import PatternSettings
15
15
  from tabalyst.scanner.detectors.base import Classification, Detector
@@ -31,6 +31,12 @@ class PatternDetector(Detector):
31
31
  def classify(self, value: str) -> Classification | None:
32
32
  return _MATCHED if self._fullmatch(value) is not None else None
33
33
 
34
+ def classify_many(self, values: Sequence[str]) -> list[Classification | None]:
35
+ fullmatch = self._fullmatch
36
+ return [
37
+ _MATCHED if fullmatch(value) is not None else None for value in values
38
+ ]
39
+
34
40
 
35
41
  def pattern_detector(settings: PatternSettings) -> type[PatternDetector]:
36
42
  """A detector class for one validated ``patterns`` entry."""