tabalyst 0.4.4__tar.gz → 0.5.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. {tabalyst-0.4.4/src/tabalyst.egg-info → tabalyst-0.5.1}/PKG-INFO +73 -20
  2. {tabalyst-0.4.4 → tabalyst-0.5.1}/README.md +72 -19
  3. {tabalyst-0.4.4 → tabalyst-0.5.1}/examples/README.md +16 -4
  4. {tabalyst-0.4.4 → tabalyst-0.5.1}/pyproject.toml +2 -1
  5. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/__init__.py +6 -0
  6. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/batch.py +24 -2
  7. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/cli/app.py +9 -2
  8. tabalyst-0.5.1/src/tabalyst/cli/inspect.py +181 -0
  9. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/cli/report.py +12 -1
  10. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/cli/scan.py +9 -3
  11. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/cli/terminal.py +30 -0
  12. tabalyst-0.5.1/src/tabalyst/inspect_service.py +209 -0
  13. tabalyst-0.5.1/src/tabalyst/inspector/__init__.py +5 -0
  14. tabalyst-0.5.1/src/tabalyst/inspector/json_inspect/__init__.py +10 -0
  15. tabalyst-0.5.1/src/tabalyst/inspector/json_inspect/build.py +314 -0
  16. tabalyst-0.5.1/src/tabalyst/inspector/json_inspect/detect.py +352 -0
  17. tabalyst-0.5.1/src/tabalyst/inspector/json_inspect/parameters.py +16 -0
  18. tabalyst-0.5.1/src/tabalyst/inspector/models.py +283 -0
  19. tabalyst-0.5.1/src/tabalyst/inspector/persistence.py +349 -0
  20. tabalyst-0.5.1/src/tabalyst/inspector/resolution.py +386 -0
  21. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/models.py +5 -5
  22. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/progress.py +30 -0
  23. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/project_open_service.py +6 -2
  24. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/_session.py +27 -14
  25. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/_staging.py +11 -4
  26. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/freshness.py +16 -3
  27. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/location.py +10 -2
  28. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/report_profile.py +22 -4
  29. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/report_service.py +15 -4
  30. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/reporting.py +1 -0
  31. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scan_reuse.py +41 -12
  32. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scan_service.py +36 -22
  33. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/api.py +15 -30
  34. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/config.py +99 -4
  35. tabalyst-0.5.1/src/tabalyst/scanner/identity.py +56 -0
  36. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/models.py +2 -2
  37. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/paths.py +30 -14
  38. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/readers/base.py +1 -1
  39. tabalyst-0.5.1/src/tabalyst/scanner/readers/json_common.py +37 -0
  40. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/readers/json_reader.py +65 -65
  41. tabalyst-0.5.1/src/tabalyst/scanner/readers/jsonl_reader.py +431 -0
  42. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/structure.py +1 -1
  43. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/service.py +60 -21
  44. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/shared_scan_service.py +28 -16
  45. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/static/report.js +20 -11
  46. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/templates/column.html +4 -4
  47. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/templates/report.html +4 -4
  48. {tabalyst-0.4.4 → tabalyst-0.5.1/src/tabalyst.egg-info}/PKG-INFO +73 -20
  49. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst.egg-info/SOURCES.txt +13 -0
  50. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_cli.py +2 -2
  51. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_project_database.py +9 -0
  52. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_project_open_service.py +4 -1
  53. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_project_queries.py +2 -1
  54. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_project_rescan.py +8 -3
  55. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_project_sessions.py +5 -3
  56. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_scan_command.py +1 -1
  57. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_scanner_config.py +6 -1
  58. {tabalyst-0.4.4 → tabalyst-0.5.1}/LICENSE +0 -0
  59. {tabalyst-0.4.4 → tabalyst-0.5.1}/MANIFEST.in +0 -0
  60. {tabalyst-0.4.4 → tabalyst-0.5.1}/examples/config.json +0 -0
  61. {tabalyst-0.4.4 → tabalyst-0.5.1}/examples/input/basic.csv +0 -0
  62. {tabalyst-0.4.4 → tabalyst-0.5.1}/examples/input/insurance-customers.csv +0 -0
  63. {tabalyst-0.4.4 → tabalyst-0.5.1}/setup.cfg +0 -0
  64. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/__main__.py +0 -0
  65. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/_version.py +0 -0
  66. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/cache_service.py +0 -0
  67. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/cli/__init__.py +0 -0
  68. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/cli/cache.py +0 -0
  69. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/cli/sample.py +0 -0
  70. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/config.py +0 -0
  71. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/errors.py +0 -0
  72. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/execution_log.py +0 -0
  73. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/project_scan_service.py +0 -0
  74. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/__init__.py +0 -0
  75. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/_bounded_values.py +0 -0
  76. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/_cache.py +0 -0
  77. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/_codec.py +0 -0
  78. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/_generation.py +0 -0
  79. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/_locks.py +0 -0
  80. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/_publication.py +0 -0
  81. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/_queries.py +0 -0
  82. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/_query_budget.py +0 -0
  83. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/_recovery.py +0 -0
  84. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/_retention.py +0 -0
  85. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/_schema.py +0 -0
  86. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/_sync.py +0 -0
  87. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/_validation.py +0 -0
  88. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/catalog.py +0 -0
  89. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/identity.py +0 -0
  90. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/index.py +0 -0
  91. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/models.py +0 -0
  92. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/projects/store.py +0 -0
  93. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/report_config.py +0 -0
  94. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/sampling.py +0 -0
  95. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/sampling_service.py +0 -0
  96. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/__init__.py +0 -0
  97. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/detectors/__init__.py +0 -0
  98. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/detectors/amount.py +0 -0
  99. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/detectors/base.py +0 -0
  100. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/detectors/boolean.py +0 -0
  101. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/detectors/currency.py +0 -0
  102. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/detectors/email.py +0 -0
  103. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/detectors/enumeration.py +0 -0
  104. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/detectors/ip.py +0 -0
  105. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/detectors/names.py +0 -0
  106. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/detectors/number.py +0 -0
  107. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/detectors/pattern.py +0 -0
  108. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/detectors/percentage.py +0 -0
  109. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/detectors/phone.py +0 -0
  110. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/detectors/postal.py +0 -0
  111. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/detectors/quantity.py +0 -0
  112. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/detectors/registry.py +0 -0
  113. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/detectors/shape.py +0 -0
  114. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/detectors/temporal.py +0 -0
  115. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/detectors/url.py +0 -0
  116. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/detectors/uuid.py +0 -0
  117. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/diagnostics.py +0 -0
  118. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/engine.py +0 -0
  119. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/exposure.py +0 -0
  120. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/field.py +0 -0
  121. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/measures.py +0 -0
  122. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/normalization.py +0 -0
  123. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/observations.py +0 -0
  124. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/readers/__init__.py +0 -0
  125. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/readers/csv_reader.py +0 -0
  126. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/records.py +0 -0
  127. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/technical.py +0 -0
  128. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/values.py +0 -0
  129. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/scanner/workers.py +0 -0
  130. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst/static/theme.css +0 -0
  131. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst.egg-info/dependency_links.txt +0 -0
  132. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst.egg-info/entry_points.txt +0 -0
  133. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst.egg-info/requires.txt +0 -0
  134. {tabalyst-0.4.4 → tabalyst-0.5.1}/src/tabalyst.egg-info/top_level.txt +0 -0
  135. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_project_cache.py +0 -0
  136. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_project_generations.py +0 -0
  137. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_project_identity.py +0 -0
  138. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_project_index.py +0 -0
  139. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_project_location.py +0 -0
  140. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_project_scan.py +0 -0
  141. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_project_store.py +0 -0
  142. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_public_api.py +0 -0
  143. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_reporting.py +0 -0
  144. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_sampling.py +0 -0
  145. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_scanner_amounts.py +0 -0
  146. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_scanner_batches.py +0 -0
  147. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_scanner_csv.py +0 -0
  148. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_scanner_email_url.py +0 -0
  149. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_scanner_exposure.py +0 -0
  150. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_scanner_paths.py +0 -0
  151. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_scanner_phone.py +0 -0
  152. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_scanner_postal.py +0 -0
  153. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_scanner_uuid_ip.py +0 -0
  154. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_shared_scan_cli.py +0 -0
  155. {tabalyst-0.4.4 → tabalyst-0.5.1}/tests/test_source_freshness.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tabalyst
3
- Version: 0.4.4
3
+ Version: 0.5.1
4
4
  Summary: Open-source toolkit for understanding and working with unfamiliar data.
5
5
  Author: Gregory Borelli
6
6
  License-Expression: MIT
@@ -44,7 +44,8 @@ with structured data.
44
44
  Its first tool is **Tabalyst Report**. The current CSV implementation,
45
45
  **Tabalyst CSV Report**, analyzes a CSV file and produces both a structured JSON
46
46
  profile and a self-contained interactive HTML report.
47
- **Tabalyst Scan** describes CSV and JSON files in a scan document, and
47
+ **Tabalyst Scan** describes CSV, JSON and JSONL files in a scan document,
48
+ **Tabalyst Inspect** finds how to read a JSON or JSONL file, and
48
49
  **Tabalyst Sample** creates smaller CSV files.
49
50
 
50
51
  Tabalyst is in beta. Its interfaces may still change while
@@ -70,6 +71,7 @@ changes often: update it before each new test.
70
71
  | Several files, automatic names | `tabalyst report *.csv` | Beside each source |
71
72
  | Several files, one directory | `tabalyst report *.csv -d reports/` | The directory `reports/` |
72
73
  | One JSON file | `tabalyst report data.json` | `data.report.html` beside the source |
74
+ | One JSONL file | `tabalyst report events.jsonl` | `events.report.html` beside the source |
73
75
  | From a scan document | `tabalyst report --scan data.scan.json` | `data.html` beside the scan |
74
76
 
75
77
  The simplest command keeps the source filename:
@@ -109,16 +111,23 @@ tabalyst report customers.csv -o customer-analysis.html
109
111
 
110
112
  ### JSON files
111
113
 
112
- A JSON file gets a report too, named `<stem>.report.html` so its profile
113
- `<stem>.report.json` never replaces the source. Each collection of records,
114
- such as the `customers` array of `{"customers": [...]}`, is a dataset of the
115
- report, and nested fields are columns named by their path, such as
116
- `address.city`:
114
+ A JSON or JSONL file gets a report too, named `<stem>.report.html` so its
115
+ profile `<stem>.report.json` never replaces the source. The report analyzes one
116
+ collection of records, such as the `customers` array of
117
+ `{"customers": [...]}`, chosen by Tabalyst Inspect, and nested fields are
118
+ columns named by their path, such as `address.city`:
117
119
 
118
120
  ```console
119
121
  tabalyst report orders.json
122
+ tabalyst report events.jsonl
120
123
  ```
121
124
 
125
+ In a JSONL file (`.jsonl` or `.ndjson`), each line is a record. A line that is
126
+ not valid JSON or not an object is excluded and counted, and the report is
127
+ partial. When a JSON file holds several arrays that are equally plausible,
128
+ Tabalyst stops and lists them instead of choosing: see
129
+ [Tabalyst Inspect](#tabalyst-inspect).
130
+
122
131
  ### Multiple files
123
132
 
124
133
  Report several CSV files at once:
@@ -236,16 +245,47 @@ Sampling reads CSV records as a stream. Random and stratified sampling keep
236
245
  only the requested sample, plus stratum counts, in memory. Existing outputs
237
246
  require `--force`, and an input file is never overwritten.
238
247
 
248
+ ## Tabalyst Inspect
249
+
250
+ A JSON file can hold several arrays. **Tabalyst Inspect** reads a JSON, JSONL
251
+ or NDJSON file once, finds which array holds the records and writes its answer
252
+ in `<source>-inspect.json` beside the source:
253
+
254
+ ```console
255
+ tabalyst inspect orders.json
256
+ ```
257
+
258
+ ```text
259
+ Inspect: orders.json-inspect.json
260
+ Selection: $.customers[] (the only eligible collection)
261
+ ```
262
+
263
+ The last section of that file, `config`, holds the rules that `tabalyst scan`
264
+ and `tabalyst report` apply to the source: the collection
265
+ (`structure.dataset_path`), the flatten settings, the array mode and the error
266
+ policy. Edit it, then scan or report as usual. Running `tabalyst inspect` again
267
+ refreshes the detection and keeps your `config`; `--reset-config` replaces it.
268
+
269
+ Inspect is optional: `tabalyst scan` and `tabalyst report` inspect a JSON file
270
+ themselves when it has no Inspect file. When several arrays are equally
271
+ plausible, or none holds objects, they stop with exit code `2` before analyzing
272
+ anything and list the candidates; set `config.structure.dataset_path` in the
273
+ Inspect file, or pass `--collection` (`--collection customers` or
274
+ `--collection '$.customers[]'`). See
275
+ [Inspect JSON and JSONL files](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/inspect/json.md)
276
+ and the [Inspect format](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/inspect/format.md).
277
+
239
278
  ## Tabalyst Scan
240
279
 
241
- Describe every field of a CSV or JSON file in one JSON document:
280
+ Describe every field of a CSV, JSON or JSONL file in one JSON document:
242
281
 
243
282
  ```console
244
283
  tabalyst scan customers.csv
245
284
  tabalyst scan orders.json --collection "$.customers[]"
285
+ tabalyst scan events.jsonl
246
286
  ```
247
287
 
248
- For a CSV, the default scan is a reusable `scan.json` under Tabalyst's local
288
+ Without `-o` or `-d`, the scan is a reusable `scan.json` under Tabalyst's local
249
289
  storage directory (or `TABALYST_HOME`). The default workflow does not build a
250
290
  DuckDB database. `-o` or `-d` writes
251
291
  a standalone `<stem>.scan.json` export instead. Tabalyst Scan reads the file once as
@@ -260,10 +300,11 @@ interrupted scan never leaves a partial file. Files of 16 MiB or more are
260
300
  analyzed by several worker processes, with the same result; `--workers`
261
301
  chooses their number, `--workers 1` keeps one process.
262
302
 
263
- `tabalyst report customers.csv` reuses the verified stored scan when the CSV
264
- content and requested scan settings are current. It creates a scan when none
265
- exists and atomically replaces a stale one. The source check uses size,
266
- modification time and SHA-256. Existing DuckDB projects from 0.4.3 are left
303
+ `tabalyst report customers.csv` reuses the verified stored scan when the
304
+ source content and requested scan settings are current, and so does a report on
305
+ a JSON or JSONL file. It creates a scan when none exists and atomically replaces
306
+ a stale one. The source check uses the SHA-256 of the whole content, and a scan
307
+ written by another version of Tabalyst is replaced. Existing DuckDB projects from 0.4.3 are left
267
308
  untouched.
268
309
 
269
310
  Build the report from a standalone scan document without reading the source again:
@@ -288,8 +329,8 @@ tabalyst cache clean customers.csv
288
329
  `cache clean` leaves project scans and databases in place. Ordinary scans and
289
330
  reports do not create these query caches.
290
331
 
291
- See [Scan CSV and JSON files](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/how-to/scan-files.md)
292
- and the [scan format](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/reference/scan-format.md).
332
+ See [Scan CSV and JSON files](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/scan/files.md)
333
+ and the [scan format](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/scan/format.md).
293
334
 
294
335
  ## What the report analyzes
295
336
 
@@ -336,14 +377,21 @@ sample = tabalyst.sample_csv(
336
377
 
337
378
  scan = tabalyst.scan("orders.json")
338
379
  scans = tabalyst.generate_scans(["data/*.json"], output_dir="scans")
380
+
381
+ inspection = tabalyst.inspect("orders.json")
382
+ inspections = tabalyst.generate_inspections(["data/*.json", "logs/*.jsonl"])
339
383
  reports = tabalyst.generate_reports(["scans/*.scan.json"], from_scan=True)
340
384
  ```
341
385
 
342
386
  `analyze()` returns the JSON-serializable profile for one report.
343
387
  `generate_reports()` returns the complete batch plan, successes, and failures.
344
- `scan()` returns a scan result without writing anything; `generate_scans()`
345
- writes one standalone `.scan.json` document per source by default. Pass
346
- `project_storage=True` to use the command's scan-only storage for CSV files. `from_scan=True`
388
+ `scan()` returns a scan result without writing anything, at the level of the
389
+ engine: it does not read an Inspect file. `generate_scans()` writes one
390
+ standalone `.scan.json` document per source by default, and applies Inspect to
391
+ JSON and JSONL sources. Pass `project_storage=True` to use the command's
392
+ scan-only storage. `inspect()` writes the Inspect file of one JSON or JSONL
393
+ source and returns its path and document; `generate_inspections()` does it for
394
+ several sources. `from_scan=True`
347
395
  builds reports from standalone scan documents.
348
396
  Expected failures derive from `tabalyst.TabalystError`.
349
397
 
@@ -367,7 +415,8 @@ Configuration files are strict JSON. A minimal file is:
367
415
  ```
368
416
 
369
417
  Analysis settings, CSV reading included, go in the `scan` object, shared by
370
- `tabalyst report` and `tabalyst scan`. Top-level settings shape the report
418
+ `tabalyst report`, `tabalyst scan` and `tabalyst inspect`. The `config` of an
419
+ Inspect file outranks the `scan` object for the source it sits beside. Top-level settings shape the report
371
420
  presentation, and `csv` configures `tabalyst sample`. Explicit CLI or Python
372
421
  arguments override the configuration file, which overrides Tabalyst defaults.
373
422
  No configuration file is loaded unless it is passed with `--config`. See the
@@ -384,7 +433,9 @@ Reports, samples, and scans read files as a stream, with memory bounded by
384
433
  configurable limits, including duplicate-row detection
385
434
  (`scan.limits.max_tracked_records`). Reports and scans show the share of the file read;
386
435
  throughput estimates, recursive directory input, and parallel batch execution
387
- will require later work.
436
+ will require later work. Inspect reads a JSON or JSONL source once from start to
437
+ end, with little memory; on the synthetic benchmarks it took roughly a tenth of the
438
+ time of a scan, a ratio that depends on the data.
388
439
 
389
440
  ## Examples and development
390
441
 
@@ -402,6 +453,8 @@ Additional documentation:
402
453
 
403
454
  - [Architecture](https://github.com/loribel-labs/tabalyst/blob/main/docs/dev/architecture.md)
404
455
  - [Configuration](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/reference/configuration.md)
456
+ - [Inspect JSON and JSONL files](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/inspect/json.md)
457
+ - [Documentation site](https://docs.tabalyst.com/)
405
458
  - [Release procedure](https://github.com/loribel-labs/tabalyst/blob/main/RELEASING.md)
406
459
 
407
460
  Please report defects and feature requests through the
@@ -8,7 +8,8 @@ with structured data.
8
8
  Its first tool is **Tabalyst Report**. The current CSV implementation,
9
9
  **Tabalyst CSV Report**, analyzes a CSV file and produces both a structured JSON
10
10
  profile and a self-contained interactive HTML report.
11
- **Tabalyst Scan** describes CSV and JSON files in a scan document, and
11
+ **Tabalyst Scan** describes CSV, JSON and JSONL files in a scan document,
12
+ **Tabalyst Inspect** finds how to read a JSON or JSONL file, and
12
13
  **Tabalyst Sample** creates smaller CSV files.
13
14
 
14
15
  Tabalyst is in beta. Its interfaces may still change while
@@ -34,6 +35,7 @@ changes often: update it before each new test.
34
35
  | Several files, automatic names | `tabalyst report *.csv` | Beside each source |
35
36
  | Several files, one directory | `tabalyst report *.csv -d reports/` | The directory `reports/` |
36
37
  | One JSON file | `tabalyst report data.json` | `data.report.html` beside the source |
38
+ | One JSONL file | `tabalyst report events.jsonl` | `events.report.html` beside the source |
37
39
  | From a scan document | `tabalyst report --scan data.scan.json` | `data.html` beside the scan |
38
40
 
39
41
  The simplest command keeps the source filename:
@@ -73,16 +75,23 @@ tabalyst report customers.csv -o customer-analysis.html
73
75
 
74
76
  ### JSON files
75
77
 
76
- A JSON file gets a report too, named `<stem>.report.html` so its profile
77
- `<stem>.report.json` never replaces the source. Each collection of records,
78
- such as the `customers` array of `{"customers": [...]}`, is a dataset of the
79
- report, and nested fields are columns named by their path, such as
80
- `address.city`:
78
+ A JSON or JSONL file gets a report too, named `<stem>.report.html` so its
79
+ profile `<stem>.report.json` never replaces the source. The report analyzes one
80
+ collection of records, such as the `customers` array of
81
+ `{"customers": [...]}`, chosen by Tabalyst Inspect, and nested fields are
82
+ columns named by their path, such as `address.city`:
81
83
 
82
84
  ```console
83
85
  tabalyst report orders.json
86
+ tabalyst report events.jsonl
84
87
  ```
85
88
 
89
+ In a JSONL file (`.jsonl` or `.ndjson`), each line is a record. A line that is
90
+ not valid JSON or not an object is excluded and counted, and the report is
91
+ partial. When a JSON file holds several arrays that are equally plausible,
92
+ Tabalyst stops and lists them instead of choosing: see
93
+ [Tabalyst Inspect](#tabalyst-inspect).
94
+
86
95
  ### Multiple files
87
96
 
88
97
  Report several CSV files at once:
@@ -200,16 +209,47 @@ Sampling reads CSV records as a stream. Random and stratified sampling keep
200
209
  only the requested sample, plus stratum counts, in memory. Existing outputs
201
210
  require `--force`, and an input file is never overwritten.
202
211
 
212
+ ## Tabalyst Inspect
213
+
214
+ A JSON file can hold several arrays. **Tabalyst Inspect** reads a JSON, JSONL
215
+ or NDJSON file once, finds which array holds the records and writes its answer
216
+ in `<source>-inspect.json` beside the source:
217
+
218
+ ```console
219
+ tabalyst inspect orders.json
220
+ ```
221
+
222
+ ```text
223
+ Inspect: orders.json-inspect.json
224
+ Selection: $.customers[] (the only eligible collection)
225
+ ```
226
+
227
+ The last section of that file, `config`, holds the rules that `tabalyst scan`
228
+ and `tabalyst report` apply to the source: the collection
229
+ (`structure.dataset_path`), the flatten settings, the array mode and the error
230
+ policy. Edit it, then scan or report as usual. Running `tabalyst inspect` again
231
+ refreshes the detection and keeps your `config`; `--reset-config` replaces it.
232
+
233
+ Inspect is optional: `tabalyst scan` and `tabalyst report` inspect a JSON file
234
+ themselves when it has no Inspect file. When several arrays are equally
235
+ plausible, or none holds objects, they stop with exit code `2` before analyzing
236
+ anything and list the candidates; set `config.structure.dataset_path` in the
237
+ Inspect file, or pass `--collection` (`--collection customers` or
238
+ `--collection '$.customers[]'`). See
239
+ [Inspect JSON and JSONL files](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/inspect/json.md)
240
+ and the [Inspect format](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/inspect/format.md).
241
+
203
242
  ## Tabalyst Scan
204
243
 
205
- Describe every field of a CSV or JSON file in one JSON document:
244
+ Describe every field of a CSV, JSON or JSONL file in one JSON document:
206
245
 
207
246
  ```console
208
247
  tabalyst scan customers.csv
209
248
  tabalyst scan orders.json --collection "$.customers[]"
249
+ tabalyst scan events.jsonl
210
250
  ```
211
251
 
212
- For a CSV, the default scan is a reusable `scan.json` under Tabalyst's local
252
+ Without `-o` or `-d`, the scan is a reusable `scan.json` under Tabalyst's local
213
253
  storage directory (or `TABALYST_HOME`). The default workflow does not build a
214
254
  DuckDB database. `-o` or `-d` writes
215
255
  a standalone `<stem>.scan.json` export instead. Tabalyst Scan reads the file once as
@@ -224,10 +264,11 @@ interrupted scan never leaves a partial file. Files of 16 MiB or more are
224
264
  analyzed by several worker processes, with the same result; `--workers`
225
265
  chooses their number, `--workers 1` keeps one process.
226
266
 
227
- `tabalyst report customers.csv` reuses the verified stored scan when the CSV
228
- content and requested scan settings are current. It creates a scan when none
229
- exists and atomically replaces a stale one. The source check uses size,
230
- modification time and SHA-256. Existing DuckDB projects from 0.4.3 are left
267
+ `tabalyst report customers.csv` reuses the verified stored scan when the
268
+ source content and requested scan settings are current, and so does a report on
269
+ a JSON or JSONL file. It creates a scan when none exists and atomically replaces
270
+ a stale one. The source check uses the SHA-256 of the whole content, and a scan
271
+ written by another version of Tabalyst is replaced. Existing DuckDB projects from 0.4.3 are left
231
272
  untouched.
232
273
 
233
274
  Build the report from a standalone scan document without reading the source again:
@@ -252,8 +293,8 @@ tabalyst cache clean customers.csv
252
293
  `cache clean` leaves project scans and databases in place. Ordinary scans and
253
294
  reports do not create these query caches.
254
295
 
255
- See [Scan CSV and JSON files](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/how-to/scan-files.md)
256
- and the [scan format](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/reference/scan-format.md).
296
+ See [Scan CSV and JSON files](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/scan/files.md)
297
+ and the [scan format](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/scan/format.md).
257
298
 
258
299
  ## What the report analyzes
259
300
 
@@ -300,14 +341,21 @@ sample = tabalyst.sample_csv(
300
341
 
301
342
  scan = tabalyst.scan("orders.json")
302
343
  scans = tabalyst.generate_scans(["data/*.json"], output_dir="scans")
344
+
345
+ inspection = tabalyst.inspect("orders.json")
346
+ inspections = tabalyst.generate_inspections(["data/*.json", "logs/*.jsonl"])
303
347
  reports = tabalyst.generate_reports(["scans/*.scan.json"], from_scan=True)
304
348
  ```
305
349
 
306
350
  `analyze()` returns the JSON-serializable profile for one report.
307
351
  `generate_reports()` returns the complete batch plan, successes, and failures.
308
- `scan()` returns a scan result without writing anything; `generate_scans()`
309
- writes one standalone `.scan.json` document per source by default. Pass
310
- `project_storage=True` to use the command's scan-only storage for CSV files. `from_scan=True`
352
+ `scan()` returns a scan result without writing anything, at the level of the
353
+ engine: it does not read an Inspect file. `generate_scans()` writes one
354
+ standalone `.scan.json` document per source by default, and applies Inspect to
355
+ JSON and JSONL sources. Pass `project_storage=True` to use the command's
356
+ scan-only storage. `inspect()` writes the Inspect file of one JSON or JSONL
357
+ source and returns its path and document; `generate_inspections()` does it for
358
+ several sources. `from_scan=True`
311
359
  builds reports from standalone scan documents.
312
360
  Expected failures derive from `tabalyst.TabalystError`.
313
361
 
@@ -331,7 +379,8 @@ Configuration files are strict JSON. A minimal file is:
331
379
  ```
332
380
 
333
381
  Analysis settings, CSV reading included, go in the `scan` object, shared by
334
- `tabalyst report` and `tabalyst scan`. Top-level settings shape the report
382
+ `tabalyst report`, `tabalyst scan` and `tabalyst inspect`. The `config` of an
383
+ Inspect file outranks the `scan` object for the source it sits beside. Top-level settings shape the report
335
384
  presentation, and `csv` configures `tabalyst sample`. Explicit CLI or Python
336
385
  arguments override the configuration file, which overrides Tabalyst defaults.
337
386
  No configuration file is loaded unless it is passed with `--config`. See the
@@ -348,7 +397,9 @@ Reports, samples, and scans read files as a stream, with memory bounded by
348
397
  configurable limits, including duplicate-row detection
349
398
  (`scan.limits.max_tracked_records`). Reports and scans show the share of the file read;
350
399
  throughput estimates, recursive directory input, and parallel batch execution
351
- will require later work.
400
+ will require later work. Inspect reads a JSON or JSONL source once from start to
401
+ end, with little memory; on the synthetic benchmarks it took roughly a tenth of the
402
+ time of a scan, a ratio that depends on the data.
352
403
 
353
404
  ## Examples and development
354
405
 
@@ -366,6 +417,8 @@ Additional documentation:
366
417
 
367
418
  - [Architecture](https://github.com/loribel-labs/tabalyst/blob/main/docs/dev/architecture.md)
368
419
  - [Configuration](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/reference/configuration.md)
420
+ - [Inspect JSON and JSONL files](https://github.com/loribel-labs/tabalyst/blob/main/docs/en/inspect/json.md)
421
+ - [Documentation site](https://docs.tabalyst.com/)
369
422
  - [Release procedure](https://github.com/loribel-labs/tabalyst/blob/main/RELEASING.md)
370
423
 
371
424
  Please report defects and feature requests through the
@@ -9,7 +9,8 @@ examples/
9
9
  |-- input/
10
10
  | |-- basic.csv
11
11
  | |-- insurance-customers.csv
12
- | `-- orders.json
12
+ | |-- orders.json
13
+ | `-- web-events.jsonl
13
14
  `-- output/
14
15
  |-- basic/
15
16
  | |-- report.html
@@ -20,7 +21,11 @@ examples/
20
21
  | |-- report.json
21
22
  | |-- report/ (34 standalone column pages with --details)
22
23
  | `-- executions.json
23
- `-- orders/
24
+ |-- orders/
25
+ | |-- report.html
26
+ | |-- report.json
27
+ | `-- executions.json
28
+ `-- web-events/
24
29
  |-- report.html
25
30
  |-- report.json
26
31
  `-- executions.json
@@ -30,8 +35,14 @@ examples/
30
35
  synthetic customer and contract records across 34 columns. Its people, contact
31
36
  details, and contracts are fictional; email addresses use reserved `.example`
32
37
  domains. `orders.json` is a synthetic shop export: 60 customers with nested
33
- addresses, tags and orders, whose report has two datasets, `$` and
34
- `$.customers[]`.
38
+ addresses, tags and orders. Its report has one dataset, `$.customers[]`, the only
39
+ collection of arrays of objects that Tabalyst Inspect finds in it.
40
+ `web-events.jsonl` is a synthetic web analytics log in JSON Lines: 300 events with
41
+ nested `user` and `device` objects, tags arrays, null values and an optional
42
+ field that appears late, plus three bad lines (a truncated record, a line that
43
+ is not JSON and a JSON array instead of an object). Its report is partial on
44
+ purpose: the three lines are excluded and counted, the 300 events are analyzed.
45
+ Its people and identifiers are fictional.
35
46
 
36
47
  From the repository root, regenerate the outputs with:
37
48
 
@@ -39,6 +50,7 @@ From the repository root, regenerate the outputs with:
39
50
  tabalyst report examples/input/basic.csv -o examples/output/basic/report.html --config examples/config.json --force
40
51
  tabalyst report examples/input/insurance-customers.csv -o examples/output/insurance-customers/report.html --config examples/config.json --details --force
41
52
  tabalyst report examples/input/orders.json -o examples/output/orders/report.html --config examples/config.json --force
53
+ tabalyst report examples/input/web-events.jsonl -o examples/output/web-events/report.html --config examples/config.json --force
42
54
  ```
43
55
 
44
56
  Every output folder is independent and contains its own HTML report, canonical
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "tabalyst"
7
- version = "0.4.4"
7
+ version = "0.5.1"
8
8
  description = "Open-source toolkit for understanding and working with unfamiliar data."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -69,6 +69,7 @@ testpaths = ["tests"]
69
69
  pythonpath = ["src"]
70
70
  markers = [
71
71
  "lot(name): Tabalyst Scan implementation lot that provides the tested behavior",
72
+ "inspect_lot(name): Tabalyst Inspect implementation lot that provides the tested behavior",
72
73
  ]
73
74
 
74
75
  [tool.ruff]
@@ -2,6 +2,9 @@
2
2
 
3
3
  from tabalyst._version import __version__
4
4
  from tabalyst.errors import ConfigurationError, InputError, ReportError, TabalystError
5
+ from tabalyst.inspect_service import InspectResult as InspectResult
6
+ from tabalyst.inspect_service import generate_inspections as generate_inspections
7
+ from tabalyst.inspect_service import inspect as inspect
5
8
  from tabalyst.report_config import ReportConfig as ReportConfig
6
9
  from tabalyst.report_service import generate_reports as generate_reports
7
10
  from tabalyst.reporting import render_report as render_report
@@ -20,15 +23,18 @@ from tabalyst.service import analyze_scan as analyze_scan
20
23
  __all__ = [
21
24
  "ConfigurationError",
22
25
  "InputError",
26
+ "InspectResult",
23
27
  "ReportError",
24
28
  "ScanConfig",
25
29
  "ScanResult",
26
30
  "TabalystError",
27
31
  "__version__",
28
32
  "analyze",
33
+ "generate_inspections",
29
34
  "generate_reports",
30
35
  "generate_samples",
31
36
  "generate_scans",
37
+ "inspect",
32
38
  "sample_csv",
33
39
  "scan",
34
40
  ]
@@ -13,6 +13,15 @@ from pathlib import Path
13
13
 
14
14
  from tabalyst.errors import ConfigurationError, InputError, ReportError
15
15
 
16
+ # The visible Inspect file of a source is its full name plus this suffix
17
+ # (design inspect 12.1). It is never a source itself.
18
+ INSPECT_SUFFIX = "-inspect.json"
19
+
20
+
21
+ def is_inspect_name(name: str) -> bool:
22
+ """Whether a file name is that of an Inspect file."""
23
+ return name.endswith(INSPECT_SUFFIX)
24
+
16
25
 
17
26
  def path_key(path: Path) -> str:
18
27
  """Identity of a path for comparisons, whatever its spelling and case."""
@@ -20,7 +29,11 @@ def path_key(path: Path) -> str:
20
29
 
21
30
 
22
31
  def resolve_input_specs(input_specs: Sequence[str | Path]) -> list[Path]:
23
- """Resolve explicit files and shell-independent, non-recursive glob patterns."""
32
+ """Resolve explicit files and shell-independent, non-recursive glob patterns.
33
+
34
+ A pattern skips the Inspect files it matches; an explicit path to one is
35
+ refused (design inspect 12.1).
36
+ """
24
37
  if not input_specs:
25
38
  raise ConfigurationError("At least one input file is required.")
26
39
 
@@ -34,7 +47,11 @@ def resolve_input_specs(input_specs: Sequence[str | Path]) -> list[Path]:
34
47
  )
35
48
  if glob.has_magic(text):
36
49
  matches = sorted(
37
- (Path(match) for match in glob.glob(text) if Path(match).is_file()),
50
+ (
51
+ Path(match)
52
+ for match in glob.glob(text)
53
+ if Path(match).is_file() and not is_inspect_name(Path(match).name)
54
+ ),
38
55
  key=path_key,
39
56
  )
40
57
  if not matches:
@@ -45,6 +62,11 @@ def resolve_input_specs(input_specs: Sequence[str | Path]) -> list[Path]:
45
62
  raise InputError(f"Input file does not exist: {path}")
46
63
  if not path.is_file():
47
64
  raise InputError(f"Input path is not a file: {path}")
65
+ if is_inspect_name(path.name):
66
+ raise ConfigurationError(
67
+ f"{path} is an Inspect file, not a source. Give the source it "
68
+ "describes; edit the Inspect file to change how it is read."
69
+ )
48
70
  matches = [path]
49
71
 
50
72
  for path in matches:
@@ -6,6 +6,7 @@ import typer
6
6
 
7
7
  from tabalyst import __version__
8
8
  from tabalyst.cli.cache import cache_app
9
+ from tabalyst.cli.inspect import inspect_command
9
10
  from tabalyst.cli.report import report_command
10
11
  from tabalyst.cli.sample import sample_command
11
12
  from tabalyst.cli.scan import scan_command
@@ -39,6 +40,7 @@ def root(
39
40
  """Understand, explore, clean, and validate structured data."""
40
41
 
41
42
 
43
+ app.command("inspect")(inspect_command)
42
44
  app.command("report")(report_command)
43
45
  app.command("sample")(sample_command)
44
46
  app.command("scan")(scan_command)
@@ -46,5 +48,10 @@ app.add_typer(cache_app, name="cache")
46
48
 
47
49
 
48
50
  def main() -> None:
49
- """Run the Tabalyst command-line application."""
50
- app()
51
+ """Run the Tabalyst command-line application.
52
+
53
+ Windows wildcard expansion is left to Tabalyst: its own pattern resolver
54
+ skips the Inspect files that a shell-expanded ``*.json`` would pass as
55
+ explicit paths.
56
+ """
57
+ app(windows_expand_args=False)