sketch-tastic 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,227 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[codz]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py.cover
50
+ *.lcov
51
+ .hypothesis/
52
+ .pytest_cache/
53
+ cover/
54
+
55
+ # Translations
56
+ *.mo
57
+ *.pot
58
+
59
+ # Django stuff:
60
+ *.log
61
+ local_settings.py
62
+ db.sqlite3
63
+ db.sqlite3-journal
64
+
65
+ # Flask stuff:
66
+ instance/
67
+ .webassets-cache
68
+
69
+ # Scrapy stuff:
70
+ .scrapy
71
+
72
+ # Sphinx documentation
73
+ docs/_build/
74
+
75
+ # PyBuilder
76
+ .pybuilder/
77
+ target/
78
+
79
+ # Jupyter Notebook
80
+ .ipynb_checkpoints
81
+
82
+ # IPython
83
+ profile_default/
84
+ ipython_config.py
85
+
86
+ # pyenv
87
+ # For a library or package, you might want to ignore these files since the code is
88
+ # intended to run in multiple environments; otherwise, check them in:
89
+ # .python-version
90
+
91
+ # pipenv
92
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
93
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
94
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
95
+ # install all needed dependencies.
96
+ # Pipfile.lock
97
+
98
+ # uv
99
+ # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
100
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
101
+ # commonly ignored for libraries.
102
+ # uv.lock
103
+
104
+ # poetry
105
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
106
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
107
+ # commonly ignored for libraries.
108
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
109
+ # poetry.lock
110
+ # poetry.toml
111
+
112
+ # pdm
113
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
114
+ # pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
115
+ # https://pdm-project.org/en/latest/usage/project/#working-with-version-control
116
+ # pdm.lock
117
+ # pdm.toml
118
+ .pdm-python
119
+ .pdm-build/
120
+
121
+ # pixi
122
+ # Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
123
+ # pixi.lock
124
+ # Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
125
+ # in the .venv directory. It is recommended not to include this directory in version control.
126
+ .pixi/*
127
+ !.pixi/config.toml
128
+
129
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
130
+ __pypackages__/
131
+
132
+ # Celery stuff
133
+ celerybeat-schedule*
134
+ celerybeat.pid
135
+
136
+ # Redis
137
+ *.rdb
138
+ *.aof
139
+ *.pid
140
+
141
+ # RabbitMQ
142
+ mnesia/
143
+ rabbitmq/
144
+ rabbitmq-data/
145
+
146
+ # ActiveMQ
147
+ activemq-data/
148
+
149
+ # SageMath parsed files
150
+ *.sage.py
151
+
152
+ # Environments
153
+ .env
154
+ .envrc
155
+ .venv
156
+ env/
157
+ venv/
158
+ ENV/
159
+ env.bak/
160
+ venv.bak/
161
+
162
+ # Spyder project settings
163
+ .spyderproject
164
+ .spyproject
165
+
166
+ # Rope project settings
167
+ .ropeproject
168
+
169
+ # mkdocs/Zensical documentation
170
+ /site
171
+
172
+ # mypy
173
+ .mypy_cache/
174
+ .dmypy.json
175
+ dmypy.json
176
+
177
+ # Pyre type checker
178
+ .pyre/
179
+
180
+ # pytype static type analyzer
181
+ .pytype/
182
+
183
+ # Cython debug symbols
184
+ cython_debug/
185
+
186
+ # PyCharm
187
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
188
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
189
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
190
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
191
+ # .idea/
192
+
193
+ # Abstra
194
+ # Abstra is an AI-powered process automation framework.
195
+ # Ignore directories containing user credentials, local state, and settings.
196
+ # Learn more at https://abstra.io/docs
197
+ .abstra/
198
+
199
+ # Visual Studio Code
200
+ # Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore that
201
+ # can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
202
+ # and can be added to the global gitignore or merged into this file. However, if you prefer, you
203
+ # could uncomment the following to ignore the entire vscode folder
204
+ # .vscode/
205
+ # Temporary file for partial code execution
206
+ tempCodeRunnerFile.py
207
+
208
+ # Ruff stuff:
209
+ .ruff_cache/
210
+
211
+ # PyPI configuration file
212
+ .pypirc
213
+
214
+ # Marimo
215
+ marimo/_static/
216
+ marimo/_lsp/
217
+ __marimo__/
218
+
219
+ # Streamlit
220
+ .streamlit/secrets.toml
221
+
222
+ # Bob Protocol / via — machine-local config and index, not portable
223
+ .mcp.json
224
+ .via/
225
+
226
+ # Downloaded example data (make gharchive); removed by make clean
227
+ data/
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Drew Gutstein
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,456 @@
1
+ Metadata-Version: 2.5
2
+ Name: sketch-tastic
3
+ Version: 0.2.0
4
+ Summary: Mergeable sketches of JSONL event streams for regression testing and flow analysis.
5
+ Project-URL: Repository, https://github.com/drusifer/sketch-tastic
6
+ Author: drusifer
7
+ License-Expression: MIT
8
+ License-File: LICENSE
9
+ Keywords: events,fingerprint,jsonl,jsonpath,logs,regression,sketch
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Environment :: Console
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Intended Audience :: System Administrators
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.14
16
+ Classifier: Programming Language :: Python :: 3.15
17
+ Classifier: Topic :: System :: Logging
18
+ Classifier: Topic :: Utilities
19
+ Requires-Python: >=3.14
20
+ Requires-Dist: jsonpath-rfc9535>=2.0.2
21
+ Requires-Dist: jsonschema>=4.23
22
+ Requires-Dist: pyyaml>=6.0
23
+ Description-Content-Type: text/markdown
24
+
25
+ # sketch-tastic
26
+
27
+ Mergeable sketches of JSONL event streams for regression testing and flow analysis.
28
+
29
+ `sketchtastic` reads events — logs, traces, audit records, anything with one JSON object per
30
+ line — and reduces each to a **skit**: a stable ID for "this shape of event". It keeps a
31
+ **sketch** (the data-sketch term for a compact, mergeable summary of a stream) with one entry per
32
+ skit: how often it was seen, statistics you ask for, and a sample. The sketch is a file that
33
+ grows across runs:
34
+
35
+ ```
36
+ existing sketch + stream of events -> updated sketch
37
+ ```
38
+
39
+ A million near-identical log lines become a handful of entries you can actually read.
40
+
41
+ ## Install
42
+
43
+ ```sh
44
+ pip install sketch-tastic
45
+ ```
46
+
47
+ Python 3.14 or newer. Installs the `sketchtastic` command and its short alias `skt` (also
48
+ runnable as `python -m sketchtastic`).
49
+
50
+ Moved from [drusifer/fingerling](https://github.com/drusifer/fingerling), where it was
51
+ `fingerprint-sampler`. While the version is 0.x, releases may break profiles and sketch files
52
+ without a migration path.
53
+
54
+ ## Quickstart
55
+
56
+ Paste this into a shell. It writes a profile and a few events, then sketches them:
57
+
58
+ ```sh
59
+ cat > profile.yaml <<'YAML'
60
+ version: 0
61
+ fields:
62
+ - {key: level, path: "$.level"}
63
+ - {key: component, path: "$.logger"}
64
+ - {key: failed, path: "$.error", modifier: IS_PRESENT}
65
+ aggregates:
66
+ - {key: took_ms, path: "$.took_ms", fn: [MIN, MAX, AVG]}
67
+ - {key: example, path: "$.msg", fn: {name: LAST, n: 2}}
68
+ YAML
69
+
70
+ cat > events.jsonl <<'JSONL'
71
+ {"level": "INFO", "logger": "db", "msg": "query ok", "took_ms": 3}
72
+ {"level": "INFO", "logger": "db", "msg": "query ok (cached)", "took_ms": 1}
73
+ {"level": "ERROR", "logger": "db", "msg": "query timed out", "error": "TimeoutError", "took_ms": 5000}
74
+ {"level": "INFO", "logger": "http", "msg": "GET /health", "took_ms": 0.4}
75
+ JSONL
76
+
77
+ sketchtastic merge -p profile.yaml -s app.sketch.yaml events.jsonl
78
+ ```
79
+
80
+ ```
81
+ sketchtastic: 4 events read from events.jsonl
82
+ skits: 3 new, 0 updated, 3 total in sketch
83
+ skipped: 0 malformed lines, 0 unusable aggregate values
84
+ wrote app.sketch.yaml (profile 3f0dd0d9f280ba32064a891dcd3b17cb)
85
+ history: app.sketch.history/261014-0001.yaml
86
+ ```
87
+
88
+ Four events, three skits: the two successful `db` queries share one. Run the same command
89
+ again and the counts add up instead of new entries appearing — that is the merge. Each run also
90
+ leaves a file with just its own skits in `app.sketch.history/` (see History below).
91
+
92
+ `app.sketch.yaml` now starts like this (the other two skits follow):
93
+
94
+ ```yaml
95
+ version: 0
96
+ profile_id: 3f0dd0d9f280ba32064a891dcd3b17cb
97
+ last_run: 261014-0001
98
+ skits:
99
+ 9312137ea290aa8d196e7036cd687af8:
100
+ tokens:
101
+ level: INFO
102
+ component: db
103
+ failed: 'FALSE'
104
+ aggregates:
105
+ took_ms:
106
+ MIN: 1
107
+ MAX: 3
108
+ AVG:
109
+ avg: 2.0
110
+ sum: 4
111
+ count: 2
112
+ example:
113
+ - query ok
114
+ - query ok (cached)
115
+ meta:
116
+ count: 2
117
+ first_seen: 261014-0001
118
+ last_seen: 261014-0001
119
+ first:
120
+ - meta: {run: 261014-0001}
121
+ record: {level: INFO, logger: db, msg: query ok, took_ms: 3}
122
+ last:
123
+ - meta: {run: 261014-0001}
124
+ record: {level: INFO, logger: db, msg: query ok (cached), took_ms: 1}
125
+ share:
126
+ meta:
127
+ count: 50.0
128
+ ```
129
+
130
+ To read a sketch without the YAML, print it as a table with `sketchtastic show app.sketch.yaml`:
131
+
132
+ ```
133
+ ID COUNT COUNT % FIRST RUN LAST RUN component failed level
134
+ d7d629a1 1 25.0 261014-0001 261014-0001 db TRUE ERROR
135
+ f5189272 1 25.0 261014-0001 261014-0001 http FALSE INFO
136
+ 9312137e 2 50.0 261014-0001 261014-0001 db FALSE INFO
137
+ ```
138
+
139
+ ## Usage
140
+
141
+ ```
142
+ sketchtastic merge -p PROFILE -s SKETCH [EVENTS|-] [-o OUTPUT] [-j N] [--force-profile]
143
+ ```
144
+
145
+ | Option | Meaning |
146
+ |---|---|
147
+ | `-p`, `--profile PROFILE` | YAML profile (below). Required. |
148
+ | `-s`, `--sketch SKETCH` | Sketch to merge into. Created if missing, **updated in place**. Required. |
149
+ | `EVENTS` | JSONL file; `-` or omitted reads stdin. |
150
+ | `-o`, `--output OUTPUT` | Write the merged sketch here instead; `SKETCH` is left unchanged. |
151
+ | `-j`, `--jobs N` | Worker processes that fold the events (default: all cores). `-j 1`, or input of up to 5,000 lines, folds in-process. |
152
+ | `--force-profile` | Merge even though `SKETCH` was built with a different profile. |
153
+
154
+ ```sh
155
+ # read from a pipe
156
+ tail -n 1000 app.jsonl | sketchtastic merge -p profile.yaml -s app.sketch.yaml -
157
+
158
+ # trial run: see the result without touching the sketch
159
+ sketchtastic merge -p profile.yaml -s app.sketch.yaml -o /tmp/preview.yaml new.jsonl
160
+ ```
161
+
162
+ The summary goes to stderr and stdout stays empty. Exit status: `0` sketch written and every line
163
+ used · `1` sketch written but some lines were skipped (listed by line number) · `2` nothing written
164
+ · `3` sketch written but its history file was not (takes precedence over `1`).
165
+
166
+ **Parallel merge.** With more than one job, the input is read in batches of 5,000 lines that
167
+ worker processes fold into sketches of their own, merged at the end. Counts, sums, MIN/MAX, AVG,
168
+ first and last run, the sequence mark and the summary are exactly what one process gives;
169
+ `FIRST`/`LAST` samples (and the built-in first and last record) come from matching events, but
170
+ with several jobs not necessarily the very first or last ones, since any of them shows the same
171
+ shape. Skipped-line warnings still name the right lines, in line order.
172
+
173
+ ### History
174
+
175
+ Each run gets a **run id**, `YYMMDD-SEQ`: the UTC date, then a number that only goes up for this
176
+ sketch (`261015-0003`). The sketch keeps the latest as `last_run`, and every skit's
177
+ `first_seen` / `last_seen` name the first and last run that saw it.
178
+
179
+ Every run that updates `SKETCH` also writes just that run's skits to a file named by its run
180
+ id in `<SKETCH without extension>.history/`:
181
+
182
+ ```
183
+ web.sketch.yaml # last_run: 261015-0003
184
+ web.sketch.history/
185
+ 261014-0001.yaml
186
+ 261014-0002.yaml
187
+ 261015-0003.yaml
188
+ ```
189
+
190
+ So `last_seen: 261014-0002` points straight at the file for that run. A run file is a sketch
191
+ (counts, aggregates and shares from that run only), so `sketchtastic show` reads it. It starts
192
+ with a `run:` header: `id`, `started`, `finished` (UTC), `source`, `events`, `skipped_lines` and
193
+ `sketch`. A new run's number is one past the highest in the sketch and its history, so a rebuilt sketch
194
+ never overwrites an old run file. History is always on; a run with `-o`, or one that writes
195
+ nothing, leaves no run file and uses up no run id. Nothing is pruned.
196
+
197
+ ### Duplicate protection
198
+
199
+ If your events carry a number that only grows (an event id, an offset), name it in the profile:
200
+
201
+ ```yaml
202
+ sequence: $.id
203
+ ```
204
+
205
+ The sketch then keeps the highest sequence merged so far (`sequence: {path: $.id, max: 34541893465}`)
206
+ and every later run drops events at or below it, so running twice over the same file, or over
207
+ files that overlap, adds nothing twice:
208
+
209
+ ```
210
+ dropped: 180387 events already merged (sequence ≤ 34541893465)
211
+ ```
212
+
213
+ Events are compared with the mark as it stood when the run started, so events slightly out of
214
+ order within one run are all kept (and duplicates within one run are not caught). Numbers and
215
+ numeric strings work. Duplicate detection has to be exact, so an event whose sequence is
216
+ missing, not a number, or selected more than once is an **error**: the run stops at that line,
217
+ nothing is written (no sketch update, no run file) and the exit status is `2`. The run file's header records the range
218
+ merged, `sequence: {min, max}`. Changing the profile's `sequence` path makes the next run ignore
219
+ the old mark and say so.
220
+
221
+ ### Check a profile
222
+
223
+ ```
224
+ sketchtastic check PROFILE [PROFILE ...]
225
+ ```
226
+
227
+ Checks profiles without reading any events and lists **every** problem, one per line, with the
228
+ file and where in it:
229
+
230
+ ```
231
+ app.yaml: fields[0] (key 'route') → modifier: 'replace' is a required property
232
+ app.yaml: fields[1] (key 'method') → modifier.flags: 'q' does not match '^[imsx]*$'
233
+ ```
234
+
235
+ A valid profile prints `app.yaml: OK`. Exit status: `0` all valid · `1` problems found. The
236
+ structure is checked against a JSON Schema (`profile.schema.json`, shipped in the package) by the
237
+ standard [`jsonschema`](https://python-jsonschema.readthedocs.io/) validator. Regexes, JSONPaths,
238
+ `IN_RANGE` bounds, duplicate keys and aggregate functions are checked as well. `merge` runs the
239
+ same check first and writes nothing if it fails.
240
+
241
+ **In your editor.** Put this line at the top of a profile, and an editor with YAML language-server
242
+ support (for example VS Code with the Red Hat YAML extension) checks the profile as you type and
243
+ completes modifier names:
244
+
245
+ ```yaml
246
+ # yaml-language-server: $schema=https://raw.githubusercontent.com/drusifer/sketch-tastic/main/src/sketchtastic/profile.schema.json
247
+ ```
248
+
249
+ ### Show a sketch
250
+
251
+ ```
252
+ sketchtastic show [--sort KEY] SKETCH
253
+ ```
254
+
255
+ Prints one row per skit, least frequent first: the first 8 characters of its ID, its count
256
+ and `COUNT %` (its share of all events), the first and last run that saw it, then a value and
257
+ a `%` column for each `SUM` or `COUNT` aggregate (`payload_bytes.SUM`, `payload_bytes.SUM %`), and
258
+ one column per token. Shares come from the sketch file's `share` section; a sketch written before
259
+ shares existed shows `-` until its next merge. A sketch with no skits prints
260
+ `SKETCH: no skits`. Long values are not cut; pipe to `less -S` to scroll sideways.
261
+
262
+ `--sort KEY` orders the rows by the column headed KEY, in any case, lowest first (default
263
+ `COUNT`), so the largest are at the end: `sketchtastic show --sort 'payload_bytes.SUM %'
264
+ gh.sketch.yaml | tail` shows the heaviest skits. `-` sorts first, numbers before text, and
265
+ equal values keep file order. Exit status: `0` printed · `2` the sketch file could not be read,
266
+ or KEY is not one of its columns (the message says why, and lists the columns).
267
+
268
+ ## Profiles
269
+
270
+ ```yaml
271
+ version: 0
272
+
273
+ # fields define identity: events with the same tokens share one skit
274
+ fields:
275
+ - key: method
276
+ path: "$.request.method" # modifier defaults to DEFAULT
277
+ - key: status_ok
278
+ path: "$.response.status"
279
+ modifier: {name: IN_RANGE, low: 200, high: 399}
280
+ - key: slow_upstreams
281
+ path: "$.upstreams[?(@.ms > 500)].name" # full JSONPath, including filters
282
+
283
+ # aggregates are tracked per skit and never affect identity
284
+ aggregates:
285
+ - key: latency_ms
286
+ path: "$.response.ms"
287
+ fn: [MIN, MAX, AVG]
288
+ - key: recent_paths
289
+ path: "$.request.path"
290
+ fn: {name: LAST, n: 3}
291
+ ```
292
+
293
+ **Paths** are standard JSONPath ([RFC 9535](https://www.rfc-editor.org/rfc/rfc9535), evaluated by
294
+ [jsonpath-rfc9535](https://github.com/jg-rp/python-jsonpath-rfc9535)), with no extensions: dot and bracket access, `[*]`, slices, negative indexes, unions,
295
+ recursive descent (`..`) and filters. If a path selects several values, each is tokenized and the
296
+ results are de-duplicated, sorted and joined, so array order never changes a skit.
297
+
298
+ **Modifiers** turn the selected value into a token. Pick ones that collapse high-cardinality
299
+ values (IDs, timestamps, sizes) so similar events share a skit:
300
+
301
+ | Modifier | Arguments | Token |
302
+ |---|---|---|
303
+ | `DEFAULT` | — | the value as a string; `NULL` if absent |
304
+ | `IS_PRESENT` | — | `TRUE` if the key exists, even when its value is `null` |
305
+ | `IS_NULL` | — | `TRUE` only if the key exists and is set to `null` (a missing key is `FALSE`) |
306
+ | `IN_RANGE` | `low`, `high` (numbers) | `TRUE` if `low <= value <= high` |
307
+ | `ANY` | `values` (a list) | `TRUE` if the value is one of those listed |
308
+ | `GLOB` | `pattern` | `TRUE` if the whole value matches the shell-style pattern (`*`, `?`, `[a-z]`), case-sensitive |
309
+ | `REGEX` | `pattern`, optional `flags` | `TRUE` if the regular expression matches anywhere in the value; anchor with `^` and `$` |
310
+ | `REGSUB` | `pattern`, `replace`, optional `flags` | the value with every match replaced (`\1` refers to a group); `NULL` if absent |
311
+ | `TRUNC` | `unit` | a `DATE`/`DATETIME` value cut to `year` … `second`, e.g. hour → `2024-01-01T15` |
312
+
313
+ A modifier without arguments is written as its bare name (`modifier: IS_PRESENT`); one with
314
+ arguments is a mapping with `name` and its arguments:
315
+
316
+ ```yaml
317
+ - key: status_ok
318
+ path: $.response.status
319
+ modifier: {name: IN_RANGE, low: 200, high: 399}
320
+ - key: port_ok
321
+ path: $.dport
322
+ modifier: {name: ANY, values: [80, 443]}
323
+ - key: in_tmp
324
+ path: $.file
325
+ modifier: {name: GLOB, pattern: '/tmp/*'} # TRUE / FALSE
326
+ - key: route
327
+ path: $.request.path
328
+ modifier: {name: REGSUB, pattern: '\d+', replace: N} # /items/17 -> /items/N
329
+ - key: greeting
330
+ path: $.body.text
331
+ modifier:
332
+ name: REGEX
333
+ pattern: '^hello\s.*\sgoodbye\.$'
334
+ flags: i
335
+ ```
336
+
337
+ `flags` is any of `i` (ignore case), `m` (`^` and `$` match at line breaks), `s` (`.` matches a
338
+ newline) and `x` (verbose). Patterns use Python's `re` syntax. Write patterns in single quotes:
339
+ YAML leaves backslashes alone there, and a pattern starting with `*`, `[` or `{`, or containing
340
+ `: ` or ` #`, would otherwise not parse as you meant. Changing a field's modifier changes the
341
+ profile's identity, like any other change to `fields`.
342
+
343
+ **Dates and times.** Add `type: DATETIME` (or `type: DATE`) to a field or aggregate whose path
344
+ holds ISO-8601 text. Values are compared as real times in UTC, so `MIN`/`MAX` are right across
345
+ time zones, and the sketch stores them as YAML timestamps. A value that isn't valid ISO-8601 counts as
346
+ missing (and is skipped by aggregates). For fields, `TRUNC` buckets a time so events group
347
+ together:
348
+
349
+ ```yaml
350
+ - key: hour
351
+ path: $.created_at
352
+ type: DATETIME
353
+ modifier: {name: TRUNC, unit: hour} # year, month, day, hour, minute or second
354
+ ```
355
+
356
+ **Aggregates** are computed across every event that lands on the same skit, over all runs:
357
+
358
+ | Function | Keeps |
359
+ |---|---|
360
+ | `SUM` | running sum of numbers |
361
+ | `MIN`, `MAX` | smallest / largest — numbers, dates and times (with `type`), or text |
362
+ | `AVG` | the average, with the sum and count that make it mergeable |
363
+ | `COUNT` | how many values the path selected (of any type) |
364
+ | `{name: FIRST, n: N}` | the `N` earliest values of the path (with `-j` above 1: some early values) |
365
+ | `{name: LAST, n: N}` | the `N` most recent values of the path (with `-j` above 1: some late values) |
366
+
367
+ `fn` is one function or a list, written like modifiers: a bare name, or a mapping for `FIRST`
368
+ and `LAST`. With one function the sketch stores its value directly (`count: 12`); with several, a
369
+ map by function name (`{MIN: 1, MAX: 3}`). Every skit holds every aggregate, in the order the
370
+ profile lists them (functions too), from its first event on: before any value, `SUM` and `COUNT`
371
+ are `0`, `FIRST`/`LAST` are `[]`, `AVG` is `{avg: null, sum: 0, count: 0}`, and `MIN`/`MAX` are
372
+ `null` (`-` in `show`). So the profile alone decides the order of keys in the file and of
373
+ columns in `show`. Numbers and numeric strings count as numbers. A value
374
+ a function can't use (text for `SUM`, text after numbers for `MIN`) is skipped and counted in the
375
+ summary as an unusable aggregate value. A record where the path matches nothing is simply not
376
+ aggregated and is not counted.
377
+
378
+ **Weighted aggregates.** Add `op: SIZE` to aggregate each value's size in bytes instead of the
379
+ value itself: UTF-8 bytes for text, the compact JSON encoding for anything else (`null` is 4).
380
+ Every function gets the size, so one entry gives `SUM(SIZE(payload))` and more:
381
+
382
+ ```yaml
383
+ aggregates:
384
+ - {key: payload_bytes, path: $.payload, op: SIZE, fn: [SUM, AVG, MAX]}
385
+ ```
386
+
387
+ `op` and `type` cannot be combined on one aggregate. Like any aggregate change, adding an `op`
388
+ keeps the profile id, so existing sketches keep merging.
389
+
390
+ `merge` checks the profile before reading any event; run `sketchtastic check` (above) to see
391
+ every problem at once.
392
+
393
+ ## The sketch
394
+
395
+ - `tokens` shows what each ID stands for.
396
+ - `aggregates` holds what your profile asked for. `meta` holds what every skit gets for
397
+ free: `count`, `first_seen` / `last_seen` (the first and last run id that saw it, see History
398
+ above), and the first and last record (one each, to keep
399
+ sketches and run files small; add `FIRST`/`LAST` aggregates to your profile for more samples).
400
+ - `meta` comes from a small built-in profile shipped with the package (`builtin.yaml`), which
401
+ uses the same aggregate functions you can. It reads each record wrapped in an envelope,
402
+ `{meta: {run: <run id>}, record: <the parsed record>}`, so the fields the tool adds can never
403
+ clash with the record's own, and the stored first/last records carry the run they came from.
404
+ - `share` gives each skit's percentage of the whole sketch for every value that adds up:
405
+ `share.meta.count` is its share of all events, and `share.aggregates` holds the share of each
406
+ `SUM` and `COUNT`, laid out like `aggregates` (`payload_bytes: {SUM: 31.25}`, or `requests: 25.0`
407
+ for a single-function aggregate). It is recomputed on every write; a value whose total is 0 has
408
+ no share.
409
+ - The tool records runs, not times, per skit; a run's start and finish times are in its run
410
+ file. To track a time field inside your records, add it to your profile's `aggregates`.
411
+ - A sketch records the profile that built it (`profile_id`, derived from `fields`). Merging under a
412
+ profile with different `fields` is refused, because the skits would not be comparable.
413
+ Changing `aggregates` is always fine.
414
+ - Splitting a stream across several runs gives the same sketch as one run over all of it.
415
+ - The file is replaced atomically; a failed or interrupted run leaves the previous sketch intact.
416
+
417
+ ## As a library
418
+
419
+ ```python
420
+ from sketchtastic import Sketch, load_profile
421
+
422
+ profile = load_profile("profile.yaml")
423
+ sketch = Sketch(profile.aggregates, profile.profile_id)
424
+
425
+ event = {"level": "INFO", "logger": "db", "msg": "query ok", "took_ms": 3}
426
+ skit_id = sketch.upsert(profile.tokens(event), event, run="261015-0001")
427
+
428
+ sketch.get(skit_id).meta["count"] # 1
429
+ sketch.save("app.sketch.yaml")
430
+ ```
431
+
432
+ `load_profile` and `parse_profile` raise `ProfileError`; its `problems` attribute lists every
433
+ problem, located, in the same form `sketchtastic check` prints. `check_profile(data, source)`
434
+ returns `(profile_or_None, problems)` without raising.
435
+
436
+ Without a profile file, build the pieces directly. `Field` and `Aggregate` take the same keys
437
+ as the YAML (`key`, `path`, `modifier` / `fn`) and are checked when created (`ValueError` if
438
+ wrong):
439
+
440
+ ```python
441
+ from sketchtastic import Field, Profile
442
+
443
+ profile = Profile(fields=(
444
+ Field("method", "$.request.method"),
445
+ Field("route", "$.request.path", {"name": "REGSUB", "pattern": r"\d+", "replace": "N"}),
446
+ ))
447
+ profile.tokens({"request": {"method": "GET", "path": "/items/17"}})
448
+ # {'method': 'GET', 'route': '/items/N'}
449
+ ```
450
+
451
+ Every modifier and aggregate function is a `Rule` subclass, registered by its `name`; the
452
+ schema checks its arguments and the class checks the rest.
453
+
454
+ ## License
455
+
456
+ MIT