sketch-tastic 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sketch_tastic-0.2.0/.gitignore +227 -0
- sketch_tastic-0.2.0/LICENSE +21 -0
- sketch_tastic-0.2.0/PKG-INFO +456 -0
- sketch_tastic-0.2.0/README.md +432 -0
- sketch_tastic-0.2.0/examples/access_log.jsonl +8 -0
- sketch_tastic-0.2.0/examples/access_log.yaml +30 -0
- sketch_tastic-0.2.0/examples/gharchive.yaml +49 -0
- sketch_tastic-0.2.0/pyproject.toml +63 -0
- sketch_tastic-0.2.0/src/sketchtastic/__init__.py +51 -0
- sketch_tastic-0.2.0/src/sketchtastic/__main__.py +13 -0
- sketch_tastic-0.2.0/src/sketchtastic/_yaml.py +24 -0
- sketch_tastic-0.2.0/src/sketchtastic/aggregators.py +319 -0
- sketch_tastic-0.2.0/src/sketchtastic/builtin.yaml +10 -0
- sketch_tastic-0.2.0/src/sketchtastic/cli.py +794 -0
- sketch_tastic-0.2.0/src/sketchtastic/model.py +358 -0
- sketch_tastic-0.2.0/src/sketchtastic/profile.py +142 -0
- sketch_tastic-0.2.0/src/sketchtastic/profile.schema.json +571 -0
- sketch_tastic-0.2.0/src/sketchtastic/schema.py +59 -0
- sketch_tastic-0.2.0/src/sketchtastic/sketch.py +260 -0
- sketch_tastic-0.2.0/tests/test_aggregators.py +188 -0
- sketch_tastic-0.2.0/tests/test_cli.py +1050 -0
- sketch_tastic-0.2.0/tests/test_model.py +450 -0
- sketch_tastic-0.2.0/tests/test_profile.py +402 -0
- sketch_tastic-0.2.0/tests/test_sketch.py +579 -0
- sketch_tastic-0.2.0/tests/test_types.py +132 -0
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[codz]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
share/python-wheels/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.installed.cfg
|
|
26
|
+
*.egg
|
|
27
|
+
MANIFEST
|
|
28
|
+
|
|
29
|
+
# PyInstaller
|
|
30
|
+
# Usually these files are written by a python script from a template
|
|
31
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
32
|
+
*.manifest
|
|
33
|
+
*.spec
|
|
34
|
+
|
|
35
|
+
# Installer logs
|
|
36
|
+
pip-log.txt
|
|
37
|
+
pip-delete-this-directory.txt
|
|
38
|
+
|
|
39
|
+
# Unit test / coverage reports
|
|
40
|
+
htmlcov/
|
|
41
|
+
.tox/
|
|
42
|
+
.nox/
|
|
43
|
+
.coverage
|
|
44
|
+
.coverage.*
|
|
45
|
+
.cache
|
|
46
|
+
nosetests.xml
|
|
47
|
+
coverage.xml
|
|
48
|
+
*.cover
|
|
49
|
+
*.py.cover
|
|
50
|
+
*.lcov
|
|
51
|
+
.hypothesis/
|
|
52
|
+
.pytest_cache/
|
|
53
|
+
cover/
|
|
54
|
+
|
|
55
|
+
# Translations
|
|
56
|
+
*.mo
|
|
57
|
+
*.pot
|
|
58
|
+
|
|
59
|
+
# Django stuff:
|
|
60
|
+
*.log
|
|
61
|
+
local_settings.py
|
|
62
|
+
db.sqlite3
|
|
63
|
+
db.sqlite3-journal
|
|
64
|
+
|
|
65
|
+
# Flask stuff:
|
|
66
|
+
instance/
|
|
67
|
+
.webassets-cache
|
|
68
|
+
|
|
69
|
+
# Scrapy stuff:
|
|
70
|
+
.scrapy
|
|
71
|
+
|
|
72
|
+
# Sphinx documentation
|
|
73
|
+
docs/_build/
|
|
74
|
+
|
|
75
|
+
# PyBuilder
|
|
76
|
+
.pybuilder/
|
|
77
|
+
target/
|
|
78
|
+
|
|
79
|
+
# Jupyter Notebook
|
|
80
|
+
.ipynb_checkpoints
|
|
81
|
+
|
|
82
|
+
# IPython
|
|
83
|
+
profile_default/
|
|
84
|
+
ipython_config.py
|
|
85
|
+
|
|
86
|
+
# pyenv
|
|
87
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
88
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
89
|
+
# .python-version
|
|
90
|
+
|
|
91
|
+
# pipenv
|
|
92
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
93
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
94
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
95
|
+
# install all needed dependencies.
|
|
96
|
+
# Pipfile.lock
|
|
97
|
+
|
|
98
|
+
# uv
|
|
99
|
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
|
100
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
101
|
+
# commonly ignored for libraries.
|
|
102
|
+
# uv.lock
|
|
103
|
+
|
|
104
|
+
# poetry
|
|
105
|
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
106
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
107
|
+
# commonly ignored for libraries.
|
|
108
|
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
109
|
+
# poetry.lock
|
|
110
|
+
# poetry.toml
|
|
111
|
+
|
|
112
|
+
# pdm
|
|
113
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
114
|
+
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
|
115
|
+
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
|
116
|
+
# pdm.lock
|
|
117
|
+
# pdm.toml
|
|
118
|
+
.pdm-python
|
|
119
|
+
.pdm-build/
|
|
120
|
+
|
|
121
|
+
# pixi
|
|
122
|
+
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
|
123
|
+
# pixi.lock
|
|
124
|
+
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
|
125
|
+
# in the .venv directory. It is recommended not to include this directory in version control.
|
|
126
|
+
.pixi/*
|
|
127
|
+
!.pixi/config.toml
|
|
128
|
+
|
|
129
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
130
|
+
__pypackages__/
|
|
131
|
+
|
|
132
|
+
# Celery stuff
|
|
133
|
+
celerybeat-schedule*
|
|
134
|
+
celerybeat.pid
|
|
135
|
+
|
|
136
|
+
# Redis
|
|
137
|
+
*.rdb
|
|
138
|
+
*.aof
|
|
139
|
+
*.pid
|
|
140
|
+
|
|
141
|
+
# RabbitMQ
|
|
142
|
+
mnesia/
|
|
143
|
+
rabbitmq/
|
|
144
|
+
rabbitmq-data/
|
|
145
|
+
|
|
146
|
+
# ActiveMQ
|
|
147
|
+
activemq-data/
|
|
148
|
+
|
|
149
|
+
# SageMath parsed files
|
|
150
|
+
*.sage.py
|
|
151
|
+
|
|
152
|
+
# Environments
|
|
153
|
+
.env
|
|
154
|
+
.envrc
|
|
155
|
+
.venv
|
|
156
|
+
env/
|
|
157
|
+
venv/
|
|
158
|
+
ENV/
|
|
159
|
+
env.bak/
|
|
160
|
+
venv.bak/
|
|
161
|
+
|
|
162
|
+
# Spyder project settings
|
|
163
|
+
.spyderproject
|
|
164
|
+
.spyproject
|
|
165
|
+
|
|
166
|
+
# Rope project settings
|
|
167
|
+
.ropeproject
|
|
168
|
+
|
|
169
|
+
# mkdocs/Zensical documentation
|
|
170
|
+
/site
|
|
171
|
+
|
|
172
|
+
# mypy
|
|
173
|
+
.mypy_cache/
|
|
174
|
+
.dmypy.json
|
|
175
|
+
dmypy.json
|
|
176
|
+
|
|
177
|
+
# Pyre type checker
|
|
178
|
+
.pyre/
|
|
179
|
+
|
|
180
|
+
# pytype static type analyzer
|
|
181
|
+
.pytype/
|
|
182
|
+
|
|
183
|
+
# Cython debug symbols
|
|
184
|
+
cython_debug/
|
|
185
|
+
|
|
186
|
+
# PyCharm
|
|
187
|
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
188
|
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
189
|
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
190
|
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
191
|
+
# .idea/
|
|
192
|
+
|
|
193
|
+
# Abstra
|
|
194
|
+
# Abstra is an AI-powered process automation framework.
|
|
195
|
+
# Ignore directories containing user credentials, local state, and settings.
|
|
196
|
+
# Learn more at https://abstra.io/docs
|
|
197
|
+
.abstra/
|
|
198
|
+
|
|
199
|
+
# Visual Studio Code
|
|
200
|
+
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore that
|
|
201
|
+
# can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
|
202
|
+
# and can be added to the global gitignore or merged into this file. However, if you prefer, you
|
|
203
|
+
# could uncomment the following to ignore the entire vscode folder
|
|
204
|
+
# .vscode/
|
|
205
|
+
# Temporary file for partial code execution
|
|
206
|
+
tempCodeRunnerFile.py
|
|
207
|
+
|
|
208
|
+
# Ruff stuff:
|
|
209
|
+
.ruff_cache/
|
|
210
|
+
|
|
211
|
+
# PyPI configuration file
|
|
212
|
+
.pypirc
|
|
213
|
+
|
|
214
|
+
# Marimo
|
|
215
|
+
marimo/_static/
|
|
216
|
+
marimo/_lsp/
|
|
217
|
+
__marimo__/
|
|
218
|
+
|
|
219
|
+
# Streamlit
|
|
220
|
+
.streamlit/secrets.toml
|
|
221
|
+
|
|
222
|
+
# Bob Protocol / via — machine-local config and index, not portable
|
|
223
|
+
.mcp.json
|
|
224
|
+
.via/
|
|
225
|
+
|
|
226
|
+
# Downloaded example data (make gharchive); removed by make clean
|
|
227
|
+
data/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Drew Gutstein
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,456 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: sketch-tastic
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Mergeable sketches of JSONL event streams for regression testing and flow analysis.
|
|
5
|
+
Project-URL: Repository, https://github.com/drusifer/sketch-tastic
|
|
6
|
+
Author: drusifer
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Keywords: events,fingerprint,jsonl,jsonpath,logs,regression,sketch
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: System Administrators
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
17
|
+
Classifier: Topic :: System :: Logging
|
|
18
|
+
Classifier: Topic :: Utilities
|
|
19
|
+
Requires-Python: >=3.14
|
|
20
|
+
Requires-Dist: jsonpath-rfc9535>=2.0.2
|
|
21
|
+
Requires-Dist: jsonschema>=4.23
|
|
22
|
+
Requires-Dist: pyyaml>=6.0
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
|
|
25
|
+
# sketch-tastic
|
|
26
|
+
|
|
27
|
+
Mergeable sketches of JSONL event streams for regression testing and flow analysis.
|
|
28
|
+
|
|
29
|
+
`sketchtastic` reads events — logs, traces, audit records, anything with one JSON object per
|
|
30
|
+
line — and reduces each to a **skit**: a stable ID for "this shape of event". It keeps a
|
|
31
|
+
**sketch** (the data-sketch term for a compact, mergeable summary of a stream) with one entry per
|
|
32
|
+
skit: how often it was seen, statistics you ask for, and a sample. The sketch is a file that
|
|
33
|
+
grows across runs:
|
|
34
|
+
|
|
35
|
+
```
|
|
36
|
+
existing sketch + stream of events -> updated sketch
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
A million near-identical log lines become a handful of entries you can actually read.
|
|
40
|
+
|
|
41
|
+
## Install
|
|
42
|
+
|
|
43
|
+
```sh
|
|
44
|
+
pip install sketch-tastic
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Python 3.14 or newer. Installs the `sketchtastic` command and its short alias `skt` (also
|
|
48
|
+
runnable as `python -m sketchtastic`).
|
|
49
|
+
|
|
50
|
+
Moved from [drusifer/fingerling](https://github.com/drusifer/fingerling), where it was
|
|
51
|
+
`fingerprint-sampler`. While the version is 0.x, releases may break profiles and sketch files
|
|
52
|
+
without a migration path.
|
|
53
|
+
|
|
54
|
+
## Quickstart
|
|
55
|
+
|
|
56
|
+
Paste this into a shell. It writes a profile and a few events, then sketches them:
|
|
57
|
+
|
|
58
|
+
```sh
|
|
59
|
+
cat > profile.yaml <<'YAML'
|
|
60
|
+
version: 0
|
|
61
|
+
fields:
|
|
62
|
+
- {key: level, path: "$.level"}
|
|
63
|
+
- {key: component, path: "$.logger"}
|
|
64
|
+
- {key: failed, path: "$.error", modifier: IS_PRESENT}
|
|
65
|
+
aggregates:
|
|
66
|
+
- {key: took_ms, path: "$.took_ms", fn: [MIN, MAX, AVG]}
|
|
67
|
+
- {key: example, path: "$.msg", fn: {name: LAST, n: 2}}
|
|
68
|
+
YAML
|
|
69
|
+
|
|
70
|
+
cat > events.jsonl <<'JSONL'
|
|
71
|
+
{"level": "INFO", "logger": "db", "msg": "query ok", "took_ms": 3}
|
|
72
|
+
{"level": "INFO", "logger": "db", "msg": "query ok (cached)", "took_ms": 1}
|
|
73
|
+
{"level": "ERROR", "logger": "db", "msg": "query timed out", "error": "TimeoutError", "took_ms": 5000}
|
|
74
|
+
{"level": "INFO", "logger": "http", "msg": "GET /health", "took_ms": 0.4}
|
|
75
|
+
JSONL
|
|
76
|
+
|
|
77
|
+
sketchtastic merge -p profile.yaml -s app.sketch.yaml events.jsonl
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
```
|
|
81
|
+
sketchtastic: 4 events read from events.jsonl
|
|
82
|
+
skits: 3 new, 0 updated, 3 total in sketch
|
|
83
|
+
skipped: 0 malformed lines, 0 unusable aggregate values
|
|
84
|
+
wrote app.sketch.yaml (profile 3f0dd0d9f280ba32064a891dcd3b17cb)
|
|
85
|
+
history: app.sketch.history/261014-0001.yaml
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Four events, three skits: the two successful `db` queries share one. Run the same command
|
|
89
|
+
again and the counts add up instead of new entries appearing — that is the merge. Each run also
|
|
90
|
+
leaves a file with just its own skits in `app.sketch.history/` (see History below).
|
|
91
|
+
|
|
92
|
+
`app.sketch.yaml` now starts like this (the other two skits follow):
|
|
93
|
+
|
|
94
|
+
```yaml
|
|
95
|
+
version: 0
|
|
96
|
+
profile_id: 3f0dd0d9f280ba32064a891dcd3b17cb
|
|
97
|
+
last_run: 261014-0001
|
|
98
|
+
skits:
|
|
99
|
+
9312137ea290aa8d196e7036cd687af8:
|
|
100
|
+
tokens:
|
|
101
|
+
level: INFO
|
|
102
|
+
component: db
|
|
103
|
+
failed: 'FALSE'
|
|
104
|
+
aggregates:
|
|
105
|
+
took_ms:
|
|
106
|
+
MIN: 1
|
|
107
|
+
MAX: 3
|
|
108
|
+
AVG:
|
|
109
|
+
avg: 2.0
|
|
110
|
+
sum: 4
|
|
111
|
+
count: 2
|
|
112
|
+
example:
|
|
113
|
+
- query ok
|
|
114
|
+
- query ok (cached)
|
|
115
|
+
meta:
|
|
116
|
+
count: 2
|
|
117
|
+
first_seen: 261014-0001
|
|
118
|
+
last_seen: 261014-0001
|
|
119
|
+
first:
|
|
120
|
+
- meta: {run: 261014-0001}
|
|
121
|
+
record: {level: INFO, logger: db, msg: query ok, took_ms: 3}
|
|
122
|
+
last:
|
|
123
|
+
- meta: {run: 261014-0001}
|
|
124
|
+
record: {level: INFO, logger: db, msg: query ok (cached), took_ms: 1}
|
|
125
|
+
share:
|
|
126
|
+
meta:
|
|
127
|
+
count: 50.0
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
To read a sketch without the YAML, print it as a table with `sketchtastic show app.sketch.yaml`:
|
|
131
|
+
|
|
132
|
+
```
|
|
133
|
+
ID COUNT COUNT % FIRST RUN LAST RUN component failed level
|
|
134
|
+
d7d629a1 1 25.0 261014-0001 261014-0001 db TRUE ERROR
|
|
135
|
+
f5189272 1 25.0 261014-0001 261014-0001 http FALSE INFO
|
|
136
|
+
9312137e 2 50.0 261014-0001 261014-0001 db FALSE INFO
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
## Usage
|
|
140
|
+
|
|
141
|
+
```
|
|
142
|
+
sketchtastic merge -p PROFILE -s SKETCH [EVENTS|-] [-o OUTPUT] [-j N] [--force-profile]
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
| Option | Meaning |
|
|
146
|
+
|---|---|
|
|
147
|
+
| `-p`, `--profile PROFILE` | YAML profile (below). Required. |
|
|
148
|
+
| `-s`, `--sketch SKETCH` | Sketch to merge into. Created if missing, **updated in place**. Required. |
|
|
149
|
+
| `EVENTS` | JSONL file; `-` or omitted reads stdin. |
|
|
150
|
+
| `-o`, `--output OUTPUT` | Write the merged sketch here instead; `SKETCH` is left unchanged. |
|
|
151
|
+
| `-j`, `--jobs N` | Worker processes that fold the events (default: all cores). `-j 1`, or input of up to 5,000 lines, folds in-process. |
|
|
152
|
+
| `--force-profile` | Merge even though `SKETCH` was built with a different profile. |
|
|
153
|
+
|
|
154
|
+
```sh
|
|
155
|
+
# read from a pipe
|
|
156
|
+
tail -n 1000 app.jsonl | sketchtastic merge -p profile.yaml -s app.sketch.yaml -
|
|
157
|
+
|
|
158
|
+
# trial run: see the result without touching the sketch
|
|
159
|
+
sketchtastic merge -p profile.yaml -s app.sketch.yaml -o /tmp/preview.yaml new.jsonl
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
The summary goes to stderr and stdout stays empty. Exit status: `0` sketch written and every line
|
|
163
|
+
used · `1` sketch written but some lines were skipped (listed by line number) · `2` nothing written
|
|
164
|
+
· `3` sketch written but its history file was not (takes precedence over `1`).
|
|
165
|
+
|
|
166
|
+
**Parallel merge.** With more than one job, the input is read in batches of 5,000 lines that
|
|
167
|
+
worker processes fold into sketches of their own, merged at the end. Counts, sums, MIN/MAX, AVG,
|
|
168
|
+
first and last run, the sequence mark and the summary are exactly what one process gives;
|
|
169
|
+
`FIRST`/`LAST` samples (and the built-in first and last record) come from matching events, but
|
|
170
|
+
with several jobs not necessarily the very first or last ones, since any of them shows the same
|
|
171
|
+
shape. Skipped-line warnings still name the right lines, in line order.
|
|
172
|
+
|
|
173
|
+
### History
|
|
174
|
+
|
|
175
|
+
Each run gets a **run id**, `YYMMDD-SEQ`: the UTC date, then a number that only goes up for this
|
|
176
|
+
sketch (`261015-0003`). The sketch keeps the latest as `last_run`, and every skit's
|
|
177
|
+
`first_seen` / `last_seen` name the first and last run that saw it.
|
|
178
|
+
|
|
179
|
+
Every run that updates `SKETCH` also writes just that run's skits to a file named by its run
|
|
180
|
+
id in `<SKETCH without extension>.history/`:
|
|
181
|
+
|
|
182
|
+
```
|
|
183
|
+
web.sketch.yaml # last_run: 261015-0003
|
|
184
|
+
web.sketch.history/
|
|
185
|
+
261014-0001.yaml
|
|
186
|
+
261014-0002.yaml
|
|
187
|
+
261015-0003.yaml
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
So `last_seen: 261014-0002` points straight at the file for that run. A run file is a sketch
|
|
191
|
+
(counts, aggregates and shares from that run only), so `sketchtastic show` reads it. It starts
|
|
192
|
+
with a `run:` header: `id`, `started`, `finished` (UTC), `source`, `events`, `skipped_lines` and
|
|
193
|
+
`sketch`. A new run's number is one past the highest in the sketch and its history, so a rebuilt sketch
|
|
194
|
+
never overwrites an old run file. History is always on; a run with `-o`, or one that writes
|
|
195
|
+
nothing, leaves no run file and uses up no run id. Nothing is pruned.
|
|
196
|
+
|
|
197
|
+
### Duplicate protection
|
|
198
|
+
|
|
199
|
+
If your events carry a number that only grows (an event id, an offset), name it in the profile:
|
|
200
|
+
|
|
201
|
+
```yaml
|
|
202
|
+
sequence: $.id
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
The sketch then keeps the highest sequence merged so far (`sequence: {path: $.id, max: 34541893465}`)
|
|
206
|
+
and every later run drops events at or below it, so running twice over the same file, or over
|
|
207
|
+
files that overlap, adds nothing twice:
|
|
208
|
+
|
|
209
|
+
```
|
|
210
|
+
dropped: 180387 events already merged (sequence ≤ 34541893465)
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
Events are compared with the mark as it stood when the run started, so events slightly out of
|
|
214
|
+
order within one run are all kept (and duplicates within one run are not caught). Numbers and
|
|
215
|
+
numeric strings work. Duplicate detection has to be exact, so an event whose sequence is
|
|
216
|
+
missing, not a number, or selected more than once is an **error**: the run stops at that line,
|
|
217
|
+
nothing is written (no sketch update, no run file) and the exit status is `2`. The run file's header records the range
|
|
218
|
+
merged, `sequence: {min, max}`. Changing the profile's `sequence` path makes the next run ignore
|
|
219
|
+
the old mark and say so.
|
|
220
|
+
|
|
221
|
+
### Check a profile
|
|
222
|
+
|
|
223
|
+
```
|
|
224
|
+
sketchtastic check PROFILE [PROFILE ...]
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
Checks profiles without reading any events and lists **every** problem, one per line, with the
|
|
228
|
+
file and where in it:
|
|
229
|
+
|
|
230
|
+
```
|
|
231
|
+
app.yaml: fields[0] (key 'route') → modifier: 'replace' is a required property
|
|
232
|
+
app.yaml: fields[1] (key 'method') → modifier.flags: 'q' does not match '^[imsx]*$'
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
A valid profile prints `app.yaml: OK`. Exit status: `0` all valid · `1` problems found. The
|
|
236
|
+
structure is checked against a JSON Schema (`profile.schema.json`, shipped in the package) by the
|
|
237
|
+
standard [`jsonschema`](https://python-jsonschema.readthedocs.io/) validator. Regexes, JSONPaths,
|
|
238
|
+
`IN_RANGE` bounds, duplicate keys and aggregate functions are checked as well. `merge` runs the
|
|
239
|
+
same check first and writes nothing if it fails.
|
|
240
|
+
|
|
241
|
+
**In your editor.** Put this line at the top of a profile, and an editor with YAML language-server
|
|
242
|
+
support (for example VS Code with the Red Hat YAML extension) checks the profile as you type and
|
|
243
|
+
completes modifier names:
|
|
244
|
+
|
|
245
|
+
```yaml
|
|
246
|
+
# yaml-language-server: $schema=https://raw.githubusercontent.com/drusifer/sketch-tastic/main/src/sketchtastic/profile.schema.json
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
### Show a sketch
|
|
250
|
+
|
|
251
|
+
```
|
|
252
|
+
sketchtastic show [--sort KEY] SKETCH
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
Prints one row per skit, least frequent first: the first 8 characters of its ID, its count
|
|
256
|
+
and `COUNT %` (its share of all events), the first and last run that saw it, then a value and
|
|
257
|
+
a `%` column for each `SUM` or `COUNT` aggregate (`payload_bytes.SUM`, `payload_bytes.SUM %`), and
|
|
258
|
+
one column per token. Shares come from the sketch file's `share` section; a sketch written before
|
|
259
|
+
shares existed shows `-` until its next merge. A sketch with no skits prints
|
|
260
|
+
`SKETCH: no skits`. Long values are not cut; pipe to `less -S` to scroll sideways.
|
|
261
|
+
|
|
262
|
+
`--sort KEY` orders the rows by the column headed KEY, in any case, lowest first (default
|
|
263
|
+
`COUNT`), so the largest are at the end: `sketchtastic show --sort 'payload_bytes.SUM %'
|
|
264
|
+
gh.sketch.yaml | tail` shows the heaviest skits. `-` sorts first, numbers before text, and
|
|
265
|
+
equal values keep file order. Exit status: `0` printed · `2` the sketch file could not be read,
|
|
266
|
+
or KEY is not one of its columns (the message says why, and lists the columns).
|
|
267
|
+
|
|
268
|
+
## Profiles
|
|
269
|
+
|
|
270
|
+
```yaml
|
|
271
|
+
version: 0
|
|
272
|
+
|
|
273
|
+
# fields define identity: events with the same tokens share one skit
|
|
274
|
+
fields:
|
|
275
|
+
- key: method
|
|
276
|
+
path: "$.request.method" # modifier defaults to DEFAULT
|
|
277
|
+
- key: status_ok
|
|
278
|
+
path: "$.response.status"
|
|
279
|
+
modifier: {name: IN_RANGE, low: 200, high: 399}
|
|
280
|
+
- key: slow_upstreams
|
|
281
|
+
path: "$.upstreams[?(@.ms > 500)].name" # full JSONPath, including filters
|
|
282
|
+
|
|
283
|
+
# aggregates are tracked per skit and never affect identity
|
|
284
|
+
aggregates:
|
|
285
|
+
- key: latency_ms
|
|
286
|
+
path: "$.response.ms"
|
|
287
|
+
fn: [MIN, MAX, AVG]
|
|
288
|
+
- key: recent_paths
|
|
289
|
+
path: "$.request.path"
|
|
290
|
+
fn: {name: LAST, n: 3}
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
**Paths** are standard JSONPath ([RFC 9535](https://www.rfc-editor.org/rfc/rfc9535), evaluated by
|
|
294
|
+
[jsonpath-rfc9535](https://github.com/jg-rp/python-jsonpath-rfc9535)), with no extensions: dot and bracket access, `[*]`, slices, negative indexes, unions,
|
|
295
|
+
recursive descent (`..`) and filters. If a path selects several values, each is tokenized and the
|
|
296
|
+
results are de-duplicated, sorted and joined, so array order never changes a skit.
|
|
297
|
+
|
|
298
|
+
**Modifiers** turn the selected value into a token. Pick ones that collapse high-cardinality
|
|
299
|
+
values (IDs, timestamps, sizes) so similar events share a skit:
|
|
300
|
+
|
|
301
|
+
| Modifier | Arguments | Token |
|
|
302
|
+
|---|---|---|
|
|
303
|
+
| `DEFAULT` | — | the value as a string; `NULL` if absent |
|
|
304
|
+
| `IS_PRESENT` | — | `TRUE` if the key exists, even when its value is `null` |
|
|
305
|
+
| `IS_NULL` | — | `TRUE` only if the key exists and is set to `null` (a missing key is `FALSE`) |
|
|
306
|
+
| `IN_RANGE` | `low`, `high` (numbers) | `TRUE` if `low <= value <= high` |
|
|
307
|
+
| `ANY` | `values` (a list) | `TRUE` if the value is one of those listed |
|
|
308
|
+
| `GLOB` | `pattern` | `TRUE` if the whole value matches the shell-style pattern (`*`, `?`, `[a-z]`), case-sensitive |
|
|
309
|
+
| `REGEX` | `pattern`, optional `flags` | `TRUE` if the regular expression matches anywhere in the value; anchor with `^` and `$` |
|
|
310
|
+
| `REGSUB` | `pattern`, `replace`, optional `flags` | the value with every match replaced (`\1` refers to a group); `NULL` if absent |
|
|
311
|
+
| `TRUNC` | `unit` | a `DATE`/`DATETIME` value cut to `year` … `second`, e.g. hour → `2024-01-01T15` |
|
|
312
|
+
|
|
313
|
+
A modifier without arguments is written as its bare name (`modifier: IS_PRESENT`); one with
|
|
314
|
+
arguments is a mapping with `name` and its arguments:
|
|
315
|
+
|
|
316
|
+
```yaml
|
|
317
|
+
- key: status_ok
|
|
318
|
+
path: $.response.status
|
|
319
|
+
modifier: {name: IN_RANGE, low: 200, high: 399}
|
|
320
|
+
- key: port_ok
|
|
321
|
+
path: $.dport
|
|
322
|
+
modifier: {name: ANY, values: [80, 443]}
|
|
323
|
+
- key: in_tmp
|
|
324
|
+
path: $.file
|
|
325
|
+
modifier: {name: GLOB, pattern: '/tmp/*'} # TRUE / FALSE
|
|
326
|
+
- key: route
|
|
327
|
+
path: $.request.path
|
|
328
|
+
modifier: {name: REGSUB, pattern: '\d+', replace: N} # /items/17 -> /items/N
|
|
329
|
+
- key: greeting
|
|
330
|
+
path: $.body.text
|
|
331
|
+
modifier:
|
|
332
|
+
name: REGEX
|
|
333
|
+
pattern: '^hello\s.*\sgoodbye\.$'
|
|
334
|
+
flags: i
|
|
335
|
+
```
|
|
336
|
+
|
|
337
|
+
`flags` is any of `i` (ignore case), `m` (`^` and `$` match at line breaks), `s` (`.` matches a
|
|
338
|
+
newline) and `x` (verbose). Patterns use Python's `re` syntax. Write patterns in single quotes:
|
|
339
|
+
YAML leaves backslashes alone there, and a pattern starting with `*`, `[` or `{`, or containing
|
|
340
|
+
`: ` or ` #`, would otherwise not parse as you meant. Changing a field's modifier changes the
|
|
341
|
+
profile's identity, like any other change to `fields`.
|
|
342
|
+
|
|
343
|
+
**Dates and times.** Add `type: DATETIME` (or `type: DATE`) to a field or aggregate whose path
|
|
344
|
+
holds ISO-8601 text. Values are compared as real times in UTC, so `MIN`/`MAX` are right across
|
|
345
|
+
time zones, and the sketch stores them as YAML timestamps. A value that isn't valid ISO-8601 counts as
|
|
346
|
+
missing (and is skipped by aggregates). For fields, `TRUNC` buckets a time so events group
|
|
347
|
+
together:
|
|
348
|
+
|
|
349
|
+
```yaml
|
|
350
|
+
- key: hour
|
|
351
|
+
path: $.created_at
|
|
352
|
+
type: DATETIME
|
|
353
|
+
modifier: {name: TRUNC, unit: hour} # year, month, day, hour, minute or second
|
|
354
|
+
```
|
|
355
|
+
|
|
356
|
+
**Aggregates** are computed across every event that lands on the same skit, over all runs:
|
|
357
|
+
|
|
358
|
+
| Function | Keeps |
|
|
359
|
+
|---|---|
|
|
360
|
+
| `SUM` | running sum of numbers |
|
|
361
|
+
| `MIN`, `MAX` | smallest / largest — numbers, dates and times (with `type`), or text |
|
|
362
|
+
| `AVG` | the average, with the sum and count that make it mergeable |
|
|
363
|
+
| `COUNT` | how many values the path selected (of any type) |
|
|
364
|
+
| `{name: FIRST, n: N}` | the `N` earliest values of the path (with `-j` above 1: some early values) |
|
|
365
|
+
| `{name: LAST, n: N}` | the `N` most recent values of the path (with `-j` above 1: some late values) |
|
|
366
|
+
|
|
367
|
+
`fn` is one function or a list, written like modifiers: a bare name, or a mapping for `FIRST`
|
|
368
|
+
and `LAST`. With one function the sketch stores its value directly (`count: 12`); with several, a
|
|
369
|
+
map by function name (`{MIN: 1, MAX: 3}`). Every skit holds every aggregate, in the order the
|
|
370
|
+
profile lists them (functions too), from its first event on: before any value, `SUM` and `COUNT`
|
|
371
|
+
are `0`, `FIRST`/`LAST` are `[]`, `AVG` is `{avg: null, sum: 0, count: 0}`, and `MIN`/`MAX` are
|
|
372
|
+
`null` (`-` in `show`). So the profile alone decides the order of keys in the file and of
|
|
373
|
+
columns in `show`. Numbers and numeric strings count as numbers. A value
|
|
374
|
+
a function can't use (text for `SUM`, text after numbers for `MIN`) is skipped and counted in the
|
|
375
|
+
summary as an unusable aggregate value. A record where the path matches nothing is simply not
|
|
376
|
+
aggregated and is not counted.
|
|
377
|
+
|
|
378
|
+
**Weighted aggregates.** Add `op: SIZE` to aggregate each value's size in bytes instead of the
|
|
379
|
+
value itself: UTF-8 bytes for text, the compact JSON encoding for anything else (`null` is 4).
|
|
380
|
+
Every function gets the size, so one entry gives `SUM(SIZE(payload))` and more:
|
|
381
|
+
|
|
382
|
+
```yaml
|
|
383
|
+
aggregates:
|
|
384
|
+
- {key: payload_bytes, path: $.payload, op: SIZE, fn: [SUM, AVG, MAX]}
|
|
385
|
+
```
|
|
386
|
+
|
|
387
|
+
`op` and `type` cannot be combined on one aggregate. Like any aggregate change, adding an `op`
|
|
388
|
+
keeps the profile id, so existing sketches keep merging.
|
|
389
|
+
|
|
390
|
+
`merge` checks the profile before reading any event; run `sketchtastic check` (above) to see
|
|
391
|
+
every problem at once.
|
|
392
|
+
|
|
393
|
+
## The sketch
|
|
394
|
+
|
|
395
|
+
- `tokens` shows what each ID stands for.
|
|
396
|
+
- `aggregates` holds what your profile asked for. `meta` holds what every skit gets for
|
|
397
|
+
free: `count`, `first_seen` / `last_seen` (the first and last run id that saw it, see History
|
|
398
|
+
above), and the first and last record (one each, to keep
|
|
399
|
+
sketches and run files small; add `FIRST`/`LAST` aggregates to your profile for more samples).
|
|
400
|
+
- `meta` comes from a small built-in profile shipped with the package (`builtin.yaml`), which
|
|
401
|
+
uses the same aggregate functions you can. It reads each record wrapped in an envelope,
|
|
402
|
+
`{meta: {run: <run id>}, record: <the parsed record>}`, so the fields the tool adds can never
|
|
403
|
+
clash with the record's own, and the stored first/last records carry the run they came from.
|
|
404
|
+
- `share` gives each skit's percentage of the whole sketch for every value that adds up:
|
|
405
|
+
`share.meta.count` is its share of all events, and `share.aggregates` holds the share of each
|
|
406
|
+
`SUM` and `COUNT`, laid out like `aggregates` (`payload_bytes: {SUM: 31.25}`, or `requests: 25.0`
|
|
407
|
+
for a single-function aggregate). It is recomputed on every write; a value whose total is 0 has
|
|
408
|
+
no share.
|
|
409
|
+
- The tool records runs, not times, per skit; a run's start and finish times are in its run
|
|
410
|
+
file. To track a time field inside your records, add it to your profile's `aggregates`.
|
|
411
|
+
- A sketch records the profile that built it (`profile_id`, derived from `fields`). Merging under a
|
|
412
|
+
profile with different `fields` is refused, because the skits would not be comparable.
|
|
413
|
+
Changing `aggregates` is always fine.
|
|
414
|
+
- Splitting a stream across several runs gives the same sketch as one run over all of it.
|
|
415
|
+
- The file is replaced atomically; a failed or interrupted run leaves the previous sketch intact.
|
|
416
|
+
|
|
417
|
+
## As a library
|
|
418
|
+
|
|
419
|
+
```python
|
|
420
|
+
from sketchtastic import Sketch, load_profile
|
|
421
|
+
|
|
422
|
+
profile = load_profile("profile.yaml")
|
|
423
|
+
sketch = Sketch(profile.aggregates, profile.profile_id)
|
|
424
|
+
|
|
425
|
+
event = {"level": "INFO", "logger": "db", "msg": "query ok", "took_ms": 3}
|
|
426
|
+
skit_id = sketch.upsert(profile.tokens(event), event, run="261015-0001")
|
|
427
|
+
|
|
428
|
+
sketch.get(skit_id).meta["count"] # 1
|
|
429
|
+
sketch.save("app.sketch.yaml")
|
|
430
|
+
```
|
|
431
|
+
|
|
432
|
+
`load_profile` and `parse_profile` raise `ProfileError`; its `problems` attribute lists every
|
|
433
|
+
problem, located, in the same form `sketchtastic check` prints. `check_profile(data, source)`
|
|
434
|
+
returns `(profile_or_None, problems)` without raising.
|
|
435
|
+
|
|
436
|
+
Without a profile file, build the pieces directly. `Field` and `Aggregate` take the same keys
|
|
437
|
+
as the YAML (`key`, `path`, `modifier` / `fn`) and are checked when created (`ValueError` if
|
|
438
|
+
wrong):
|
|
439
|
+
|
|
440
|
+
```python
|
|
441
|
+
from sketchtastic import Field, Profile
|
|
442
|
+
|
|
443
|
+
profile = Profile(fields=(
|
|
444
|
+
Field("method", "$.request.method"),
|
|
445
|
+
Field("route", "$.request.path", {"name": "REGSUB", "pattern": r"\d+", "replace": "N"}),
|
|
446
|
+
))
|
|
447
|
+
profile.tokens({"request": {"method": "GET", "path": "/items/17"}})
|
|
448
|
+
# {'method': 'GET', 'route': '/items/N'}
|
|
449
|
+
```
|
|
450
|
+
|
|
451
|
+
Every modifier and aggregate function is a `Rule` subclass, registered by its `name`; the
|
|
452
|
+
schema checks its arguments and the class checks the rest.
|
|
453
|
+
|
|
454
|
+
## License
|
|
455
|
+
|
|
456
|
+
MIT
|