logit-classifier 0.3.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/FINDINGS.md +26 -1
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/PKG-INFO +32 -22
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/README.md +31 -21
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/_version.py +1 -1
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/toolkit/comfyui/__init__.py +12 -12
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/toolkit/comfyui/scopes.py +1 -1
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/toolkit/comfyui/tagger.py +71 -66
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/toolkit/tags.py +54 -37
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests/test_toolkit.py +143 -111
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/.gitignore +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/LICENSE +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/examples/compare_models.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/examples/http_client.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/examples/images.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/examples/own_backend.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/examples/question_types.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/examples/quickstart.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/pyproject.toml +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/__init__.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/__main__.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/backends/__init__.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/backends/_torch_window.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/backends/base.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/backends/comfy_clip.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/backends/hf.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/calibrate.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/classifier.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/cli.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/config.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/deps.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/errors.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/labels.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/prompt.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/py.typed +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/schema.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/scoring.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/service.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/tags.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/toolkit/__init__.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/toolkit/comfyui/classifier.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/toolkit/comfyui/generate.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/toolkit/comfyui/progress.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/vision.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/web/index.html +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests/fixtures/banking77_test.json +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests/fixtures/eval_set.json +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests/fixtures/many_options_request.json +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests/fixtures/quickstart_request.json +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests/test_model.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests/test_unit.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests-AB/ab_branch_packing.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests-AB/ab_determinism_scope.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests-AB/ab_env.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests-AB/ab_math_sdp_reduction.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests-AB/ab_multi_label.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests-AB/ab_noul_wording.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests-AB/ab_temperature.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests-AB/banking77.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests-AB/benchmark.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests-AB/comfy_env.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests-AB/evaluate.py +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests-AB/inputs/_full.png +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests-AB/inputs/_sheet.png +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests-AB/inputs/low-L.png +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests-AB/inputs/low-R.png +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests-AB/inputs/mid-L.png +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests-AB/inputs/mid-R.png +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests-AB/inputs/top-L.png +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests-AB/inputs/top-R.png +0 -0
- {logit_classifier-0.3.0 → logit_classifier-0.4.0}/tests-AB/tune_groups.py +0 -0
|
@@ -295,6 +295,31 @@ dispatcher reaches the math backend, which builds the full attention matrix. Mea
|
|
|
295
295
|
`HFBackend` probes the backends at startup, at the model's own dtype, and pins the working
|
|
296
296
|
ones. `benchmark.py`.
|
|
297
297
|
|
|
298
|
+
## One presence wording, two threshold pairs
|
|
299
|
+
|
|
300
|
+
Correct tags per tile at a pooled precision, over 905 judged tile candidates on 23 tiles,
|
|
301
|
+
from the Logit Tagger's `tests-AB/ab_presence_wording.py` (round 4).
|
|
302
|
+
|
|
303
|
+
| Wording | P 0.95 | P 0.97 | P 0.98 | P 0.99 |
|
|
304
|
+
|---|---|---|---|---|
|
|
305
|
+
| "This image visibly contains {TAG}" | 17.43 | 12.78 | 9.87 | 6.04 |
|
|
306
|
+
| "Is there a {TAG} in this image?" (0.3.0, article by rule) | 17.35 | 11.30 | 2.39 | 2.09 |
|
|
307
|
+
| "Present in this image: {TAG}" | 17.61 | 8.74 | 8.13 | 6.74 |
|
|
308
|
+
|
|
309
|
+
The shipped fp8 tile pass cut offline, per tile and prompt over 69 runs, from
|
|
310
|
+
ContextAnchoredTileRefine's `tests-AB/ab_tags_paired.py`.
|
|
311
|
+
|
|
312
|
+
| Thresholds | Correct | Wrong or vague | Precision |
|
|
313
|
+
|---|---|---|---|
|
|
314
|
+
| broad (0.999, 0.9999) | 12.32 | 0.54 | 0.958 |
|
|
315
|
+
| strict (0.99998, 0.99998) | 7.33 | 0.17 | 0.977 |
|
|
316
|
+
|
|
317
|
+
Every paired interval against "This image visibly contains {TAG}" at 0.95 and 0.98 spans zero.
|
|
318
|
+
It needs no grammar, and the tile pass already asked it, so its thresholds carry over. On
|
|
319
|
+
declared COCO tag lists "Is there a {TAG} in this image?" (0.3.0, article by rule) kept more
|
|
320
|
+
correct tags at precision 0.98 (3.79 against 1.90 per image), which is the cost of one
|
|
321
|
+
wording.
|
|
322
|
+
|
|
298
323
|
## Bugs found and fixed
|
|
299
324
|
|
|
300
325
|
| Bug | Effect | Fix |
|
|
@@ -329,6 +354,6 @@ and 23.66 GB against 15.12 GB, because a cropped cache is a non-contiguous view.
|
|
|
329
354
|
|
|
330
355
|
## Layout
|
|
331
356
|
|
|
332
|
-
`tests/` holds the pytest suite,
|
|
357
|
+
`tests/` holds the pytest suite, the unit tests and 14 tests that need the GPU. `tests-AB/` holds
|
|
333
358
|
the measurement harnesses, matching ComfyUI-ContextAnchoredTileRefine. Fixtures live in
|
|
334
359
|
`tests/fixtures/` and `tests-AB/ab_env.py` points at them.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: logit-classifier
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Local zero-shot classifier for text and images. Declares options and reads a calibrated probability for each from the logits. Accepts the TypeSafe System One request format, and ships a toolkit of image tag tools and ComfyUI node tools.
|
|
5
5
|
Project-URL: Repository, https://github.com/Blakeem/logit-classifier
|
|
6
6
|
Project-URL: Bug Tracker, https://github.com/Blakeem/logit-classifier/issues
|
|
@@ -178,7 +178,7 @@ the model.
|
|
|
178
178
|
|
|
179
179
|
```json
|
|
180
180
|
{
|
|
181
|
-
"model": "logit-classifier-0.
|
|
181
|
+
"model": "logit-classifier-0.4.0",
|
|
182
182
|
"answers": {
|
|
183
183
|
"department": {
|
|
184
184
|
"type": "choice",
|
|
@@ -371,7 +371,8 @@ and the classifier never imports the toolkit.
|
|
|
371
371
|
| `drop_subsets` | drops a tag whose words all appear in a longer tag |
|
|
372
372
|
| `prompt_word_forms`, `in_prompt` | test whether a tag's words appear in a prompt, singular or plural |
|
|
373
373
|
| `merge_candidates` | merges proposed tags with a prompt's tags under a cap |
|
|
374
|
-
| `
|
|
374
|
+
| `presence_statement` | writes "This image visibly contains car" with the tag as given |
|
|
375
|
+
| `BROAD_THRESHOLDS`, `STRICT_THRESHOLDS` | the scores a tag must reach, broad for more tags and strict for fewer wrong ones |
|
|
375
376
|
|
|
376
377
|
```python
|
|
377
378
|
from logit_classifier.toolkit.tags import drop_subsets, parse_candidates
|
|
@@ -387,35 +388,44 @@ stays beside "cat ear", and "signage" is dropped beside "signage in chinese".
|
|
|
387
388
|
### ComfyUI Tools
|
|
388
389
|
|
|
389
390
|
These run inside a ComfyUI node over the CLIP the workflow loaded. Each one imports comfy and
|
|
390
|
-
torch only when it is called.
|
|
391
|
+
torch only when it is called. `logit_classifier.toolkit.comfyui` exports every tool. The table
|
|
392
|
+
names the module of each tool it lists.
|
|
391
393
|
|
|
392
|
-
| Tool | Does |
|
|
393
|
-
|
|
394
|
-
| `comfy_classifier` | builds a classifier over the CLIP, with errors that name the node and the Load CLIP fix |
|
|
395
|
-
| `generate_text` | generates one greedy reply with the Qwen chat template, and stops early on a condition |
|
|
396
|
-
| `
|
|
397
|
-
| `
|
|
398
|
-
| `
|
|
399
|
-
| `
|
|
400
|
-
| `
|
|
401
|
-
| `
|
|
402
|
-
| `
|
|
394
|
+
| Module | Tool | Does |
|
|
395
|
+
|---|---|---|
|
|
396
|
+
| `classifier` | `comfy_classifier` | builds a classifier over the CLIP, with errors that name the node and the Load CLIP fix |
|
|
397
|
+
| `generate` | `generate_text` | generates one greedy reply with the Qwen chat template, and stops early on a condition |
|
|
398
|
+
| `generate` | `clip_generate` | runs core's generate, then clears the CUDA graphs it captured |
|
|
399
|
+
| `scopes` | `shared_vision_encode` | runs the vision tower once per image for every generate and classify inside the block |
|
|
400
|
+
| `scopes` | `skip_resident_loads` | skips core's model load while the CLIP is already on the GPU |
|
|
401
|
+
| `progress` | `routed_progress_bars`, `send_status`, `run_outcome` | show one progress bar per run and a status line under the node |
|
|
402
|
+
| `tagger` | `fit_image` | downscales an image to a pixel cap |
|
|
403
|
+
| `tagger` | `tag_prompt` | lists the physical things a prompt names |
|
|
404
|
+
| `tagger` | `tag_image` | tags an image, with a prompt's things added as candidates |
|
|
405
|
+
| `tagger` | `TagSettings` | holds every wording, budget, cap and threshold of the tagger |
|
|
403
406
|
|
|
404
407
|
The tagger generates candidate tags over the CLIP and verifies each one with the classifier.
|
|
405
408
|
|
|
406
409
|
```python
|
|
407
|
-
from logit_classifier.toolkit.comfyui import TagSettings, comfy_classifier,
|
|
410
|
+
from logit_classifier.toolkit.comfyui import TagSettings, comfy_classifier, fit_image, tag_image
|
|
408
411
|
|
|
409
412
|
classifier = comfy_classifier(clip, node="My Tagger")
|
|
410
|
-
|
|
411
|
-
trace = tag_picture(clip, classifier, picture, settings=TagSettings(), known={})
|
|
413
|
+
trace = tag_image(clip, classifier, fit_image(image, 1024 * 1024), settings=TagSettings(), thing_cache={})
|
|
412
414
|
tags = trace.kept
|
|
413
415
|
```
|
|
414
416
|
|
|
415
|
-
`
|
|
417
|
+
`thing_cache` holds the thing check's answers across a run, so each tag is asked once. The
|
|
416
418
|
`TagSettings` defaults are the current measured values, and a later release may change them.
|
|
417
419
|
Pass every value your output depends on to keep it fixed.
|
|
418
420
|
|
|
421
|
+
For fewer wrong tags, pass `STRICT_THRESHOLDS` as `verify_thresholds`.
|
|
422
|
+
|
|
423
|
+
```python
|
|
424
|
+
from logit_classifier.toolkit.tags import STRICT_THRESHOLDS
|
|
425
|
+
|
|
426
|
+
settings = TagSettings(verify_thresholds=STRICT_THRESHOLDS)
|
|
427
|
+
```
|
|
428
|
+
|
|
419
429
|
## Differences From Jev
|
|
420
430
|
|
|
421
431
|
This project matches the System One request and response format and the documented limits.
|
|
@@ -434,13 +444,13 @@ Jev publishes status codes but no error body. The error shape here is our own.
|
|
|
434
444
|
|
|
435
445
|
```
|
|
436
446
|
uv sync --all-extras
|
|
437
|
-
uv run pytest tests
|
|
438
|
-
uv run pytest tests/test_model.py -m model
|
|
447
|
+
uv run python -m pytest tests -m "not model"
|
|
448
|
+
uv run python -m pytest tests/test_model.py -m model
|
|
439
449
|
uv run ruff check .
|
|
440
450
|
uv run mypy
|
|
441
451
|
```
|
|
442
452
|
|
|
443
|
-
The unit tests need no weights. The model tests load the GPU and take
|
|
453
|
+
The unit tests need no weights. The model tests load the GPU and take 24 to 126 seconds, the longer run being the first of a session.
|
|
444
454
|
|
|
445
455
|
`FINDINGS.md` is the measurement record. Every number there names the script in
|
|
446
456
|
`tests-AB/` that produced it.
|
|
@@ -145,7 +145,7 @@ the model.
|
|
|
145
145
|
|
|
146
146
|
```json
|
|
147
147
|
{
|
|
148
|
-
"model": "logit-classifier-0.
|
|
148
|
+
"model": "logit-classifier-0.4.0",
|
|
149
149
|
"answers": {
|
|
150
150
|
"department": {
|
|
151
151
|
"type": "choice",
|
|
@@ -338,7 +338,8 @@ and the classifier never imports the toolkit.
|
|
|
338
338
|
| `drop_subsets` | drops a tag whose words all appear in a longer tag |
|
|
339
339
|
| `prompt_word_forms`, `in_prompt` | test whether a tag's words appear in a prompt, singular or plural |
|
|
340
340
|
| `merge_candidates` | merges proposed tags with a prompt's tags under a cap |
|
|
341
|
-
| `
|
|
341
|
+
| `presence_statement` | writes "This image visibly contains car" with the tag as given |
|
|
342
|
+
| `BROAD_THRESHOLDS`, `STRICT_THRESHOLDS` | the scores a tag must reach, broad for more tags and strict for fewer wrong ones |
|
|
342
343
|
|
|
343
344
|
```python
|
|
344
345
|
from logit_classifier.toolkit.tags import drop_subsets, parse_candidates
|
|
@@ -354,35 +355,44 @@ stays beside "cat ear", and "signage" is dropped beside "signage in chinese".
|
|
|
354
355
|
### ComfyUI Tools
|
|
355
356
|
|
|
356
357
|
These run inside a ComfyUI node over the CLIP the workflow loaded. Each one imports comfy and
|
|
357
|
-
torch only when it is called.
|
|
358
|
+
torch only when it is called. `logit_classifier.toolkit.comfyui` exports every tool. The table
|
|
359
|
+
names the module of each tool it lists.
|
|
358
360
|
|
|
359
|
-
| Tool | Does |
|
|
360
|
-
|
|
361
|
-
| `comfy_classifier` | builds a classifier over the CLIP, with errors that name the node and the Load CLIP fix |
|
|
362
|
-
| `generate_text` | generates one greedy reply with the Qwen chat template, and stops early on a condition |
|
|
363
|
-
| `
|
|
364
|
-
| `
|
|
365
|
-
| `
|
|
366
|
-
| `
|
|
367
|
-
| `
|
|
368
|
-
| `
|
|
369
|
-
| `
|
|
361
|
+
| Module | Tool | Does |
|
|
362
|
+
|---|---|---|
|
|
363
|
+
| `classifier` | `comfy_classifier` | builds a classifier over the CLIP, with errors that name the node and the Load CLIP fix |
|
|
364
|
+
| `generate` | `generate_text` | generates one greedy reply with the Qwen chat template, and stops early on a condition |
|
|
365
|
+
| `generate` | `clip_generate` | runs core's generate, then clears the CUDA graphs it captured |
|
|
366
|
+
| `scopes` | `shared_vision_encode` | runs the vision tower once per image for every generate and classify inside the block |
|
|
367
|
+
| `scopes` | `skip_resident_loads` | skips core's model load while the CLIP is already on the GPU |
|
|
368
|
+
| `progress` | `routed_progress_bars`, `send_status`, `run_outcome` | show one progress bar per run and a status line under the node |
|
|
369
|
+
| `tagger` | `fit_image` | downscales an image to a pixel cap |
|
|
370
|
+
| `tagger` | `tag_prompt` | lists the physical things a prompt names |
|
|
371
|
+
| `tagger` | `tag_image` | tags an image, with a prompt's things added as candidates |
|
|
372
|
+
| `tagger` | `TagSettings` | holds every wording, budget, cap and threshold of the tagger |
|
|
370
373
|
|
|
371
374
|
The tagger generates candidate tags over the CLIP and verifies each one with the classifier.
|
|
372
375
|
|
|
373
376
|
```python
|
|
374
|
-
from logit_classifier.toolkit.comfyui import TagSettings, comfy_classifier,
|
|
377
|
+
from logit_classifier.toolkit.comfyui import TagSettings, comfy_classifier, fit_image, tag_image
|
|
375
378
|
|
|
376
379
|
classifier = comfy_classifier(clip, node="My Tagger")
|
|
377
|
-
|
|
378
|
-
trace = tag_picture(clip, classifier, picture, settings=TagSettings(), known={})
|
|
380
|
+
trace = tag_image(clip, classifier, fit_image(image, 1024 * 1024), settings=TagSettings(), thing_cache={})
|
|
379
381
|
tags = trace.kept
|
|
380
382
|
```
|
|
381
383
|
|
|
382
|
-
`
|
|
384
|
+
`thing_cache` holds the thing check's answers across a run, so each tag is asked once. The
|
|
383
385
|
`TagSettings` defaults are the current measured values, and a later release may change them.
|
|
384
386
|
Pass every value your output depends on to keep it fixed.
|
|
385
387
|
|
|
388
|
+
For fewer wrong tags, pass `STRICT_THRESHOLDS` as `verify_thresholds`.
|
|
389
|
+
|
|
390
|
+
```python
|
|
391
|
+
from logit_classifier.toolkit.tags import STRICT_THRESHOLDS
|
|
392
|
+
|
|
393
|
+
settings = TagSettings(verify_thresholds=STRICT_THRESHOLDS)
|
|
394
|
+
```
|
|
395
|
+
|
|
386
396
|
## Differences From Jev
|
|
387
397
|
|
|
388
398
|
This project matches the System One request and response format and the documented limits.
|
|
@@ -401,13 +411,13 @@ Jev publishes status codes but no error body. The error shape here is our own.
|
|
|
401
411
|
|
|
402
412
|
```
|
|
403
413
|
uv sync --all-extras
|
|
404
|
-
uv run pytest tests
|
|
405
|
-
uv run pytest tests/test_model.py -m model
|
|
414
|
+
uv run python -m pytest tests -m "not model"
|
|
415
|
+
uv run python -m pytest tests/test_model.py -m model
|
|
406
416
|
uv run ruff check .
|
|
407
417
|
uv run mypy
|
|
408
418
|
```
|
|
409
419
|
|
|
410
|
-
The unit tests need no weights. The model tests load the GPU and take
|
|
420
|
+
The unit tests need no weights. The model tests load the GPU and take 24 to 126 seconds, the longer run being the first of a session.
|
|
411
421
|
|
|
412
422
|
`FINDINGS.md` is the measurement record. Every number there names the script in
|
|
413
423
|
`tests-AB/` that produced it.
|
{logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/toolkit/comfyui/__init__.py
RENAMED
|
@@ -13,14 +13,14 @@ from .generate import IMAGE_SLOT, chat, clip_generate, generate_text, stop_when
|
|
|
13
13
|
from .progress import routed_progress_bars, run_outcome, send_status, token_fill
|
|
14
14
|
from .scopes import shared_vision_encode, skip_resident_loads
|
|
15
15
|
from .tagger import (
|
|
16
|
-
|
|
16
|
+
ImageTagTrace,
|
|
17
|
+
PromptTagTrace,
|
|
17
18
|
TagSettings,
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
prompt_tags,
|
|
22
|
-
tag_picture,
|
|
19
|
+
fit_image,
|
|
20
|
+
tag_image,
|
|
21
|
+
tag_prompt,
|
|
23
22
|
thing_scores,
|
|
23
|
+
to_english,
|
|
24
24
|
)
|
|
25
25
|
|
|
26
26
|
__all__ = [
|
|
@@ -28,23 +28,23 @@ __all__ = [
|
|
|
28
28
|
"IMAGE_SLOT",
|
|
29
29
|
"PACKED_TOKEN_CEILING",
|
|
30
30
|
"ComfyClipBackend",
|
|
31
|
-
"
|
|
31
|
+
"ImageTagTrace",
|
|
32
|
+
"PromptTagTrace",
|
|
32
33
|
"TagSettings",
|
|
33
|
-
"TagTrace",
|
|
34
34
|
"chat",
|
|
35
35
|
"clip_generate",
|
|
36
36
|
"comfy_classifier",
|
|
37
|
-
"
|
|
38
|
-
"fit_picture",
|
|
37
|
+
"fit_image",
|
|
39
38
|
"generate_text",
|
|
40
|
-
"prompt_tags",
|
|
41
39
|
"routed_progress_bars",
|
|
42
40
|
"run_outcome",
|
|
43
41
|
"send_status",
|
|
44
42
|
"shared_vision_encode",
|
|
45
43
|
"skip_resident_loads",
|
|
46
44
|
"stop_when",
|
|
47
|
-
"
|
|
45
|
+
"tag_image",
|
|
46
|
+
"tag_prompt",
|
|
48
47
|
"thing_scores",
|
|
48
|
+
"to_english",
|
|
49
49
|
"token_fill",
|
|
50
50
|
]
|
{logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/toolkit/comfyui/scopes.py
RENAMED
|
@@ -12,7 +12,7 @@ from ...backends.comfy_clip import _clip_parts, _resident
|
|
|
12
12
|
|
|
13
13
|
@contextmanager
|
|
14
14
|
def shared_vision_encode(clip: Any) -> Iterator[None]:
|
|
15
|
-
"""Run the vision tower once per
|
|
15
|
+
"""Run the vision tower once per image tensor for the duration, shared by generate and classify.
|
|
16
16
|
|
|
17
17
|
Each encode runs inside the determinism window, where classify always ran it, so its
|
|
18
18
|
scores are unchanged. A transformer without core's preprocess_embed passes through.
|
{logit_classifier-0.3.0 → logit_classifier-0.4.0}/src/logit_classifier/toolkit/comfyui/tagger.py
RENAMED
|
@@ -15,12 +15,16 @@ from typing import Any
|
|
|
15
15
|
from ...classifier import Classifier
|
|
16
16
|
from ...schema import ChoiceAnswer, ChoiceQuestion, JSONContent, Question, SystemOneRequest
|
|
17
17
|
from ..tags import (
|
|
18
|
+
BROAD_THRESHOLDS,
|
|
19
|
+
PRESENCE_STATEMENT,
|
|
18
20
|
MergedCandidates,
|
|
21
|
+
PresenceThresholds,
|
|
19
22
|
drop_subsets,
|
|
20
23
|
in_prompt,
|
|
21
24
|
merge_candidates,
|
|
22
25
|
parse_candidates,
|
|
23
26
|
passed_by_origin,
|
|
27
|
+
presence_statement,
|
|
24
28
|
prompt_word_forms,
|
|
25
29
|
tag_stop,
|
|
26
30
|
)
|
|
@@ -31,7 +35,7 @@ _PROMPT = "{PROMPT}"
|
|
|
31
35
|
_TAG = "{TAG}"
|
|
32
36
|
_THING_CHECKS = ("all", "prompt")
|
|
33
37
|
_SUBSETS_RULES = ("words", "head-noun")
|
|
34
|
-
_ORDERS = ("score", "
|
|
38
|
+
_ORDERS = ("score", "candidate")
|
|
35
39
|
|
|
36
40
|
|
|
37
41
|
@dataclass(frozen=True, kw_only=True)
|
|
@@ -61,15 +65,14 @@ class TagSettings:
|
|
|
61
65
|
# On fp8, 40 over 25 added 0.85 correct and 0.15 wrong or vague tags per render, and 0.87 and 0.10 per tile
|
|
62
66
|
# (the Logit Tagger's tests-AB/ab_tagger_score.py --replay, ContextAnchoredTileRefine tests-AB/ab_tags_bench.py).
|
|
63
67
|
propose_cap: int = 25
|
|
64
|
-
# The model's 25 and a long prompt's
|
|
65
|
-
# (ContextAnchoredTileRefine tests-AB/tags-bench-log.md).
|
|
68
|
+
# The model's 25 and a long prompt's about 40 tags, in one packed verify pass beside a 1 MP image
|
|
69
|
+
# (ContextAnchoredTileRefine tests-AB/tags-bench-log.md, section 7).
|
|
66
70
|
merge_cap: int = 64
|
|
67
71
|
# The propose wording's own nouns came back as tags and passed verify (ContextAnchoredTileRefine
|
|
68
72
|
# tests-AB/ab_tile_tags.py).
|
|
69
73
|
category_nouns: frozenset[str] = frozenset({"objects", "people", "animals", "clothing", "materials", "setting"})
|
|
70
74
|
|
|
71
|
-
#
|
|
72
|
-
# the prompt's own things the model pads the list with words the prompt lacks, so a streak of them ends it.
|
|
75
|
+
# Past the prompt's own things the model pads the list with words the prompt lacks, so a streak of them ends it.
|
|
73
76
|
prompt_tags_instruction: str = (
|
|
74
77
|
"This is a prompt for an image:\n{PROMPT}\n\nList the physical things the prompt names that "
|
|
75
78
|
"could be pointed at in the image: objects, people, animals, plants, clothing, materials, "
|
|
@@ -100,21 +103,17 @@ class TagSettings:
|
|
|
100
103
|
"shape or form, lighting in general, a camera or render effect, a style, a mood, a "
|
|
101
104
|
"quality or an idea"),
|
|
102
105
|
)
|
|
103
|
-
#
|
|
104
|
-
thing_threshold: float = 0.
|
|
106
|
+
# Low p("thing") scores mix things with non-things ("robots" 0.15 on fp8), so only a score below 0.1 drops a tag.
|
|
107
|
+
thing_threshold: float = 0.1
|
|
105
108
|
# Judging only the prompt's tags kept "watermark" and "pixel art", and added 0.28 vague tags per render and 0.81
|
|
106
109
|
# per tile on fp8 (ab_tagger_score.py --replay, ab_tags_bench.py).
|
|
107
110
|
thing_check: str = "all"
|
|
108
111
|
|
|
109
|
-
# The
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
#
|
|
113
|
-
|
|
114
|
-
verify_threshold: float = 0.999
|
|
115
|
-
# A near name passes 0.9, and 0.9999 cut wrong tags per tile from 0.92 to 0.29 (ContextAnchoredTileRefine
|
|
116
|
-
# tests-AB/tags-bench-log.md).
|
|
117
|
-
prompt_only_threshold: float = 0.9999
|
|
112
|
+
# The toolkit's presence wording (toolkit/tags.py), which the Logit Tagger asks of a listed tag and a prompt's thing.
|
|
113
|
+
verify_statement: str = PRESENCE_STATEMENT
|
|
114
|
+
# On fp8, 0.999 over 0.9 removed a wrong or vague tag per 2.6 to 4.0 correct ones, and 0.9999 on a declared tag cut
|
|
115
|
+
# wrong tags per tile from 0.83 to 0.29 (ab_tagger_score.py --replay, ContextAnchoredTileRefine tags-bench-log.md).
|
|
116
|
+
verify_thresholds: PresenceThresholds = BROAD_THRESHOLDS
|
|
118
117
|
|
|
119
118
|
# The "head-noun" rule keeps "cat" beside "cat ear", and also "stone" beside "stone walls" and "castle" beside
|
|
120
119
|
# "stone castle tower". On fp8 tiles it added 0.03 wrong or vague tags per tile (ab_tags_bench.py).
|
|
@@ -127,12 +126,7 @@ class TagSettings:
|
|
|
127
126
|
("translate_instruction", self.translate_instruction, _PROMPT),
|
|
128
127
|
("thing_question", self.thing_question, _TAG),
|
|
129
128
|
]
|
|
130
|
-
thresholds = {
|
|
131
|
-
"english_threshold": self.english_threshold,
|
|
132
|
-
"thing_threshold": self.thing_threshold,
|
|
133
|
-
"verify_threshold": self.verify_threshold,
|
|
134
|
-
"prompt_only_threshold": self.prompt_only_threshold,
|
|
135
|
-
}
|
|
129
|
+
thresholds: dict[str, float] = {}
|
|
136
130
|
budgets = {
|
|
137
131
|
"propose_max_tokens": self.propose_max_tokens,
|
|
138
132
|
"propose_cap": self.propose_cap,
|
|
@@ -147,6 +141,15 @@ class TagSettings:
|
|
|
147
141
|
}
|
|
148
142
|
criteria_keys = {key for key, _text in self.thing_criteria}
|
|
149
143
|
|
|
144
|
+
if not isinstance(self.verify_thresholds, PresenceThresholds):
|
|
145
|
+
raise ValueError(f"TagSettings got verify_thresholds={self.verify_thresholds!r}. Use a PresenceThresholds, "
|
|
146
|
+
f"such as BROAD_THRESHOLDS or STRICT_THRESHOLDS from logit_classifier.toolkit.tags.")
|
|
147
|
+
thresholds = {
|
|
148
|
+
"english_threshold": self.english_threshold,
|
|
149
|
+
"thing_threshold": self.thing_threshold,
|
|
150
|
+
"verify_thresholds.proposed": self.verify_thresholds.proposed,
|
|
151
|
+
"verify_thresholds.declared": self.verify_thresholds.declared,
|
|
152
|
+
}
|
|
150
153
|
# An empty verify statement turns verify off, so it needs no placeholder.
|
|
151
154
|
if self.verify_statement:
|
|
152
155
|
wordings.append(("verify_statement", self.verify_statement, _TAG))
|
|
@@ -175,23 +178,23 @@ class TagSettings:
|
|
|
175
178
|
|
|
176
179
|
|
|
177
180
|
@dataclass(frozen=True, kw_only=True)
|
|
178
|
-
class
|
|
179
|
-
"""One prompt's
|
|
181
|
+
class PromptTagTrace:
|
|
182
|
+
"""One prompt's pass through tag_prompt, whose tags the pack merges into every image's candidates.
|
|
180
183
|
|
|
181
|
-
text is the English text the listing read. listed holds the tags in_prompt keeps,
|
|
184
|
+
text is the English text the listing read. listed holds the tags in_prompt keeps, p_thing each one's thing check
|
|
182
185
|
score, and tags the listed tags the thing check keeps.
|
|
183
186
|
"""
|
|
184
187
|
|
|
185
188
|
text: str
|
|
186
189
|
reply: str
|
|
187
190
|
listed: tuple[str, ...]
|
|
188
|
-
|
|
191
|
+
p_thing: tuple[float, ...]
|
|
189
192
|
tags: tuple[str, ...]
|
|
190
193
|
|
|
191
194
|
|
|
192
195
|
@dataclass(frozen=True, kw_only=True)
|
|
193
|
-
class
|
|
194
|
-
"""One
|
|
196
|
+
class ImageTagTrace:
|
|
197
|
+
"""One image's pass through tag_image.
|
|
195
198
|
|
|
196
199
|
echo_reply holds the replaced reply when the echo fallback ran. dropped holds (name, reason) pairs from the merge
|
|
197
200
|
and the thing check. scores holds None per candidate with verify off. verified is before drop_subsets, and kept
|
|
@@ -211,7 +214,7 @@ class TagTrace:
|
|
|
211
214
|
verify_ms: float
|
|
212
215
|
|
|
213
216
|
|
|
214
|
-
def
|
|
217
|
+
def fit_image(image: Any, max_pixels: int) -> Any:
|
|
215
218
|
"""Return the first three channels of a [B, H, W, C] image, area-downscaled to at most max_pixels, never upscaled.
|
|
216
219
|
|
|
217
220
|
Core resizes a vision image only above 12.8 MP, so a 4k input would cost about 12k vision tokens.
|
|
@@ -225,7 +228,7 @@ def fit_picture(image: Any, max_pixels: int) -> Any:
|
|
|
225
228
|
fitted_width = 0
|
|
226
229
|
|
|
227
230
|
if max_pixels < 1:
|
|
228
|
-
raise ValueError(f"
|
|
231
|
+
raise ValueError(f"fit_image got max_pixels={max_pixels!r}. Use 1 or more.")
|
|
229
232
|
if height * width <= max_pixels:
|
|
230
233
|
return rgb
|
|
231
234
|
scale = math.sqrt(max_pixels / (height * width))
|
|
@@ -238,13 +241,13 @@ def fit_picture(image: Any, max_pixels: int) -> Any:
|
|
|
238
241
|
|
|
239
242
|
|
|
240
243
|
def thing_scores(
|
|
241
|
-
classifier: Classifier, tags: Sequence[str], settings: TagSettings,
|
|
244
|
+
classifier: Classifier, tags: Sequence[str], settings: TagSettings, thing_cache: dict[str, float]
|
|
242
245
|
) -> tuple[float, ...]:
|
|
243
|
-
"""Return the thing check's p("
|
|
246
|
+
"""Return the thing check's p("thing") per tag, asking every tag missing from thing_cache in one text-only request.
|
|
244
247
|
|
|
245
|
-
|
|
248
|
+
thing_cache carries the answers across a run, so each tag is asked once per cache.
|
|
246
249
|
"""
|
|
247
|
-
unknown = tuple(dict.fromkeys(tag for tag in tags if tag not in
|
|
250
|
+
unknown = tuple(dict.fromkeys(tag for tag in tags if tag not in thing_cache))
|
|
248
251
|
criteria: dict[str, JSONContent | None] = dict(settings.thing_criteria)
|
|
249
252
|
questions: dict[str, Question] = {}
|
|
250
253
|
|
|
@@ -258,11 +261,11 @@ def thing_scores(
|
|
|
258
261
|
answer = response.answers[qid]
|
|
259
262
|
if not isinstance(answer, ChoiceAnswer):
|
|
260
263
|
raise TypeError(f"thing question {qid} got a {answer.type} answer, not a choice")
|
|
261
|
-
|
|
262
|
-
return tuple(
|
|
264
|
+
thing_cache[tag] = answer.probabilities["thing"]
|
|
265
|
+
return tuple(thing_cache[tag] for tag in tags)
|
|
263
266
|
|
|
264
267
|
|
|
265
|
-
def
|
|
268
|
+
def to_english(clip: Any, classifier: Classifier, prompt: str, settings: TagSettings) -> str:
|
|
266
269
|
"""Return the prompt when the language gate is off or reads it as English, otherwise its English translation.
|
|
267
270
|
|
|
268
271
|
A blank translation gives the prompt back.
|
|
@@ -279,9 +282,9 @@ def english_text(clip: Any, classifier: Classifier, prompt: str, settings: TagSe
|
|
|
279
282
|
return translation or prompt
|
|
280
283
|
|
|
281
284
|
|
|
282
|
-
def
|
|
283
|
-
clip: Any, classifier: Classifier, prompt: str, settings: TagSettings,
|
|
284
|
-
) ->
|
|
285
|
+
def tag_prompt(
|
|
286
|
+
clip: Any, classifier: Classifier, prompt: str, settings: TagSettings, thing_cache: dict[str, float]
|
|
287
|
+
) -> PromptTagTrace:
|
|
285
288
|
"""List the physical things the prompt names that its English text grounds and the thing check keeps.
|
|
286
289
|
|
|
287
290
|
The listing reads the prompt's English text, since in_prompt and the streak stop match the text's own words.
|
|
@@ -290,33 +293,34 @@ def prompt_tags(
|
|
|
290
293
|
word_forms: frozenset[str] = frozenset()
|
|
291
294
|
reply = ""
|
|
292
295
|
listed: tuple[str, ...] = ()
|
|
293
|
-
|
|
296
|
+
p_thing: tuple[float, ...] = ()
|
|
294
297
|
|
|
295
298
|
if not prompt.strip():
|
|
296
|
-
return
|
|
297
|
-
text =
|
|
299
|
+
return PromptTagTrace(text="", reply="", listed=(), p_thing=(), tags=())
|
|
300
|
+
text = to_english(clip, classifier, prompt, settings)
|
|
298
301
|
word_forms = prompt_word_forms(text)
|
|
299
302
|
reply = generate_text(clip, settings.prompt_tags_instruction.replace(_PROMPT, text),
|
|
300
303
|
max_tokens=settings.prompt_tags_max_tokens,
|
|
301
304
|
should_stop=tag_stop(max_tags=None, ungrounded_streak=settings.ungrounded_streak,
|
|
302
305
|
word_forms=word_forms))
|
|
303
306
|
listed = tuple(tag for tag in parse_candidates(reply) if in_prompt(tag, word_forms))
|
|
304
|
-
|
|
305
|
-
return
|
|
306
|
-
|
|
307
|
+
p_thing = thing_scores(classifier, listed, settings, thing_cache)
|
|
308
|
+
return PromptTagTrace(text=text, reply=reply, listed=listed, p_thing=p_thing,
|
|
309
|
+
tags=tuple(tag for tag, p in zip(listed, p_thing, strict=True)
|
|
310
|
+
if p >= settings.thing_threshold))
|
|
307
311
|
|
|
308
312
|
|
|
309
|
-
def _propose(clip: Any,
|
|
310
|
-
reply = generate_text(clip, instruction, image=
|
|
313
|
+
def _propose(clip: Any, image: Any, instruction: str, settings: TagSettings) -> tuple[str, tuple[str, ...]]:
|
|
314
|
+
reply = generate_text(clip, instruction, image=image, max_tokens=settings.propose_max_tokens,
|
|
311
315
|
should_stop=tag_stop(max_tags=settings.propose_cap, ungrounded_streak=0))
|
|
312
316
|
|
|
313
317
|
return reply, tuple(parse_candidates(reply, max_candidates=settings.propose_cap))
|
|
314
318
|
|
|
315
319
|
|
|
316
320
|
def _things_only(classifier: Classifier, merged: MergedCandidates, settings: TagSettings,
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
is_thing = [p
|
|
321
|
+
thing_cache: dict[str, float]) -> MergedCandidates:
|
|
322
|
+
p_thing = thing_scores(classifier, merged.candidates, settings, thing_cache)
|
|
323
|
+
is_thing = [p >= settings.thing_threshold for p in p_thing]
|
|
320
324
|
not_things = tuple((tag, "not a thing") for tag, thing in zip(merged.candidates, is_thing, strict=True)
|
|
321
325
|
if not thing)
|
|
322
326
|
|
|
@@ -324,24 +328,24 @@ def _things_only(classifier: Classifier, merged: MergedCandidates, settings: Tag
|
|
|
324
328
|
tuple(itertools.compress(merged.origins, is_thing)), merged.dropped + not_things)
|
|
325
329
|
|
|
326
330
|
|
|
327
|
-
def
|
|
331
|
+
def tag_image(
|
|
328
332
|
clip: Any,
|
|
329
333
|
classifier: Classifier,
|
|
330
|
-
|
|
334
|
+
image: Any,
|
|
331
335
|
*,
|
|
332
336
|
prompt_tags: Sequence[str] = (),
|
|
333
337
|
settings: TagSettings,
|
|
334
|
-
|
|
338
|
+
thing_cache: dict[str, float],
|
|
335
339
|
on_stage: Callable[[str], None] | None = None,
|
|
336
|
-
) ->
|
|
337
|
-
"""Propose, merge with prompt_tags, check, verify and clean the tags of
|
|
340
|
+
) -> ImageTagTrace:
|
|
341
|
+
"""Propose, merge with prompt_tags, check, verify and clean the tags of an image the caller already fitted.
|
|
338
342
|
|
|
339
|
-
Propose and verify run in one shared_vision_encode scope, so the vision tower encodes the
|
|
343
|
+
Propose and verify run in one shared_vision_encode scope, so the vision tower encodes the image once. on_stage
|
|
340
344
|
hears "propose", then "verify".
|
|
341
345
|
"""
|
|
342
346
|
announce = on_stage if on_stage is not None else (lambda _stage: None)
|
|
343
|
-
thresholds = {"model": settings.
|
|
344
|
-
"prompt": settings.
|
|
347
|
+
thresholds = {"model": settings.verify_thresholds.proposed, "both": settings.verify_thresholds.proposed,
|
|
348
|
+
"prompt": settings.verify_thresholds.declared}
|
|
345
349
|
reply = ""
|
|
346
350
|
echo_reply: str | None = None
|
|
347
351
|
proposed: tuple[str, ...] = ()
|
|
@@ -358,21 +362,22 @@ def tag_picture(
|
|
|
358
362
|
with shared_vision_encode(clip):
|
|
359
363
|
announce("propose")
|
|
360
364
|
started = time.perf_counter()
|
|
361
|
-
reply, proposed = _propose(clip,
|
|
365
|
+
reply, proposed = _propose(clip, image, settings.propose_instruction, settings)
|
|
362
366
|
if (settings.echo_fallback_instruction is not None
|
|
363
367
|
and not merge_candidates(proposed, (), cap=settings.merge_cap,
|
|
364
368
|
drop=settings.category_nouns).candidates):
|
|
365
369
|
echo_reply = reply
|
|
366
|
-
reply, proposed = _propose(clip,
|
|
370
|
+
reply, proposed = _propose(clip, image, settings.echo_fallback_instruction, settings)
|
|
367
371
|
propose_ms = (time.perf_counter() - started) * 1000.0
|
|
368
372
|
merged = merge_candidates(proposed, prompt_tags, cap=settings.merge_cap, drop=settings.category_nouns)
|
|
369
373
|
if settings.thing_check == "all":
|
|
370
|
-
merged = _things_only(classifier, merged, settings,
|
|
374
|
+
merged = _things_only(classifier, merged, settings, thing_cache)
|
|
371
375
|
announce("verify")
|
|
372
376
|
started = time.perf_counter()
|
|
373
377
|
if settings.verify_statement:
|
|
374
378
|
verify_scores = tuple(classifier.nouls(
|
|
375
|
-
[settings.verify_statement
|
|
379
|
+
[presence_statement(tag, wording=settings.verify_statement) for tag in merged.candidates],
|
|
380
|
+
image=image))
|
|
376
381
|
verified = passed_by_origin(merged.candidates, merged.origins, verify_scores, thresholds=thresholds)
|
|
377
382
|
scores = verify_scores
|
|
378
383
|
else:
|
|
@@ -385,6 +390,6 @@ def tag_picture(
|
|
|
385
390
|
score_of = dict(zip(merged.candidates, verify_scores, strict=True))
|
|
386
391
|
# sorted is stable, so tied tags keep candidate order.
|
|
387
392
|
kept = tuple(sorted(kept, key=score_of.__getitem__, reverse=True))
|
|
388
|
-
return
|
|
389
|
-
|
|
390
|
-
|
|
393
|
+
return ImageTagTrace(reply=reply, echo_reply=echo_reply, proposed=proposed, candidates=merged.candidates,
|
|
394
|
+
origins=merged.origins, dropped=merged.dropped, scores=scores, verified=verified, kept=kept,
|
|
395
|
+
propose_ms=propose_ms, verify_ms=verify_ms)
|