vlm-toolkit 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vlm_toolkit-0.1.0/LICENSE +21 -0
- vlm_toolkit-0.1.0/PKG-INFO +264 -0
- vlm_toolkit-0.1.0/README.md +212 -0
- vlm_toolkit-0.1.0/pyproject.toml +114 -0
- vlm_toolkit-0.1.0/pyproject.toml.orig +114 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/__init__.py +0 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/bbox.py +62 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/cmd/__init__.py +0 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/cmd/caption_images.py +44 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/cmd/detect_logo.py +42 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/cmd/main.py +87 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/cmd/remove_logo.py +155 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/cmd/upload_dataset.py +53 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/cmd/view_vlm_output.py +135 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/cmd/vlm_process.py +51 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/cmd/vlm_server.py +48 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/hf_dataset.py +77 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/img_utils.py +54 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/logo/__init__.py +7 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/logo/base.py +14 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/logo/gray.py +29 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/logo/opencv.py +35 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/models.py +20 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/prompts/caption.txt +1 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/prompts/detect_logo.txt +1 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/prompts/inat_classify.txt +24 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/py.typed +0 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/results.py +17 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/templates/item.html +9 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/templates/viewer.html +66 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/util.py +10 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/vlm.py +97 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/vlm_process.py +196 -0
- vlm_toolkit-0.1.0/src/vlm_toolkit/vlm_server.py +173 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ben Lambert
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: vlm-toolkit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Batch image processing with vision language models: captioning, logo and watermark detection, and logo removal
|
|
5
|
+
Keywords: vlm,vision-language-model,captioning,image-processing,qwen-vl
|
|
6
|
+
Author: Ben Lambert
|
|
7
|
+
Author-email: Ben Lambert <blambert@gmail.com>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
19
|
+
Classifier: Topic :: Multimedia :: Graphics
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
|
+
Requires-Dist: click>=8.1.0
|
|
22
|
+
Requires-Dist: httpx>=0.28.1
|
|
23
|
+
Requires-Dist: pillow>=12.0.0
|
|
24
|
+
Requires-Dist: tqdm>=4.67.1
|
|
25
|
+
Requires-Dist: typer>=0.21.0
|
|
26
|
+
Requires-Dist: vlm-toolkit[local,logo,hub] ; extra == 'all'
|
|
27
|
+
Requires-Dist: vlm-toolkit[local,logo,hub] ; extra == 'dev'
|
|
28
|
+
Requires-Dist: black>=24.10.0 ; extra == 'dev'
|
|
29
|
+
Requires-Dist: isort>=5.13.2 ; extra == 'dev'
|
|
30
|
+
Requires-Dist: mypy>=1.14.0 ; extra == 'dev'
|
|
31
|
+
Requires-Dist: pytest>=8.3.4 ; extra == 'dev'
|
|
32
|
+
Requires-Dist: pytest-cov>=7.1.0 ; extra == 'dev'
|
|
33
|
+
Requires-Dist: datasets>=3.2.0 ; extra == 'hub'
|
|
34
|
+
Requires-Dist: accelerate>=1.12.0 ; extra == 'local'
|
|
35
|
+
Requires-Dist: kernels>=0.12.3 ; extra == 'local'
|
|
36
|
+
Requires-Dist: xgrammar>=0.2.6rc1 ; extra == 'local'
|
|
37
|
+
Requires-Dist: torch>=2.9.1 ; extra == 'local'
|
|
38
|
+
Requires-Dist: torchvision>=0.25.0 ; extra == 'local'
|
|
39
|
+
Requires-Dist: transformers>=5.5.0 ; extra == 'local'
|
|
40
|
+
Requires-Dist: numpy>=2.2.1 ; extra == 'logo'
|
|
41
|
+
Requires-Dist: opencv-python>=4.10.0 ; extra == 'logo'
|
|
42
|
+
Requires-Python: >=3.11
|
|
43
|
+
Project-URL: Homepage, https://github.com/belambert/vlm-toolkit
|
|
44
|
+
Project-URL: Repository, https://github.com/belambert/vlm-toolkit
|
|
45
|
+
Project-URL: Issues, https://github.com/belambert/vlm-toolkit/issues
|
|
46
|
+
Provides-Extra: all
|
|
47
|
+
Provides-Extra: dev
|
|
48
|
+
Provides-Extra: hub
|
|
49
|
+
Provides-Extra: local
|
|
50
|
+
Provides-Extra: logo
|
|
51
|
+
Description-Content-Type: text/markdown
|
|
52
|
+
|
|
53
|
+
# vlm-toolkit
|
|
54
|
+
|
|
55
|
+
[](https://github.com/belambert/vlm-toolkit/actions/workflows/lint.yml)
|
|
56
|
+
[](https://github.com/belambert/vlm-toolkit/actions/workflows/test.yml)
|
|
57
|
+
[](https://codecov.io/gh/belambert/vlm-toolkit)
|
|
58
|
+
|
|
59
|
+
Batch image processing with vision language models: captioning, logo and
|
|
60
|
+
watermark detection, and logo removal.
|
|
61
|
+
|
|
62
|
+
## Installation
|
|
63
|
+
|
|
64
|
+
pip install vlm-toolkit
|
|
65
|
+
|
|
66
|
+
The base install is small: it covers `vlm server`, `vlm view-output`, and the
|
|
67
|
+
shared plumbing. The heavier pieces are optional extras, so you only pay for
|
|
68
|
+
what you use:
|
|
69
|
+
|
|
70
|
+
| Extra | Pulls in | Needed by |
|
|
71
|
+
| ------- | ----------------------------- | ------------------------------------------ |
|
|
72
|
+
| `local` | torch, transformers, xgrammar | `process`, `caption-images`, `detect-logo` |
|
|
73
|
+
| `logo` | opencv, numpy | `remove-logo` |
|
|
74
|
+
| `hub` | datasets | `upload-dataset` |
|
|
75
|
+
| `all` | all of the above | everything |
|
|
76
|
+
|
|
77
|
+
pip install 'vlm-toolkit[local]'
|
|
78
|
+
pip install 'vlm-toolkit[all]'
|
|
79
|
+
|
|
80
|
+
Running a command whose extra is missing tells you which one to install rather
|
|
81
|
+
than raising an ImportError.
|
|
82
|
+
|
|
83
|
+
For a source checkout:
|
|
84
|
+
|
|
85
|
+
uv sync --extra dev
|
|
86
|
+
|
|
87
|
+
## Commands
|
|
88
|
+
|
|
89
|
+
Everything is a subcommand of `vlm`; run `vlm --help` for the list. From a
|
|
90
|
+
source checkout, prefix with `uv run`.
|
|
91
|
+
|
|
92
|
+
### process
|
|
93
|
+
|
|
94
|
+
Run a VLM over a folder of images locally.
|
|
95
|
+
|
|
96
|
+
vlm process <image-dir>
|
|
97
|
+
vlm process <image-dir> --batch-size 8 --max-dim 1024
|
|
98
|
+
vlm process <image-dir> --prompt "Describe this image in one sentence"
|
|
99
|
+
vlm process <image-dir> --prompt-file my_prompt.txt
|
|
100
|
+
vlm process <image-dir> --schema schema.json
|
|
101
|
+
|
|
102
|
+
`--schema` takes a JSON schema file and constrains decoding to match it, via
|
|
103
|
+
xgrammar. `--prompt-file` overrides `--prompt`. Output is JSONL, written to
|
|
104
|
+
`--output`.
|
|
105
|
+
|
|
106
|
+
### server
|
|
107
|
+
|
|
108
|
+
Same idea, but against a running OpenAI-compatible vision endpoint instead of a
|
|
109
|
+
local model.
|
|
110
|
+
|
|
111
|
+
vlm server <image-dir> --base-url http://localhost:8000/v1
|
|
112
|
+
vlm server <image-dir> --concurrency 8 --max-tokens 512
|
|
113
|
+
|
|
114
|
+
### caption-images
|
|
115
|
+
|
|
116
|
+
`vlm process` preset that captions with `prompts/caption.txt`, writing
|
|
117
|
+
`captions.json` into the image folder by default.
|
|
118
|
+
|
|
119
|
+
vlm caption-images <image-dir>
|
|
120
|
+
|
|
121
|
+
### detect-logo
|
|
122
|
+
|
|
123
|
+
`vlm process` preset that finds logos and watermarks using
|
|
124
|
+
`prompts/detect_logo.txt`, writing `logo_bbox_output.json` into the image folder
|
|
125
|
+
by default.
|
|
126
|
+
|
|
127
|
+
vlm detect-logo <image-dir>
|
|
128
|
+
|
|
129
|
+
### remove-logo
|
|
130
|
+
|
|
131
|
+
Erases the boxes found by `detect-logo`. Takes that command's JSON output, not
|
|
132
|
+
an image folder. Two methods: `gray` replaces each box with a gray rectangle,
|
|
133
|
+
`opencv` inpaints it (the default).
|
|
134
|
+
|
|
135
|
+
vlm remove-logo <image-dir>/logo_bbox_output.json <output-dir>
|
|
136
|
+
vlm remove-logo <image-dir>/logo_bbox_output.json <output-dir> --method gray
|
|
137
|
+
|
|
138
|
+
### upload-dataset
|
|
139
|
+
|
|
140
|
+
Publishes a `vlm process` run to the Hugging Face Hub as an image dataset.
|
|
141
|
+
|
|
142
|
+
vlm upload-dataset <output.jsonl> <user>/<dataset>
|
|
143
|
+
vlm upload-dataset <output.jsonl> <user>/<dataset> --no-private --split test
|
|
144
|
+
vlm upload-dataset <output.jsonl> <user>/<dataset> --card CARD.md
|
|
145
|
+
|
|
146
|
+
The images are embedded in the dataset rather than referenced by path, so the
|
|
147
|
+
result is self-contained and the Hub's dataset viewer works. Columns are
|
|
148
|
+
`image`, `output`, and the original `file_name`. Rows whose image is missing
|
|
149
|
+
from disk are reported and skipped.
|
|
150
|
+
|
|
151
|
+
Repos are created private by default; pass `--no-private` for a public one.
|
|
152
|
+
Authentication comes from `--token`, else `HF_TOKEN`, else a cached
|
|
153
|
+
`huggingface-cli login`.
|
|
154
|
+
|
|
155
|
+
`--card` attaches a markdown file as the dataset's README. The upload generates
|
|
156
|
+
a `dataset_info` block that the Hub viewer depends on, so the card is merged
|
|
157
|
+
rather than overwritten: your prose becomes the body, and any YAML frontmatter
|
|
158
|
+
in your file (`license`, `task_categories`, …) is layered on top of the
|
|
159
|
+
generated keys. The card path is checked before the upload starts, so a typo
|
|
160
|
+
fails immediately instead of after transferring the images.
|
|
161
|
+
|
|
162
|
+
### view-output
|
|
163
|
+
|
|
164
|
+
Renders a JSONL output file as an HTML page and opens it in a browser.
|
|
165
|
+
|
|
166
|
+
vlm view-output <output.jsonl>
|
|
167
|
+
vlm view-output <output.jsonl> --serve --port 8000
|
|
168
|
+
vlm view-output <output.jsonl> --serve --host 0.0.0.0
|
|
169
|
+
|
|
170
|
+
By default it writes `<output.jsonl>.html` next to the input and opens it over
|
|
171
|
+
`file://`, with absolute paths to the images. `--serve` skips the file and hosts
|
|
172
|
+
the page instead, which is what you want when the images sit on a remote
|
|
173
|
+
machine.
|
|
174
|
+
|
|
175
|
+
The server roots itself at the closest directory containing both the JSONL and
|
|
176
|
+
every image it references, so `file_name` entries that point outside the JSON's
|
|
177
|
+
own directory still resolve.
|
|
178
|
+
|
|
179
|
+
`--host` controls the bind address. It defaults to `127.0.0.1`, so only the
|
|
180
|
+
local machine can connect; `--host 0.0.0.0` accepts external connections and
|
|
181
|
+
prints the LAN URL to open from another machine. There is no authentication and
|
|
182
|
+
the whole server root is readable, so on an untrusted network prefer the default
|
|
183
|
+
and forward the port over SSH instead:
|
|
184
|
+
|
|
185
|
+
ssh -L 8000:localhost:8000 <remote>
|
|
186
|
+
|
|
187
|
+
## Prompts
|
|
188
|
+
|
|
189
|
+
The presets read their prompts from `src/vlm_toolkit/prompts/`, which ships as
|
|
190
|
+
package data, so they work from an installed wheel as well as a source checkout.
|
|
191
|
+
To use your own prompt, pass `vlm process --prompt-file <path>`.
|
|
192
|
+
|
|
193
|
+
The viewer page is packaged the same way. Its markup and CSS live in
|
|
194
|
+
`src/vlm_toolkit/templates/` as `viewer.html` (the shell, with an `$items`
|
|
195
|
+
placeholder) and `item.html` (one image plus its output), so restyling the
|
|
196
|
+
viewer means editing HTML rather than a Python string.
|
|
197
|
+
|
|
198
|
+
## How Batching Works
|
|
199
|
+
|
|
200
|
+
`--batch-size` sets how many images go through the model at once. Images are
|
|
201
|
+
never resized to a common shape first, so a batch can mix sizes freely. Two
|
|
202
|
+
different mechanisms handle the resulting variation.
|
|
203
|
+
|
|
204
|
+
**Images are concatenated, not padded.** Qwen-VL processors resize each image on
|
|
205
|
+
its own, preserving aspect ratio and rounding to the processor's patch multiple,
|
|
206
|
+
then flatten it into a sequence of patches and concatenate every image's patches
|
|
207
|
+
into one 2D tensor. `pixel_values` has shape `(total_patches, feature_dim)` —
|
|
208
|
+
there is no per-image batch dimension, so there is nothing to pad. A companion
|
|
209
|
+
`image_grid_thw` records each image's `(t, h, w)` grid so the model can split
|
|
210
|
+
them apart again. A batch of three images with grids `[1,18,18]`, `[1,38,28]`
|
|
211
|
+
and `[1,22,14]` produces 1696 patch rows:
|
|
212
|
+
|
|
213
|
+
```
|
|
214
|
+
18×18 = 324
|
|
215
|
+
38×28 = 1064
|
|
216
|
+
22×14 = 308
|
|
217
|
+
----
|
|
218
|
+
1696
|
|
219
|
+
```
|
|
220
|
+
|
|
221
|
+
Bigger images simply cost more tokens; none are spent on padding. Use
|
|
222
|
+
`--max-dim` to cap the longest edge before the processor sees the image, which
|
|
223
|
+
is the practical lever on both memory and token count.
|
|
224
|
+
|
|
225
|
+
**Text is left padded.** Each image expands into placeholder tokens
|
|
226
|
+
proportional to its patch grid, so differently sized images yield different
|
|
227
|
+
prompt lengths, and `padding=True` pads them to a common length.
|
|
228
|
+
`load_model` sets `padding_side = "left"` for two reasons: decoder-only
|
|
229
|
+
generation needs the last real token flush against the end of the sequence, and
|
|
230
|
+
`_run_inference` trims prompts with a single `out_ids[len(in_ids):]` offset
|
|
231
|
+
applied to every row. That slice is only correct when the padding sits on the
|
|
232
|
+
left — with right padding it would cut at the wrong point and leak pad and
|
|
233
|
+
prompt tokens into the decoded text.
|
|
234
|
+
|
|
235
|
+
**Batches are prefetched.** `vlm process` prepares the next batch on the CPU in
|
|
236
|
+
a background thread while the current one runs inference. Only the main thread
|
|
237
|
+
moves tensors onto the device.
|
|
238
|
+
|
|
239
|
+
The concatenated-patch layout above is specific to the Qwen-VL family. Models
|
|
240
|
+
that resize to a fixed square instead, such as the gemma entries in the
|
|
241
|
+
suggested list, produce a conventional stacked 4D `pixel_values`. Both work,
|
|
242
|
+
because images are handed to the processor one per prompt and each processor
|
|
243
|
+
applies its own preprocessing.
|
|
244
|
+
|
|
245
|
+
## Environment Variables
|
|
246
|
+
|
|
247
|
+
- `HF_TOKEN` - Hugging Face token, for gated models (optional)
|
|
248
|
+
|
|
249
|
+
## Notes
|
|
250
|
+
|
|
251
|
+
Detect logo took 1 hour to do 5000 imgs on an L4 GPU at ~$1/hour.
|
|
252
|
+
|
|
253
|
+
## Releasing
|
|
254
|
+
|
|
255
|
+
Bump `version` in `pyproject.toml`, then push a matching tag:
|
|
256
|
+
|
|
257
|
+
git tag v0.1.0 && git push origin v0.1.0
|
|
258
|
+
|
|
259
|
+
`publish.yml` checks the tag against the project version, builds, and publishes
|
|
260
|
+
to PyPI via trusted publishing.
|
|
261
|
+
|
|
262
|
+
## License
|
|
263
|
+
|
|
264
|
+
MIT - see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
# vlm-toolkit
|
|
2
|
+
|
|
3
|
+
[](https://github.com/belambert/vlm-toolkit/actions/workflows/lint.yml)
|
|
4
|
+
[](https://github.com/belambert/vlm-toolkit/actions/workflows/test.yml)
|
|
5
|
+
[](https://codecov.io/gh/belambert/vlm-toolkit)
|
|
6
|
+
|
|
7
|
+
Batch image processing with vision language models: captioning, logo and
|
|
8
|
+
watermark detection, and logo removal.
|
|
9
|
+
|
|
10
|
+
## Installation
|
|
11
|
+
|
|
12
|
+
pip install vlm-toolkit
|
|
13
|
+
|
|
14
|
+
The base install is small: it covers `vlm server`, `vlm view-output`, and the
|
|
15
|
+
shared plumbing. The heavier pieces are optional extras, so you only pay for
|
|
16
|
+
what you use:
|
|
17
|
+
|
|
18
|
+
| Extra | Pulls in | Needed by |
|
|
19
|
+
| ------- | ----------------------------- | ------------------------------------------ |
|
|
20
|
+
| `local` | torch, transformers, xgrammar | `process`, `caption-images`, `detect-logo` |
|
|
21
|
+
| `logo` | opencv, numpy | `remove-logo` |
|
|
22
|
+
| `hub` | datasets | `upload-dataset` |
|
|
23
|
+
| `all` | all of the above | everything |
|
|
24
|
+
|
|
25
|
+
pip install 'vlm-toolkit[local]'
|
|
26
|
+
pip install 'vlm-toolkit[all]'
|
|
27
|
+
|
|
28
|
+
Running a command whose extra is missing tells you which one to install rather
|
|
29
|
+
than raising an ImportError.
|
|
30
|
+
|
|
31
|
+
For a source checkout:
|
|
32
|
+
|
|
33
|
+
uv sync --extra dev
|
|
34
|
+
|
|
35
|
+
## Commands
|
|
36
|
+
|
|
37
|
+
Everything is a subcommand of `vlm`; run `vlm --help` for the list. From a
|
|
38
|
+
source checkout, prefix with `uv run`.
|
|
39
|
+
|
|
40
|
+
### process
|
|
41
|
+
|
|
42
|
+
Run a VLM over a folder of images locally.
|
|
43
|
+
|
|
44
|
+
vlm process <image-dir>
|
|
45
|
+
vlm process <image-dir> --batch-size 8 --max-dim 1024
|
|
46
|
+
vlm process <image-dir> --prompt "Describe this image in one sentence"
|
|
47
|
+
vlm process <image-dir> --prompt-file my_prompt.txt
|
|
48
|
+
vlm process <image-dir> --schema schema.json
|
|
49
|
+
|
|
50
|
+
`--schema` takes a JSON schema file and constrains decoding to match it, via
|
|
51
|
+
xgrammar. `--prompt-file` overrides `--prompt`. Output is JSONL, written to
|
|
52
|
+
`--output`.
|
|
53
|
+
|
|
54
|
+
### server
|
|
55
|
+
|
|
56
|
+
Same idea, but against a running OpenAI-compatible vision endpoint instead of a
|
|
57
|
+
local model.
|
|
58
|
+
|
|
59
|
+
vlm server <image-dir> --base-url http://localhost:8000/v1
|
|
60
|
+
vlm server <image-dir> --concurrency 8 --max-tokens 512
|
|
61
|
+
|
|
62
|
+
### caption-images
|
|
63
|
+
|
|
64
|
+
`vlm process` preset that captions with `prompts/caption.txt`, writing
|
|
65
|
+
`captions.json` into the image folder by default.
|
|
66
|
+
|
|
67
|
+
vlm caption-images <image-dir>
|
|
68
|
+
|
|
69
|
+
### detect-logo
|
|
70
|
+
|
|
71
|
+
`vlm process` preset that finds logos and watermarks using
|
|
72
|
+
`prompts/detect_logo.txt`, writing `logo_bbox_output.json` into the image folder
|
|
73
|
+
by default.
|
|
74
|
+
|
|
75
|
+
vlm detect-logo <image-dir>
|
|
76
|
+
|
|
77
|
+
### remove-logo
|
|
78
|
+
|
|
79
|
+
Erases the boxes found by `detect-logo`. Takes that command's JSON output, not
|
|
80
|
+
an image folder. Two methods: `gray` replaces each box with a gray rectangle,
|
|
81
|
+
`opencv` inpaints it (the default).
|
|
82
|
+
|
|
83
|
+
vlm remove-logo <image-dir>/logo_bbox_output.json <output-dir>
|
|
84
|
+
vlm remove-logo <image-dir>/logo_bbox_output.json <output-dir> --method gray
|
|
85
|
+
|
|
86
|
+
### upload-dataset
|
|
87
|
+
|
|
88
|
+
Publishes a `vlm process` run to the Hugging Face Hub as an image dataset.
|
|
89
|
+
|
|
90
|
+
vlm upload-dataset <output.jsonl> <user>/<dataset>
|
|
91
|
+
vlm upload-dataset <output.jsonl> <user>/<dataset> --no-private --split test
|
|
92
|
+
vlm upload-dataset <output.jsonl> <user>/<dataset> --card CARD.md
|
|
93
|
+
|
|
94
|
+
The images are embedded in the dataset rather than referenced by path, so the
|
|
95
|
+
result is self-contained and the Hub's dataset viewer works. Columns are
|
|
96
|
+
`image`, `output`, and the original `file_name`. Rows whose image is missing
|
|
97
|
+
from disk are reported and skipped.
|
|
98
|
+
|
|
99
|
+
Repos are created private by default; pass `--no-private` for a public one.
|
|
100
|
+
Authentication comes from `--token`, else `HF_TOKEN`, else a cached
|
|
101
|
+
`huggingface-cli login`.
|
|
102
|
+
|
|
103
|
+
`--card` attaches a markdown file as the dataset's README. The upload generates
|
|
104
|
+
a `dataset_info` block that the Hub viewer depends on, so the card is merged
|
|
105
|
+
rather than overwritten: your prose becomes the body, and any YAML frontmatter
|
|
106
|
+
in your file (`license`, `task_categories`, …) is layered on top of the
|
|
107
|
+
generated keys. The card path is checked before the upload starts, so a typo
|
|
108
|
+
fails immediately instead of after transferring the images.
|
|
109
|
+
|
|
110
|
+
### view-output
|
|
111
|
+
|
|
112
|
+
Renders a JSONL output file as an HTML page and opens it in a browser.
|
|
113
|
+
|
|
114
|
+
vlm view-output <output.jsonl>
|
|
115
|
+
vlm view-output <output.jsonl> --serve --port 8000
|
|
116
|
+
vlm view-output <output.jsonl> --serve --host 0.0.0.0
|
|
117
|
+
|
|
118
|
+
By default it writes `<output.jsonl>.html` next to the input and opens it over
|
|
119
|
+
`file://`, with absolute paths to the images. `--serve` skips the file and hosts
|
|
120
|
+
the page instead, which is what you want when the images sit on a remote
|
|
121
|
+
machine.
|
|
122
|
+
|
|
123
|
+
The server roots itself at the closest directory containing both the JSONL and
|
|
124
|
+
every image it references, so `file_name` entries that point outside the JSON's
|
|
125
|
+
own directory still resolve.
|
|
126
|
+
|
|
127
|
+
`--host` controls the bind address. It defaults to `127.0.0.1`, so only the
|
|
128
|
+
local machine can connect; `--host 0.0.0.0` accepts external connections and
|
|
129
|
+
prints the LAN URL to open from another machine. There is no authentication and
|
|
130
|
+
the whole server root is readable, so on an untrusted network prefer the default
|
|
131
|
+
and forward the port over SSH instead:
|
|
132
|
+
|
|
133
|
+
ssh -L 8000:localhost:8000 <remote>
|
|
134
|
+
|
|
135
|
+
## Prompts
|
|
136
|
+
|
|
137
|
+
The presets read their prompts from `src/vlm_toolkit/prompts/`, which ships as
|
|
138
|
+
package data, so they work from an installed wheel as well as a source checkout.
|
|
139
|
+
To use your own prompt, pass `vlm process --prompt-file <path>`.
|
|
140
|
+
|
|
141
|
+
The viewer page is packaged the same way. Its markup and CSS live in
|
|
142
|
+
`src/vlm_toolkit/templates/` as `viewer.html` (the shell, with an `$items`
|
|
143
|
+
placeholder) and `item.html` (one image plus its output), so restyling the
|
|
144
|
+
viewer means editing HTML rather than a Python string.
|
|
145
|
+
|
|
146
|
+
## How Batching Works
|
|
147
|
+
|
|
148
|
+
`--batch-size` sets how many images go through the model at once. Images are
|
|
149
|
+
never resized to a common shape first, so a batch can mix sizes freely. Two
|
|
150
|
+
different mechanisms handle the resulting variation.
|
|
151
|
+
|
|
152
|
+
**Images are concatenated, not padded.** Qwen-VL processors resize each image on
|
|
153
|
+
its own, preserving aspect ratio and rounding to the processor's patch multiple,
|
|
154
|
+
then flatten it into a sequence of patches and concatenate every image's patches
|
|
155
|
+
into one 2D tensor. `pixel_values` has shape `(total_patches, feature_dim)` —
|
|
156
|
+
there is no per-image batch dimension, so there is nothing to pad. A companion
|
|
157
|
+
`image_grid_thw` records each image's `(t, h, w)` grid so the model can split
|
|
158
|
+
them apart again. A batch of three images with grids `[1,18,18]`, `[1,38,28]`
|
|
159
|
+
and `[1,22,14]` produces 1696 patch rows:
|
|
160
|
+
|
|
161
|
+
```
|
|
162
|
+
18×18 = 324
|
|
163
|
+
38×28 = 1064
|
|
164
|
+
22×14 = 308
|
|
165
|
+
----
|
|
166
|
+
1696
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
Bigger images simply cost more tokens; none are spent on padding. Use
|
|
170
|
+
`--max-dim` to cap the longest edge before the processor sees the image, which
|
|
171
|
+
is the practical lever on both memory and token count.
|
|
172
|
+
|
|
173
|
+
**Text is left padded.** Each image expands into placeholder tokens
|
|
174
|
+
proportional to its patch grid, so differently sized images yield different
|
|
175
|
+
prompt lengths, and `padding=True` pads them to a common length.
|
|
176
|
+
`load_model` sets `padding_side = "left"` for two reasons: decoder-only
|
|
177
|
+
generation needs the last real token flush against the end of the sequence, and
|
|
178
|
+
`_run_inference` trims prompts with a single `out_ids[len(in_ids):]` offset
|
|
179
|
+
applied to every row. That slice is only correct when the padding sits on the
|
|
180
|
+
left — with right padding it would cut at the wrong point and leak pad and
|
|
181
|
+
prompt tokens into the decoded text.
|
|
182
|
+
|
|
183
|
+
**Batches are prefetched.** `vlm process` prepares the next batch on the CPU in
|
|
184
|
+
a background thread while the current one runs inference. Only the main thread
|
|
185
|
+
moves tensors onto the device.
|
|
186
|
+
|
|
187
|
+
The concatenated-patch layout above is specific to the Qwen-VL family. Models
|
|
188
|
+
that resize to a fixed square instead, such as the gemma entries in the
|
|
189
|
+
suggested list, produce a conventional stacked 4D `pixel_values`. Both work,
|
|
190
|
+
because images are handed to the processor one per prompt and each processor
|
|
191
|
+
applies its own preprocessing.
|
|
192
|
+
|
|
193
|
+
## Environment Variables
|
|
194
|
+
|
|
195
|
+
- `HF_TOKEN` - Hugging Face token, for gated models (optional)
|
|
196
|
+
|
|
197
|
+
## Notes
|
|
198
|
+
|
|
199
|
+
Detect logo took 1 hour to do 5000 imgs on an L4 GPU at ~$1/hour.
|
|
200
|
+
|
|
201
|
+
## Releasing
|
|
202
|
+
|
|
203
|
+
Bump `version` in `pyproject.toml`, then push a matching tag:
|
|
204
|
+
|
|
205
|
+
git tag v0.1.0 && git push origin v0.1.0
|
|
206
|
+
|
|
207
|
+
`publish.yml` checks the tag against the project version, builds, and publishes
|
|
208
|
+
to PyPI via trusted publishing.
|
|
209
|
+
|
|
210
|
+
## License
|
|
211
|
+
|
|
212
|
+
MIT - see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "vlm-toolkit"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Batch image processing with vision language models: captioning, logo and watermark detection, and logo removal"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
keywords = [
|
|
9
|
+
"vlm",
|
|
10
|
+
"vision-language-model",
|
|
11
|
+
"captioning",
|
|
12
|
+
"image-processing",
|
|
13
|
+
"qwen-vl",
|
|
14
|
+
]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Environment :: Console",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Intended Audience :: Science/Research",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"Programming Language :: Python :: 3.14",
|
|
25
|
+
"Topic :: Multimedia :: Graphics",
|
|
26
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
27
|
+
]
|
|
28
|
+
requires-python = ">=3.11"
|
|
29
|
+
dependencies = [
|
|
30
|
+
"click>=8.1.0",
|
|
31
|
+
"httpx>=0.28.1",
|
|
32
|
+
"pillow>=12.0.0",
|
|
33
|
+
"tqdm>=4.67.1",
|
|
34
|
+
"typer>=0.21.0",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
[[project.authors]]
|
|
38
|
+
name = "Ben Lambert"
|
|
39
|
+
email = "blambert@gmail.com"
|
|
40
|
+
|
|
41
|
+
[project.urls]
|
|
42
|
+
Homepage = "https://github.com/belambert/vlm-toolkit"
|
|
43
|
+
Repository = "https://github.com/belambert/vlm-toolkit"
|
|
44
|
+
Issues = "https://github.com/belambert/vlm-toolkit/issues"
|
|
45
|
+
|
|
46
|
+
[project.optional-dependencies]
|
|
47
|
+
local = [
|
|
48
|
+
"accelerate>=1.12.0",
|
|
49
|
+
"kernels>=0.12.3",
|
|
50
|
+
"xgrammar>=0.2.6rc1",
|
|
51
|
+
"torch>=2.9.1",
|
|
52
|
+
"torchvision>=0.25.0",
|
|
53
|
+
"transformers>=5.5.0",
|
|
54
|
+
]
|
|
55
|
+
logo = [
|
|
56
|
+
"numpy>=2.2.1",
|
|
57
|
+
"opencv-python>=4.10.0",
|
|
58
|
+
]
|
|
59
|
+
hub = ["datasets>=3.2.0"]
|
|
60
|
+
all = ["vlm-toolkit[local,logo,hub]"]
|
|
61
|
+
dev = [
|
|
62
|
+
"vlm-toolkit[local,logo,hub]",
|
|
63
|
+
"black>=24.10.0",
|
|
64
|
+
"isort>=5.13.2",
|
|
65
|
+
"mypy>=1.14.0",
|
|
66
|
+
"pytest>=8.3.4",
|
|
67
|
+
"pytest-cov>=7.1.0",
|
|
68
|
+
]
|
|
69
|
+
|
|
70
|
+
[project.scripts]
|
|
71
|
+
vlm = "vlm_toolkit.cmd.main:cli"
|
|
72
|
+
|
|
73
|
+
[[tool.uv.sources.torch]]
|
|
74
|
+
index = "pytorch-cpu"
|
|
75
|
+
marker = "sys_platform != 'linux'"
|
|
76
|
+
|
|
77
|
+
[[tool.uv.sources.torch]]
|
|
78
|
+
index = "pytorch-cuda"
|
|
79
|
+
marker = "sys_platform == 'linux'"
|
|
80
|
+
|
|
81
|
+
[[tool.uv.index]]
|
|
82
|
+
name = "pytorch-cpu"
|
|
83
|
+
url = "https://download.pytorch.org/whl/cpu"
|
|
84
|
+
explicit = true
|
|
85
|
+
|
|
86
|
+
[[tool.uv.index]]
|
|
87
|
+
name = "pytorch-cuda"
|
|
88
|
+
url = "https://download.pytorch.org/whl/cu130"
|
|
89
|
+
explicit = true
|
|
90
|
+
|
|
91
|
+
[tool.black]
|
|
92
|
+
line-length = 88
|
|
93
|
+
target-version = ["py311"]
|
|
94
|
+
include = '\.pyi?$'
|
|
95
|
+
|
|
96
|
+
[tool.isort]
|
|
97
|
+
profile = "black"
|
|
98
|
+
line_length = 88
|
|
99
|
+
known_third_party = ["wandb"]
|
|
100
|
+
|
|
101
|
+
[tool.mypy]
|
|
102
|
+
python_version = "3.13"
|
|
103
|
+
files = ["src"]
|
|
104
|
+
ignore_missing_imports = true
|
|
105
|
+
|
|
106
|
+
[tool.pytest.ini_options]
|
|
107
|
+
testpaths = ["tests"]
|
|
108
|
+
python_files = ["test_*.py"]
|
|
109
|
+
python_classes = ["Test*"]
|
|
110
|
+
python_functions = ["test_*"]
|
|
111
|
+
|
|
112
|
+
[build-system]
|
|
113
|
+
requires = ["uv_build>=0.9.3,<0.10.0"]
|
|
114
|
+
build-backend = "uv_build"
|