vlm-toolkit 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. vlm_toolkit-0.1.0/LICENSE +21 -0
  2. vlm_toolkit-0.1.0/PKG-INFO +264 -0
  3. vlm_toolkit-0.1.0/README.md +212 -0
  4. vlm_toolkit-0.1.0/pyproject.toml +114 -0
  5. vlm_toolkit-0.1.0/pyproject.toml.orig +114 -0
  6. vlm_toolkit-0.1.0/src/vlm_toolkit/__init__.py +0 -0
  7. vlm_toolkit-0.1.0/src/vlm_toolkit/bbox.py +62 -0
  8. vlm_toolkit-0.1.0/src/vlm_toolkit/cmd/__init__.py +0 -0
  9. vlm_toolkit-0.1.0/src/vlm_toolkit/cmd/caption_images.py +44 -0
  10. vlm_toolkit-0.1.0/src/vlm_toolkit/cmd/detect_logo.py +42 -0
  11. vlm_toolkit-0.1.0/src/vlm_toolkit/cmd/main.py +87 -0
  12. vlm_toolkit-0.1.0/src/vlm_toolkit/cmd/remove_logo.py +155 -0
  13. vlm_toolkit-0.1.0/src/vlm_toolkit/cmd/upload_dataset.py +53 -0
  14. vlm_toolkit-0.1.0/src/vlm_toolkit/cmd/view_vlm_output.py +135 -0
  15. vlm_toolkit-0.1.0/src/vlm_toolkit/cmd/vlm_process.py +51 -0
  16. vlm_toolkit-0.1.0/src/vlm_toolkit/cmd/vlm_server.py +48 -0
  17. vlm_toolkit-0.1.0/src/vlm_toolkit/hf_dataset.py +77 -0
  18. vlm_toolkit-0.1.0/src/vlm_toolkit/img_utils.py +54 -0
  19. vlm_toolkit-0.1.0/src/vlm_toolkit/logo/__init__.py +7 -0
  20. vlm_toolkit-0.1.0/src/vlm_toolkit/logo/base.py +14 -0
  21. vlm_toolkit-0.1.0/src/vlm_toolkit/logo/gray.py +29 -0
  22. vlm_toolkit-0.1.0/src/vlm_toolkit/logo/opencv.py +35 -0
  23. vlm_toolkit-0.1.0/src/vlm_toolkit/models.py +20 -0
  24. vlm_toolkit-0.1.0/src/vlm_toolkit/prompts/caption.txt +1 -0
  25. vlm_toolkit-0.1.0/src/vlm_toolkit/prompts/detect_logo.txt +1 -0
  26. vlm_toolkit-0.1.0/src/vlm_toolkit/prompts/inat_classify.txt +24 -0
  27. vlm_toolkit-0.1.0/src/vlm_toolkit/py.typed +0 -0
  28. vlm_toolkit-0.1.0/src/vlm_toolkit/results.py +17 -0
  29. vlm_toolkit-0.1.0/src/vlm_toolkit/templates/item.html +9 -0
  30. vlm_toolkit-0.1.0/src/vlm_toolkit/templates/viewer.html +66 -0
  31. vlm_toolkit-0.1.0/src/vlm_toolkit/util.py +10 -0
  32. vlm_toolkit-0.1.0/src/vlm_toolkit/vlm.py +97 -0
  33. vlm_toolkit-0.1.0/src/vlm_toolkit/vlm_process.py +196 -0
  34. vlm_toolkit-0.1.0/src/vlm_toolkit/vlm_server.py +173 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Ben Lambert
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,264 @@
1
+ Metadata-Version: 2.4
2
+ Name: vlm-toolkit
3
+ Version: 0.1.0
4
+ Summary: Batch image processing with vision language models: captioning, logo and watermark detection, and logo removal
5
+ Keywords: vlm,vision-language-model,captioning,image-processing,qwen-vl
6
+ Author: Ben Lambert
7
+ Author-email: Ben Lambert <blambert@gmail.com>
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Environment :: Console
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Intended Audience :: Science/Research
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Programming Language :: Python :: 3.13
18
+ Classifier: Programming Language :: Python :: 3.14
19
+ Classifier: Topic :: Multimedia :: Graphics
20
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
21
+ Requires-Dist: click>=8.1.0
22
+ Requires-Dist: httpx>=0.28.1
23
+ Requires-Dist: pillow>=12.0.0
24
+ Requires-Dist: tqdm>=4.67.1
25
+ Requires-Dist: typer>=0.21.0
26
+ Requires-Dist: vlm-toolkit[local,logo,hub] ; extra == 'all'
27
+ Requires-Dist: vlm-toolkit[local,logo,hub] ; extra == 'dev'
28
+ Requires-Dist: black>=24.10.0 ; extra == 'dev'
29
+ Requires-Dist: isort>=5.13.2 ; extra == 'dev'
30
+ Requires-Dist: mypy>=1.14.0 ; extra == 'dev'
31
+ Requires-Dist: pytest>=8.3.4 ; extra == 'dev'
32
+ Requires-Dist: pytest-cov>=7.1.0 ; extra == 'dev'
33
+ Requires-Dist: datasets>=3.2.0 ; extra == 'hub'
34
+ Requires-Dist: accelerate>=1.12.0 ; extra == 'local'
35
+ Requires-Dist: kernels>=0.12.3 ; extra == 'local'
36
+ Requires-Dist: xgrammar>=0.2.6rc1 ; extra == 'local'
37
+ Requires-Dist: torch>=2.9.1 ; extra == 'local'
38
+ Requires-Dist: torchvision>=0.25.0 ; extra == 'local'
39
+ Requires-Dist: transformers>=5.5.0 ; extra == 'local'
40
+ Requires-Dist: numpy>=2.2.1 ; extra == 'logo'
41
+ Requires-Dist: opencv-python>=4.10.0 ; extra == 'logo'
42
+ Requires-Python: >=3.11
43
+ Project-URL: Homepage, https://github.com/belambert/vlm-toolkit
44
+ Project-URL: Repository, https://github.com/belambert/vlm-toolkit
45
+ Project-URL: Issues, https://github.com/belambert/vlm-toolkit/issues
46
+ Provides-Extra: all
47
+ Provides-Extra: dev
48
+ Provides-Extra: hub
49
+ Provides-Extra: local
50
+ Provides-Extra: logo
51
+ Description-Content-Type: text/markdown
52
+
53
+ # vlm-toolkit
54
+
55
+ [![Lint](https://github.com/belambert/vlm-toolkit/actions/workflows/lint.yml/badge.svg?branch=main)](https://github.com/belambert/vlm-toolkit/actions/workflows/lint.yml)
56
+ [![Test](https://github.com/belambert/vlm-toolkit/actions/workflows/test.yml/badge.svg?branch=main)](https://github.com/belambert/vlm-toolkit/actions/workflows/test.yml)
57
+ [![codecov](https://codecov.io/gh/belambert/vlm-toolkit/graph/badge.svg)](https://codecov.io/gh/belambert/vlm-toolkit)
58
+
59
+ Batch image processing with vision language models: captioning, logo and
60
+ watermark detection, and logo removal.
61
+
62
+ ## Installation
63
+
64
+ pip install vlm-toolkit
65
+
66
+ The base install is small: it covers `vlm server`, `vlm view-output`, and the
67
+ shared plumbing. The heavier pieces are optional extras, so you only pay for
68
+ what you use:
69
+
70
+ | Extra | Pulls in | Needed by |
71
+ | ------- | ----------------------------- | ------------------------------------------ |
72
+ | `local` | torch, transformers, xgrammar | `process`, `caption-images`, `detect-logo` |
73
+ | `logo` | opencv, numpy | `remove-logo` |
74
+ | `hub` | datasets | `upload-dataset` |
75
+ | `all` | all of the above | everything |
76
+
77
+ pip install 'vlm-toolkit[local]'
78
+ pip install 'vlm-toolkit[all]'
79
+
80
+ Running a command whose extra is missing tells you which one to install rather
81
+ than raising an ImportError.
82
+
83
+ For a source checkout:
84
+
85
+ uv sync --extra dev
86
+
87
+ ## Commands
88
+
89
+ Everything is a subcommand of `vlm`; run `vlm --help` for the list. From a
90
+ source checkout, prefix with `uv run`.
91
+
92
+ ### process
93
+
94
+ Run a VLM over a folder of images locally.
95
+
96
+ vlm process <image-dir>
97
+ vlm process <image-dir> --batch-size 8 --max-dim 1024
98
+ vlm process <image-dir> --prompt "Describe this image in one sentence"
99
+ vlm process <image-dir> --prompt-file my_prompt.txt
100
+ vlm process <image-dir> --schema schema.json
101
+
102
+ `--schema` takes a JSON schema file and constrains decoding to match it, via
103
+ xgrammar. `--prompt-file` overrides `--prompt`. Output is JSONL, written to
104
+ `--output`.
105
+
106
+ ### server
107
+
108
+ Same idea, but against a running OpenAI-compatible vision endpoint instead of a
109
+ local model.
110
+
111
+ vlm server <image-dir> --base-url http://localhost:8000/v1
112
+ vlm server <image-dir> --concurrency 8 --max-tokens 512
113
+
114
+ ### caption-images
115
+
116
+ `vlm process` preset that captions with `prompts/caption.txt`, writing
117
+ `captions.json` into the image folder by default.
118
+
119
+ vlm caption-images <image-dir>
120
+
121
+ ### detect-logo
122
+
123
+ `vlm process` preset that finds logos and watermarks using
124
+ `prompts/detect_logo.txt`, writing `logo_bbox_output.json` into the image folder
125
+ by default.
126
+
127
+ vlm detect-logo <image-dir>
128
+
129
+ ### remove-logo
130
+
131
+ Erases the boxes found by `detect-logo`. Takes that command's JSON output, not
132
+ an image folder. Two methods: `gray` replaces each box with a gray rectangle,
133
+ `opencv` inpaints it (the default).
134
+
135
+ vlm remove-logo <image-dir>/logo_bbox_output.json <output-dir>
136
+ vlm remove-logo <image-dir>/logo_bbox_output.json <output-dir> --method gray
137
+
138
+ ### upload-dataset
139
+
140
+ Publishes a `vlm process` run to the Hugging Face Hub as an image dataset.
141
+
142
+ vlm upload-dataset <output.jsonl> <user>/<dataset>
143
+ vlm upload-dataset <output.jsonl> <user>/<dataset> --no-private --split test
144
+ vlm upload-dataset <output.jsonl> <user>/<dataset> --card CARD.md
145
+
146
+ The images are embedded in the dataset rather than referenced by path, so the
147
+ result is self-contained and the Hub's dataset viewer works. Columns are
148
+ `image`, `output`, and the original `file_name`. Rows whose image is missing
149
+ from disk are reported and skipped.
150
+
151
+ Repos are created private by default; pass `--no-private` for a public one.
152
+ Authentication comes from `--token`, else `HF_TOKEN`, else a cached
153
+ `huggingface-cli login`.
154
+
155
+ `--card` attaches a markdown file as the dataset's README. The upload generates
156
+ a `dataset_info` block that the Hub viewer depends on, so the card is merged
157
+ rather than overwritten: your prose becomes the body, and any YAML frontmatter
158
+ in your file (`license`, `task_categories`, …) is layered on top of the
159
+ generated keys. The card path is checked before the upload starts, so a typo
160
+ fails immediately instead of after transferring the images.
161
+
162
+ ### view-output
163
+
164
+ Renders a JSONL output file as an HTML page and opens it in a browser.
165
+
166
+ vlm view-output <output.jsonl>
167
+ vlm view-output <output.jsonl> --serve --port 8000
168
+ vlm view-output <output.jsonl> --serve --host 0.0.0.0
169
+
170
+ By default it writes `<output.jsonl>.html` next to the input and opens it over
171
+ `file://`, with absolute paths to the images. `--serve` skips the file and hosts
172
+ the page instead, which is what you want when the images sit on a remote
173
+ machine.
174
+
175
+ The server roots itself at the closest directory containing both the JSONL and
176
+ every image it references, so `file_name` entries that point outside the JSON's
177
+ own directory still resolve.
178
+
179
+ `--host` controls the bind address. It defaults to `127.0.0.1`, so only the
180
+ local machine can connect; `--host 0.0.0.0` accepts external connections and
181
+ prints the LAN URL to open from another machine. There is no authentication and
182
+ the whole server root is readable, so on an untrusted network prefer the default
183
+ and forward the port over SSH instead:
184
+
185
+ ssh -L 8000:localhost:8000 <remote>
186
+
187
+ ## Prompts
188
+
189
+ The presets read their prompts from `src/vlm_toolkit/prompts/`, which ships as
190
+ package data, so they work from an installed wheel as well as a source checkout.
191
+ To use your own prompt, pass `vlm process --prompt-file <path>`.
192
+
193
+ The viewer page is packaged the same way. Its markup and CSS live in
194
+ `src/vlm_toolkit/templates/` as `viewer.html` (the shell, with an `$items`
195
+ placeholder) and `item.html` (one image plus its output), so restyling the
196
+ viewer means editing HTML rather than a Python string.
197
+
198
+ ## How Batching Works
199
+
200
+ `--batch-size` sets how many images go through the model at once. Images are
201
+ never resized to a common shape first, so a batch can mix sizes freely. Two
202
+ different mechanisms handle the resulting variation.
203
+
204
+ **Images are concatenated, not padded.** Qwen-VL processors resize each image on
205
+ its own, preserving aspect ratio and rounding to the processor's patch multiple,
206
+ then flatten it into a sequence of patches and concatenate every image's patches
207
+ into one 2D tensor. `pixel_values` has shape `(total_patches, feature_dim)` —
208
+ there is no per-image batch dimension, so there is nothing to pad. A companion
209
+ `image_grid_thw` records each image's `(t, h, w)` grid so the model can split
210
+ them apart again. A batch of three images with grids `[1,18,18]`, `[1,38,28]`
211
+ and `[1,22,14]` produces 1696 patch rows:
212
+
213
+ ```
214
+ 18×18 = 324
215
+ 38×28 = 1064
216
+ 22×14 = 308
217
+ ----
218
+ 1696
219
+ ```
220
+
221
+ Bigger images simply cost more tokens; none are spent on padding. Use
222
+ `--max-dim` to cap the longest edge before the processor sees the image, which
223
+ is the practical lever on both memory and token count.
224
+
225
+ **Text is left padded.** Each image expands into placeholder tokens
226
+ proportional to its patch grid, so differently sized images yield different
227
+ prompt lengths, and `padding=True` pads them to a common length.
228
+ `load_model` sets `padding_side = "left"` for two reasons: decoder-only
229
+ generation needs the last real token flush against the end of the sequence, and
230
+ `_run_inference` trims prompts with a single `out_ids[len(in_ids):]` offset
231
+ applied to every row. That slice is only correct when the padding sits on the
232
+ left — with right padding it would cut at the wrong point and leak pad and
233
+ prompt tokens into the decoded text.
234
+
235
+ **Batches are prefetched.** `vlm process` prepares the next batch on the CPU in
236
+ a background thread while the current one runs inference. Only the main thread
237
+ moves tensors onto the device.
238
+
239
+ The concatenated-patch layout above is specific to the Qwen-VL family. Models
240
+ that resize to a fixed square instead, such as the gemma entries in the
241
+ suggested list, produce a conventional stacked 4D `pixel_values`. Both work,
242
+ because images are handed to the processor one per prompt and each processor
243
+ applies its own preprocessing.
244
+
245
+ ## Environment Variables
246
+
247
+ - `HF_TOKEN` - Hugging Face token, for gated models (optional)
248
+
249
+ ## Notes
250
+
251
+ Detect logo took 1 hour to do 5000 imgs on an L4 GPU at ~$1/hour.
252
+
253
+ ## Releasing
254
+
255
+ Bump `version` in `pyproject.toml`, then push a matching tag:
256
+
257
+ git tag v0.1.0 && git push origin v0.1.0
258
+
259
+ `publish.yml` checks the tag against the project version, builds, and publishes
260
+ to PyPI via trusted publishing.
261
+
262
+ ## License
263
+
264
+ MIT - see [LICENSE](LICENSE).
@@ -0,0 +1,212 @@
1
+ # vlm-toolkit
2
+
3
+ [![Lint](https://github.com/belambert/vlm-toolkit/actions/workflows/lint.yml/badge.svg?branch=main)](https://github.com/belambert/vlm-toolkit/actions/workflows/lint.yml)
4
+ [![Test](https://github.com/belambert/vlm-toolkit/actions/workflows/test.yml/badge.svg?branch=main)](https://github.com/belambert/vlm-toolkit/actions/workflows/test.yml)
5
+ [![codecov](https://codecov.io/gh/belambert/vlm-toolkit/graph/badge.svg)](https://codecov.io/gh/belambert/vlm-toolkit)
6
+
7
+ Batch image processing with vision language models: captioning, logo and
8
+ watermark detection, and logo removal.
9
+
10
+ ## Installation
11
+
12
+ pip install vlm-toolkit
13
+
14
+ The base install is small: it covers `vlm server`, `vlm view-output`, and the
15
+ shared plumbing. The heavier pieces are optional extras, so you only pay for
16
+ what you use:
17
+
18
+ | Extra | Pulls in | Needed by |
19
+ | ------- | ----------------------------- | ------------------------------------------ |
20
+ | `local` | torch, transformers, xgrammar | `process`, `caption-images`, `detect-logo` |
21
+ | `logo` | opencv, numpy | `remove-logo` |
22
+ | `hub` | datasets | `upload-dataset` |
23
+ | `all` | all of the above | everything |
24
+
25
+ pip install 'vlm-toolkit[local]'
26
+ pip install 'vlm-toolkit[all]'
27
+
28
+ Running a command whose extra is missing tells you which one to install rather
29
+ than raising an ImportError.
30
+
31
+ For a source checkout:
32
+
33
+ uv sync --extra dev
34
+
35
+ ## Commands
36
+
37
+ Everything is a subcommand of `vlm`; run `vlm --help` for the list. From a
38
+ source checkout, prefix with `uv run`.
39
+
40
+ ### process
41
+
42
+ Run a VLM over a folder of images locally.
43
+
44
+ vlm process <image-dir>
45
+ vlm process <image-dir> --batch-size 8 --max-dim 1024
46
+ vlm process <image-dir> --prompt "Describe this image in one sentence"
47
+ vlm process <image-dir> --prompt-file my_prompt.txt
48
+ vlm process <image-dir> --schema schema.json
49
+
50
+ `--schema` takes a JSON schema file and constrains decoding to match it, via
51
+ xgrammar. `--prompt-file` overrides `--prompt`. Output is JSONL, written to
52
+ `--output`.
53
+
54
+ ### server
55
+
56
+ Same idea, but against a running OpenAI-compatible vision endpoint instead of a
57
+ local model.
58
+
59
+ vlm server <image-dir> --base-url http://localhost:8000/v1
60
+ vlm server <image-dir> --concurrency 8 --max-tokens 512
61
+
62
+ ### caption-images
63
+
64
+ `vlm process` preset that captions with `prompts/caption.txt`, writing
65
+ `captions.json` into the image folder by default.
66
+
67
+ vlm caption-images <image-dir>
68
+
69
+ ### detect-logo
70
+
71
+ `vlm process` preset that finds logos and watermarks using
72
+ `prompts/detect_logo.txt`, writing `logo_bbox_output.json` into the image folder
73
+ by default.
74
+
75
+ vlm detect-logo <image-dir>
76
+
77
+ ### remove-logo
78
+
79
+ Erases the boxes found by `detect-logo`. Takes that command's JSON output, not
80
+ an image folder. Two methods: `gray` replaces each box with a gray rectangle,
81
+ `opencv` inpaints it (the default).
82
+
83
+ vlm remove-logo <image-dir>/logo_bbox_output.json <output-dir>
84
+ vlm remove-logo <image-dir>/logo_bbox_output.json <output-dir> --method gray
85
+
86
+ ### upload-dataset
87
+
88
+ Publishes a `vlm process` run to the Hugging Face Hub as an image dataset.
89
+
90
+ vlm upload-dataset <output.jsonl> <user>/<dataset>
91
+ vlm upload-dataset <output.jsonl> <user>/<dataset> --no-private --split test
92
+ vlm upload-dataset <output.jsonl> <user>/<dataset> --card CARD.md
93
+
94
+ The images are embedded in the dataset rather than referenced by path, so the
95
+ result is self-contained and the Hub's dataset viewer works. Columns are
96
+ `image`, `output`, and the original `file_name`. Rows whose image is missing
97
+ from disk are reported and skipped.
98
+
99
+ Repos are created private by default; pass `--no-private` for a public one.
100
+ Authentication comes from `--token`, else `HF_TOKEN`, else a cached
101
+ `huggingface-cli login`.
102
+
103
+ `--card` attaches a markdown file as the dataset's README. The upload generates
104
+ a `dataset_info` block that the Hub viewer depends on, so the card is merged
105
+ rather than overwritten: your prose becomes the body, and any YAML frontmatter
106
+ in your file (`license`, `task_categories`, …) is layered on top of the
107
+ generated keys. The card path is checked before the upload starts, so a typo
108
+ fails immediately instead of after transferring the images.
109
+
110
+ ### view-output
111
+
112
+ Renders a JSONL output file as an HTML page and opens it in a browser.
113
+
114
+ vlm view-output <output.jsonl>
115
+ vlm view-output <output.jsonl> --serve --port 8000
116
+ vlm view-output <output.jsonl> --serve --host 0.0.0.0
117
+
118
+ By default it writes `<output.jsonl>.html` next to the input and opens it over
119
+ `file://`, with absolute paths to the images. `--serve` skips the file and hosts
120
+ the page instead, which is what you want when the images sit on a remote
121
+ machine.
122
+
123
+ The server roots itself at the closest directory containing both the JSONL and
124
+ every image it references, so `file_name` entries that point outside the JSON's
125
+ own directory still resolve.
126
+
127
+ `--host` controls the bind address. It defaults to `127.0.0.1`, so only the
128
+ local machine can connect; `--host 0.0.0.0` accepts external connections and
129
+ prints the LAN URL to open from another machine. There is no authentication and
130
+ the whole server root is readable, so on an untrusted network prefer the default
131
+ and forward the port over SSH instead:
132
+
133
+ ssh -L 8000:localhost:8000 <remote>
134
+
135
+ ## Prompts
136
+
137
+ The presets read their prompts from `src/vlm_toolkit/prompts/`, which ships as
138
+ package data, so they work from an installed wheel as well as a source checkout.
139
+ To use your own prompt, pass `vlm process --prompt-file <path>`.
140
+
141
+ The viewer page is packaged the same way. Its markup and CSS live in
142
+ `src/vlm_toolkit/templates/` as `viewer.html` (the shell, with an `$items`
143
+ placeholder) and `item.html` (one image plus its output), so restyling the
144
+ viewer means editing HTML rather than a Python string.
145
+
146
+ ## How Batching Works
147
+
148
+ `--batch-size` sets how many images go through the model at once. Images are
149
+ never resized to a common shape first, so a batch can mix sizes freely. Two
150
+ different mechanisms handle the resulting variation.
151
+
152
+ **Images are concatenated, not padded.** Qwen-VL processors resize each image on
153
+ its own, preserving aspect ratio and rounding to the processor's patch multiple,
154
+ then flatten it into a sequence of patches and concatenate every image's patches
155
+ into one 2D tensor. `pixel_values` has shape `(total_patches, feature_dim)` —
156
+ there is no per-image batch dimension, so there is nothing to pad. A companion
157
+ `image_grid_thw` records each image's `(t, h, w)` grid so the model can split
158
+ them apart again. A batch of three images with grids `[1,18,18]`, `[1,38,28]`
159
+ and `[1,22,14]` produces 1696 patch rows:
160
+
161
+ ```
162
+ 18×18 = 324
163
+ 38×28 = 1064
164
+ 22×14 = 308
165
+ ----
166
+ 1696
167
+ ```
168
+
169
+ Bigger images simply cost more tokens; none are spent on padding. Use
170
+ `--max-dim` to cap the longest edge before the processor sees the image, which
171
+ is the practical lever on both memory and token count.
172
+
173
+ **Text is left padded.** Each image expands into placeholder tokens
174
+ proportional to its patch grid, so differently sized images yield different
175
+ prompt lengths, and `padding=True` pads them to a common length.
176
+ `load_model` sets `padding_side = "left"` for two reasons: decoder-only
177
+ generation needs the last real token flush against the end of the sequence, and
178
+ `_run_inference` trims prompts with a single `out_ids[len(in_ids):]` offset
179
+ applied to every row. That slice is only correct when the padding sits on the
180
+ left — with right padding it would cut at the wrong point and leak pad and
181
+ prompt tokens into the decoded text.
182
+
183
+ **Batches are prefetched.** `vlm process` prepares the next batch on the CPU in
184
+ a background thread while the current one runs inference. Only the main thread
185
+ moves tensors onto the device.
186
+
187
+ The concatenated-patch layout above is specific to the Qwen-VL family. Models
188
+ that resize to a fixed square instead, such as the gemma entries in the
189
+ suggested list, produce a conventional stacked 4D `pixel_values`. Both work,
190
+ because images are handed to the processor one per prompt and each processor
191
+ applies its own preprocessing.
192
+
193
+ ## Environment Variables
194
+
195
+ - `HF_TOKEN` - Hugging Face token, for gated models (optional)
196
+
197
+ ## Notes
198
+
199
+ Detect logo took 1 hour to do 5000 imgs on an L4 GPU at ~$1/hour.
200
+
201
+ ## Releasing
202
+
203
+ Bump `version` in `pyproject.toml`, then push a matching tag:
204
+
205
+ git tag v0.1.0 && git push origin v0.1.0
206
+
207
+ `publish.yml` checks the tag against the project version, builds, and publishes
208
+ to PyPI via trusted publishing.
209
+
210
+ ## License
211
+
212
+ MIT - see [LICENSE](LICENSE).
@@ -0,0 +1,114 @@
1
+ [project]
2
+ name = "vlm-toolkit"
3
+ version = "0.1.0"
4
+ description = "Batch image processing with vision language models: captioning, logo and watermark detection, and logo removal"
5
+ readme = "README.md"
6
+ license = "MIT"
7
+ license-files = ["LICENSE"]
8
+ keywords = [
9
+ "vlm",
10
+ "vision-language-model",
11
+ "captioning",
12
+ "image-processing",
13
+ "qwen-vl",
14
+ ]
15
+ classifiers = [
16
+ "Development Status :: 3 - Alpha",
17
+ "Environment :: Console",
18
+ "Intended Audience :: Developers",
19
+ "Intended Audience :: Science/Research",
20
+ "Programming Language :: Python :: 3",
21
+ "Programming Language :: Python :: 3.11",
22
+ "Programming Language :: Python :: 3.12",
23
+ "Programming Language :: Python :: 3.13",
24
+ "Programming Language :: Python :: 3.14",
25
+ "Topic :: Multimedia :: Graphics",
26
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
27
+ ]
28
+ requires-python = ">=3.11"
29
+ dependencies = [
30
+ "click>=8.1.0",
31
+ "httpx>=0.28.1",
32
+ "pillow>=12.0.0",
33
+ "tqdm>=4.67.1",
34
+ "typer>=0.21.0",
35
+ ]
36
+
37
+ [[project.authors]]
38
+ name = "Ben Lambert"
39
+ email = "blambert@gmail.com"
40
+
41
+ [project.urls]
42
+ Homepage = "https://github.com/belambert/vlm-toolkit"
43
+ Repository = "https://github.com/belambert/vlm-toolkit"
44
+ Issues = "https://github.com/belambert/vlm-toolkit/issues"
45
+
46
+ [project.optional-dependencies]
47
+ local = [
48
+ "accelerate>=1.12.0",
49
+ "kernels>=0.12.3",
50
+ "xgrammar>=0.2.6rc1",
51
+ "torch>=2.9.1",
52
+ "torchvision>=0.25.0",
53
+ "transformers>=5.5.0",
54
+ ]
55
+ logo = [
56
+ "numpy>=2.2.1",
57
+ "opencv-python>=4.10.0",
58
+ ]
59
+ hub = ["datasets>=3.2.0"]
60
+ all = ["vlm-toolkit[local,logo,hub]"]
61
+ dev = [
62
+ "vlm-toolkit[local,logo,hub]",
63
+ "black>=24.10.0",
64
+ "isort>=5.13.2",
65
+ "mypy>=1.14.0",
66
+ "pytest>=8.3.4",
67
+ "pytest-cov>=7.1.0",
68
+ ]
69
+
70
+ [project.scripts]
71
+ vlm = "vlm_toolkit.cmd.main:cli"
72
+
73
+ [[tool.uv.sources.torch]]
74
+ index = "pytorch-cpu"
75
+ marker = "sys_platform != 'linux'"
76
+
77
+ [[tool.uv.sources.torch]]
78
+ index = "pytorch-cuda"
79
+ marker = "sys_platform == 'linux'"
80
+
81
+ [[tool.uv.index]]
82
+ name = "pytorch-cpu"
83
+ url = "https://download.pytorch.org/whl/cpu"
84
+ explicit = true
85
+
86
+ [[tool.uv.index]]
87
+ name = "pytorch-cuda"
88
+ url = "https://download.pytorch.org/whl/cu130"
89
+ explicit = true
90
+
91
+ [tool.black]
92
+ line-length = 88
93
+ target-version = ["py311"]
94
+ include = '\.pyi?$'
95
+
96
+ [tool.isort]
97
+ profile = "black"
98
+ line_length = 88
99
+ known_third_party = ["wandb"]
100
+
101
+ [tool.mypy]
102
+ python_version = "3.13"
103
+ files = ["src"]
104
+ ignore_missing_imports = true
105
+
106
+ [tool.pytest.ini_options]
107
+ testpaths = ["tests"]
108
+ python_files = ["test_*.py"]
109
+ python_classes = ["Test*"]
110
+ python_functions = ["test_*"]
111
+
112
+ [build-system]
113
+ requires = ["uv_build>=0.9.3,<0.10.0"]
114
+ build-backend = "uv_build"