witbitz-code 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- witbitz_code/__init__.py +7 -0
- witbitz_code/__main__.py +3 -0
- witbitz_code/_js.py +312 -0
- witbitz_code/account.py +127 -0
- witbitz_code/asks.py +119 -0
- witbitz_code/attachments.py +266 -0
- witbitz_code/auto.py +561 -0
- witbitz_code/auto_runner.py +481 -0
- witbitz_code/cli.py +223 -0
- witbitz_code/codetools.py +128 -0
- witbitz_code/compress.py +25 -0
- witbitz_code/computers.py +189 -0
- witbitz_code/connector.py +818 -0
- witbitz_code/devicelink.py +159 -0
- witbitz_code/folders.py +250 -0
- witbitz_code/link.py +205 -0
- witbitz_code/net.py +67 -0
- witbitz_code/opencode_plugins/notes-init.md +8 -0
- witbitz_code/opencode_plugins/witbitz-media-server.mjs +791 -0
- witbitz_code/opencode_plugins/witbitz-notes.js +356 -0
- witbitz_code/opencode_plugins/witbitz-progress.js +120 -0
- witbitz_code/outputs.py +115 -0
- witbitz_code/pair.py +222 -0
- witbitz_code/pairings.py +214 -0
- witbitz_code/plugins.py +140 -0
- witbitz_code/policy.py +48 -0
- witbitz_code/qr.py +59 -0
- witbitz_code/relay.py +783 -0
- witbitz_code/seen.py +49 -0
- witbitz_code/tools_probe.py +67 -0
- witbitz_code-1.2.0.dist-info/METADATA +145 -0
- witbitz_code-1.2.0.dist-info/RECORD +36 -0
- witbitz_code-1.2.0.dist-info/WHEEL +5 -0
- witbitz_code-1.2.0.dist-info/entry_points.txt +2 -0
- witbitz_code-1.2.0.dist-info/licenses/LICENSE +21 -0
- witbitz_code-1.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,791 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// witbitz-media — a stdio MCP server of image, PDF, video and audio tools for coding agents (docs/media-mcp.md).
|
|
3
|
+
//
|
|
4
|
+
// Every tool that produces a picture returns it as an MCP `image` block — measured 2026-10-06: OpenCode, Codex and Claude
|
|
5
|
+
// Code all put such a block in front of the model — plus a text summary and the path of the saved file. Grouped tools with
|
|
6
|
+
// options, not one tool per operation: every definition rides along on every turn of the agent.
|
|
7
|
+
//
|
|
8
|
+
// It shells out to ImageMagick 6 (convert/identify/compare/montage), Tesseract, Poppler (pdftoppm/pdfinfo/pdftotext) and
|
|
9
|
+
// FFmpeg (ffmpeg/ffprobe) — what the cloud-computer image ships. Always an argument list, never a shell; every number is
|
|
10
|
+
// bounded; nothing touches the network. A program that is missing makes its tool answer which one to install.
|
|
11
|
+
//
|
|
12
|
+
// Speaks newline-delimited JSON-RPC over stdin/stdout (MCP: initialize → tools/list → tools/call), like agent/pdf-mcp.mjs.
|
|
13
|
+
import { execFile } from 'node:child_process'
|
|
14
|
+
import { createInterface } from 'node:readline'
|
|
15
|
+
import { mkdirSync, statSync, readFileSync, existsSync, readdirSync, rmSync, renameSync, createWriteStream, copyFileSync, writeFileSync } from 'node:fs'
|
|
16
|
+
import { createRequire } from 'node:module'
|
|
17
|
+
import { pathToFileURL } from 'node:url'
|
|
18
|
+
import { join, resolve, basename, dirname } from 'node:path'
|
|
19
|
+
import { tmpdir, homedir, availableParallelism } from 'node:os'
|
|
20
|
+
import { randomUUID, createHash } from 'node:crypto'
|
|
21
|
+
|
|
22
|
+
const PROTOCOL_VERSION = '2025-06-18'
|
|
23
|
+
const OUT = process.env.WITBITZ_MEDIA_DIR || join(tmpdir(), 'witbitz-media')
|
|
24
|
+
const MAX_SIDE = 1568 // the long side of any image handed to the model (larger is downscaled by every vision API anyway)
|
|
25
|
+
const MAX_FRAMES = 48
|
|
26
|
+
const TIMEOUT = Number(process.env.WITBITZ_MEDIA_TIMEOUT_MS || 120_000)
|
|
27
|
+
|
|
28
|
+
// ── plumbing ───────────────────────────────────────────────────────────────────────────────────────────────────────────
|
|
29
|
+
class ToolError extends Error {}
|
|
30
|
+
const fail = (msg) => { throw new ToolError(msg) }
|
|
31
|
+
const INSTALL = { convert: 'ImageMagick', identify: 'ImageMagick', compare: 'ImageMagick', montage: 'ImageMagick', tesseract: 'tesseract-ocr', pdftoppm: 'poppler-utils', pdfinfo: 'poppler-utils', pdftotext: 'poppler-utils', ffmpeg: 'ffmpeg', ffprobe: 'ffmpeg' }
|
|
32
|
+
|
|
33
|
+
/** Run a program (never a shell). → { stdout, stderr, code }. `ok` lists exit codes that are not failures. */
|
|
34
|
+
function run (cmd, args, { ok = [0], encoding = 'utf8', timeout = TIMEOUT } = {}) {
|
|
35
|
+
return new Promise((res, rej) => {
|
|
36
|
+
execFile(cmd, args, { timeout, maxBuffer: 64 * 1024 * 1024, encoding }, (err, stdout, stderr) => {
|
|
37
|
+
const code = err && typeof err.code === 'number' ? err.code : err ? -1 : 0
|
|
38
|
+
if (err && err.code === 'ENOENT') return rej(new ToolError(`${cmd} is not installed on this computer (install ${INSTALL[cmd] || cmd})`))
|
|
39
|
+
if (err && err.killed) return rej(new ToolError(`${cmd} took longer than ${Math.round(timeout / 1000)} s and was stopped`))
|
|
40
|
+
if (!ok.includes(code)) return rej(new ToolError(`${cmd} failed (${code}): ${String(stderr || err.message).trim().split('\n').slice(-2).join(' ').slice(0, 300)}`))
|
|
41
|
+
res({ stdout, stderr: String(stderr || ''), code })
|
|
42
|
+
})
|
|
43
|
+
})
|
|
44
|
+
}
|
|
45
|
+
const outFile = (ext) => { mkdirSync(OUT, { recursive: true }); return join(OUT, `${Date.now().toString(36)}-${randomUUID().slice(0, 8)}.${ext}`) }
|
|
46
|
+
function inputFile (p, what = 'file') {
|
|
47
|
+
if (typeof p !== 'string' || !p.trim()) fail(`${what}: give a path`)
|
|
48
|
+
const abs = resolve(p)
|
|
49
|
+
let st; try { st = statSync(abs) } catch { fail(`${what}: no such file: ${abs}`) }
|
|
50
|
+
if (!st.isFile()) fail(`${what}: not a file: ${abs}`)
|
|
51
|
+
return abs
|
|
52
|
+
}
|
|
53
|
+
const num = (v, lo, hi, name) => { if (v === undefined || v === null) return undefined; const n = Number(v); if (!Number.isFinite(n) || n < lo || n > hi) fail(`${name} must be a number from ${lo} to ${hi}`); return n }
|
|
54
|
+
const int = (v, lo, hi, name) => { const n = num(v, lo, hi, name); return n === undefined ? undefined : Math.round(n) }
|
|
55
|
+
/** [x, y, w, h] in pixels, all ≥ 0, w/h ≥ 1. */
|
|
56
|
+
function boxOf (b, name = 'box') {
|
|
57
|
+
if (b === undefined || b === null) return null
|
|
58
|
+
if (!Array.isArray(b) || b.length !== 4) fail(`${name} must be [x, y, width, height] in pixels`)
|
|
59
|
+
const [x, y, w, h] = b.map((v, i) => int(v, i < 2 ? 0 : 1, 100000, name))
|
|
60
|
+
return { x, y, w, h }
|
|
61
|
+
}
|
|
62
|
+
const geom = (b) => `${b.w}x${b.h}+${b.x}+${b.y}`
|
|
63
|
+
const ts = (s) => { const t = Math.max(0, Number(s) || 0); const m = Math.floor(t / 60); return `${m}:${(t - m * 60).toFixed(2).padStart(5, '0')}` }
|
|
64
|
+
|
|
65
|
+
/** The image block + a text block, the file capped at MAX_SIDE. */
|
|
66
|
+
async function picture (file, text) {
|
|
67
|
+
await run('convert', [file, '-resize', `${MAX_SIDE}x${MAX_SIDE}>`, file])
|
|
68
|
+
const data = readFileSync(file).toString('base64')
|
|
69
|
+
return [{ type: 'image', data, mimeType: 'image/png' }, { type: 'text', text: `${text}\nSaved at ${file}` }]
|
|
70
|
+
}
|
|
71
|
+
async function sizeOf (file) {
|
|
72
|
+
const { stdout } = await run('identify', ['-format', '%w %h\n', file + '[0]'])
|
|
73
|
+
const [w, h] = stdout.trim().split('\n')[0].split(' ').map(Number)
|
|
74
|
+
return { w, h }
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// ── images ─────────────────────────────────────────────────────────────────────────────────────────────────────────────
|
|
78
|
+
async function imageView (a) {
|
|
79
|
+
const src = inputFile(a.path, 'path')
|
|
80
|
+
const box = boxOf(a.box)
|
|
81
|
+
const scale = num(a.scale, 0.05, 16, 'scale')
|
|
82
|
+
const rotate = num(a.rotate, -360, 360, 'rotate')
|
|
83
|
+
const e = a.enhance && typeof a.enhance === 'object' ? a.enhance : {}
|
|
84
|
+
const args = [src + '[0]', '-auto-orient']
|
|
85
|
+
const did = []
|
|
86
|
+
if (box) { args.push('-crop', geom(box), '+repage'); did.push(`crop ${geom(box)}`) }
|
|
87
|
+
if (Array.isArray(a.perspective)) {
|
|
88
|
+
const pts = a.perspective
|
|
89
|
+
if (pts.length !== 4 || !pts.every((p) => Array.isArray(p) && p.length === 2)) fail('perspective must be four [x, y] corners: top-left, top-right, bottom-right, bottom-left')
|
|
90
|
+
const c = pts.map(([x, y]) => [num(x, 0, 100000, 'perspective'), num(y, 0, 100000, 'perspective')])
|
|
91
|
+
const w = Math.round(Math.max(Math.hypot(c[1][0] - c[0][0], c[1][1] - c[0][1]), Math.hypot(c[2][0] - c[3][0], c[2][1] - c[3][1])))
|
|
92
|
+
const h = Math.round(Math.max(Math.hypot(c[3][0] - c[0][0], c[3][1] - c[0][1]), Math.hypot(c[2][0] - c[1][0], c[2][1] - c[1][1])))
|
|
93
|
+
const map = [[c[0], [0, 0]], [c[1], [w, 0]], [c[2], [w, h]], [c[3], [0, h]]].map(([s, d]) => `${s[0]},${s[1]} ${d[0]},${d[1]}`).join(' ')
|
|
94
|
+
args.push('-distort', 'Perspective', map, '-crop', `${w}x${h}+0+0`, '+repage'); did.push('perspective-corrected')
|
|
95
|
+
}
|
|
96
|
+
if (a.deskew) { args.push('-deskew', '40%', '+repage'); did.push('deskewed') }
|
|
97
|
+
if (rotate) { args.push('-background', 'white', '-rotate', String(rotate)); did.push(`rotated ${rotate}°`) }
|
|
98
|
+
if (a.flip === 'horizontal') { args.push('-flop'); did.push('flipped horizontally') } else if (a.flip === 'vertical') { args.push('-flip'); did.push('flipped vertically') } else if (a.flip !== undefined) fail('flip is "horizontal" or "vertical"')
|
|
99
|
+
if (a.channel) { const ch = { red: 'R', green: 'G', blue: 'B', alpha: 'A' }[a.channel]; if (!ch) fail('channel is red, green, blue or alpha'); args.push('-alpha', ch === 'A' ? 'extract' : 'off', ...(ch === 'A' ? [] : ['-channel', ch, '-separate'])); did.push(`${a.channel} channel`) }
|
|
100
|
+
if (e.grayscale) { args.push('-colorspace', 'Gray'); did.push('grayscale') }
|
|
101
|
+
if (e.normalize) { args.push('-normalize'); did.push('normalized') }
|
|
102
|
+
const bri = num(e.brightness, -100, 100, 'enhance.brightness'); const con = num(e.contrast, -100, 100, 'enhance.contrast')
|
|
103
|
+
if (bri || con) { args.push('-brightness-contrast', `${bri || 0}x${con || 0}`); did.push(`brightness ${bri || 0} contrast ${con || 0}`) }
|
|
104
|
+
const gamma = num(e.gamma, 0.1, 5, 'enhance.gamma'); if (gamma) { args.push('-gamma', String(gamma)); did.push(`gamma ${gamma}`) }
|
|
105
|
+
if (e.denoise) { args.push('-despeckle'); did.push('denoised') }
|
|
106
|
+
if (e.sharpen) { args.push('-unsharp', '0x1.5+1.2+0.02'); did.push('sharpened') }
|
|
107
|
+
if (e.invert) { args.push('-negate'); did.push('inverted') }
|
|
108
|
+
if (e.threshold !== undefined) { const t = num(e.threshold, 0, 100, 'enhance.threshold'); args.push('-colorspace', 'Gray', '-threshold', `${t}%`); did.push(`threshold ${t}%`) }
|
|
109
|
+
if (e.adaptive_threshold) { args.push('-colorspace', 'Gray', '-lat', '25x25-5%'); did.push('adaptive threshold') }
|
|
110
|
+
if (e.edges) { args.push('-colorspace', 'Gray', '-canny', '0x1+10%+30%'); did.push('edges (Canny)') }
|
|
111
|
+
if (scale && scale !== 1) { args.push('-filter', 'Lanczos', '-resize', `${scale * 100}%`); did.push(`scaled ×${scale}`) }
|
|
112
|
+
const w = int(a.width, 1, 20000, 'width'), h = int(a.height, 1, 20000, 'height')
|
|
113
|
+
if (w || h) { args.push('-resize', `${w || ''}x${h || ''}`); did.push(`resized to ${w || 'auto'}×${h || 'auto'}`) }
|
|
114
|
+
const pad = int(a.pad, 1, 2000, 'pad'); if (pad) { args.push('-bordercolor', 'white', '-border', String(pad)); did.push(`padded ${pad}px`) }
|
|
115
|
+
if (a.grid) { const step = int(a.grid, 10, 2000, 'grid'); did.push(`grid every ${step}px`); return gridOverlay(args, step, did, src) }
|
|
116
|
+
const out = outFile('png')
|
|
117
|
+
await run('convert', [...args, out])
|
|
118
|
+
const before = await sizeOf(src), after = await sizeOf(out)
|
|
119
|
+
return picture(out, `${basename(src)}: ${before.w}×${before.h} → ${after.w}×${after.h}${did.length ? ' · ' + did.join(' · ') : ''}`)
|
|
120
|
+
}
|
|
121
|
+
async function gridOverlay (args, step, did, src) {
|
|
122
|
+
const base = outFile('png'); await run('convert', [...args, base])
|
|
123
|
+
const { w, h } = await sizeOf(base)
|
|
124
|
+
const lines = [], labels = []
|
|
125
|
+
for (let x = 0; x <= w; x += step) { lines.push('-draw', `line ${x},0 ${x},${h}`); labels.push('-draw', `text ${x + 2},11 '${x}'`) }
|
|
126
|
+
for (let y = step; y <= h; y += step) { lines.push('-draw', `line 0,${y} ${w},${y}`); labels.push('-draw', `text 2,${y - 2} '${y}'`) }
|
|
127
|
+
const out = outFile('png')
|
|
128
|
+
await run('convert', [base, '-fill', 'none', '-stroke', 'rgba(255,0,255,0.6)', '-strokewidth', '1', ...lines, '-stroke', 'none', '-fill', 'magenta', '-pointsize', '11', ...labels, out])
|
|
129
|
+
rmSync(base, { force: true })
|
|
130
|
+
return picture(out, `${basename(src)}: ${w}×${h} · ${did.join(' · ')} (labels are pixel coordinates of THIS image)`)
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
async function imageInfo (a) {
|
|
134
|
+
const src = inputFile(a.path, 'path')
|
|
135
|
+
const { stdout } = await run('identify', ['-format', '%m|%w|%h|%[colorspace]|%z|%A|%[channels]|%n\n', src])
|
|
136
|
+
const first = stdout.trim().split('\n')[0].split('|')
|
|
137
|
+
const frames = stdout.trim().split('\n').length
|
|
138
|
+
const stats = (await run('convert', [src + '[0]', '-colorspace', 'sRGB', '-format', '%[fx:mean.r*255]|%[fx:mean.g*255]|%[fx:mean.b*255]|%[fx:standard_deviation*255]|%[fx:minima*255]|%[fx:maxima*255]', 'info:'])).stdout.split('|').map((v) => Math.round(Number(v)))
|
|
139
|
+
const exif = (await run('identify', ['-format', '%[EXIF:*]', src + '[0]'], { ok: [0, 1] }).catch(() => ({ stdout: '' }))).stdout.trim().split('\n').filter(Boolean).map((l) => l.replace(/^exif:/, '')).slice(0, 40)
|
|
140
|
+
const bytes = statSync(src).size
|
|
141
|
+
const lines = [
|
|
142
|
+
`${basename(src)}: ${first[0]} · ${first[1]}×${first[2]} px · ${first[3]} · ${first[4]}-bit · ${first[6]}${first[5] === 'True' || /a/i.test(first[6]) ? ' · has alpha' : ''} · ${frames} frame(s) · ${bytes} bytes`,
|
|
143
|
+
`mean RGB ${stats[0]}, ${stats[1]}, ${stats[2]} · spread ${stats[3]} · min ${stats[4]} · max ${stats[5]}${stats[5] - stats[4] < 128 ? ' (low contrast: try enhance.normalize)' : ''}`,
|
|
144
|
+
exif.length ? 'EXIF:\n ' + exif.join('\n ') : 'no EXIF',
|
|
145
|
+
]
|
|
146
|
+
return [{ type: 'text', text: lines.join('\n') }]
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
async function imageMeasure (a) {
|
|
150
|
+
const src = inputFile(a.path, 'path')
|
|
151
|
+
const pts = Array.isArray(a.points) ? a.points : []
|
|
152
|
+
if (pts.length > 50) fail('at most 50 points')
|
|
153
|
+
const out = []
|
|
154
|
+
const P = pts.map((p, i) => { if (!Array.isArray(p) || p.length !== 2) fail(`points[${i}] must be [x, y]`); return [int(p[0], 0, 100000, 'x'), int(p[1], 0, 100000, 'y')] })
|
|
155
|
+
for (const [x, y] of P) {
|
|
156
|
+
const { stdout } = await run('convert', [src + '[0]', '-crop', `1x1+${x}+${y}`, '+repage', '-format', '%[hex:u.p{0,0}] %[pixel:u.p{0,0}]', 'info:'])
|
|
157
|
+
out.push(`(${x}, ${y}): #${stdout.trim().split(' ')[0].slice(0, 6)} ${stdout.trim().split(' ').slice(1).join(' ')}`)
|
|
158
|
+
}
|
|
159
|
+
for (let i = 1; i < P.length; i++) {
|
|
160
|
+
const [dx, dy] = [P[i][0] - P[i - 1][0], P[i][1] - P[i - 1][1]]
|
|
161
|
+
out.push(`(${P[i - 1]}) → (${P[i]}): ${Math.hypot(dx, dy).toFixed(1)} px at ${(Math.atan2(-dy, dx) * 180 / Math.PI).toFixed(1)}° (0° = right, 90° = up)`)
|
|
162
|
+
}
|
|
163
|
+
for (let i = 2; i < P.length; i++) {
|
|
164
|
+
const v1 = [P[i - 2][0] - P[i - 1][0], P[i - 2][1] - P[i - 1][1]], v2 = [P[i][0] - P[i - 1][0], P[i][1] - P[i - 1][1]]
|
|
165
|
+
const ang = Math.acos(Math.max(-1, Math.min(1, (v1[0] * v2[0] + v1[1] * v2[1]) / (Math.hypot(...v1) * Math.hypot(...v2) || 1)))) * 180 / Math.PI
|
|
166
|
+
out.push(`angle at (${P[i - 1]}): ${ang.toFixed(1)}°`)
|
|
167
|
+
}
|
|
168
|
+
const box = boxOf(a.box)
|
|
169
|
+
if (box) {
|
|
170
|
+
const { stdout } = await run('convert', [src + '[0]', '-crop', geom(box), '+repage', '-colorspace', 'sRGB', '-format', '%[fx:mean.r*255] %[fx:mean.g*255] %[fx:mean.b*255]', 'info:'])
|
|
171
|
+
const m = stdout.trim().split(' ').map((v) => Math.round(Number(v)))
|
|
172
|
+
out.push(`box ${geom(box)}: ${box.w * box.h} px², mean RGB ${m.join(', ')}`)
|
|
173
|
+
}
|
|
174
|
+
if (a.regions) { // connected components: count and bound the distinct dark/light regions (IM ≥ 6.8.9)
|
|
175
|
+
const min = int(a.regions.min_area, 1, 1e9, 'regions.min_area') || 20
|
|
176
|
+
const { stdout } = await run('convert', [src + '[0]', '-colorspace', 'Gray', '-threshold', `${num(a.regions.threshold, 0, 100, 'regions.threshold') ?? 50}%`, ...(a.regions.dark === false ? [] : ['-negate']), '-define', 'connected-components:verbose=true', '-define', `connected-components:area-threshold=${min}`, '-connected-components', '8', 'null:'])
|
|
177
|
+
const rows = stdout.trim().split('\n').slice(1).map((l) => l.trim()).filter((l) => /gray\(255\)|#FFFFFF|white/i.test(l)).slice(0, 100)
|
|
178
|
+
out.push(`${rows.length} region(s) of at least ${min} px² (id: WxH+X+Y centroid area):\n ` + rows.join('\n '))
|
|
179
|
+
}
|
|
180
|
+
if (!out.length) fail('give points (colors, distances, angles), a box (area, mean color) or regions (connected components)')
|
|
181
|
+
return [{ type: 'text', text: `${basename(src)}\n` + out.join('\n') }]
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
async function imageOcr (a) {
|
|
185
|
+
const orig = inputFile(a.path, 'path')
|
|
186
|
+
const box = boxOf(a.box)
|
|
187
|
+
const k = num(a.scale, 1, 8, 'scale') || 1
|
|
188
|
+
// ALWAYS a grayscale copy: Tesseract's layout step read a colored page as "Empty page!!" and found nothing (measured on
|
|
189
|
+
// text on a red/blue background, 2026-10-06); in grayscale it read it at once. Positions are mapped back to the original.
|
|
190
|
+
const src = outFile('png')
|
|
191
|
+
await run('convert', [orig + '[0]', '-auto-orient', ...(box ? ['-crop', geom(box), '+repage'] : []), '-colorspace', 'Gray', ...(k !== 1 ? ['-resize', `${k * 100}%`] : []), src])
|
|
192
|
+
const lang = typeof a.lang === 'string' && /^[a-z_+]{3,40}$/.test(a.lang) ? a.lang : 'eng'
|
|
193
|
+
const { stdout } = await run('tesseract', [src, 'stdout', '-l', lang, '--psm', String(int(a.psm, 0, 13, 'psm') ?? 3), 'tsv'])
|
|
194
|
+
const rows = stdout.split('\n').slice(1).map((l) => l.split('\t')).filter((r) => r.length >= 12 && Number(r[10]) >= 0 && r[11].trim())
|
|
195
|
+
const back = (v, off) => Math.round(v / k) + off // from the enlarged crop back to the ORIGINAL image's pixels
|
|
196
|
+
const words = rows.map((r) => ({ line: `${r[2]}.${r[3]}.${r[4]}`, x: back(+r[6], box ? box.x : 0), y: back(+r[7], box ? box.y : 0), w: Math.round(+r[8] / k), h: Math.round(+r[9] / k), conf: Math.round(+r[10]), text: r[11] }))
|
|
197
|
+
rmSync(src, { force: true })
|
|
198
|
+
const lines = new Map(); for (const w of words) { if (!lines.has(w.line)) lines.set(w.line, []); lines.get(w.line).push(w) }
|
|
199
|
+
const text = [...lines.values()].map((ws) => { const x = Math.min(...ws.map((w) => w.x)), y = Math.min(...ws.map((w) => w.y)), x2 = Math.max(...ws.map((w) => w.x + w.w)), y2 = Math.max(...ws.map((w) => w.y + w.h)); return `[${x},${y},${x2 - x},${y2 - y}] ${ws.map((w) => w.text).join(' ')}` })
|
|
200
|
+
const low = words.filter((w) => w.conf < 60).map((w) => `${w.text}(${w.conf})`).slice(0, 30)
|
|
201
|
+
return [{ type: 'text', text: `${words.length} word(s), ${lines.size} line(s) — [x,y,w,h] in the ORIGINAL image's pixels\n${text.join('\n') || '(no text found — try enhance in image_view, a box, or scale 2–3)'}${low.length ? `\nlow confidence: ${low.join(' ')}` : ''}` }]
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
async function imageDiff (a) {
|
|
205
|
+
const A = inputFile(a.a, 'a'), B0 = inputFile(a.b, 'b')
|
|
206
|
+
const sa = await sizeOf(A), sb = await sizeOf(B0)
|
|
207
|
+
let B = B0, note = ''
|
|
208
|
+
if (sa.w !== sb.w || sa.h !== sb.h) { B = outFile('png'); await run('convert', [B0 + '[0]', '-resize', `${sa.w}x${sa.h}!`, B]); note = ` (b was ${sb.w}×${sb.h}, resized to compare)` }
|
|
209
|
+
const fuzz = num(a.fuzz, 0, 100, 'fuzz') ?? 3
|
|
210
|
+
const heat = outFile('png')
|
|
211
|
+
const ae = await run('compare', ['-metric', 'AE', '-fuzz', `${fuzz}%`, A + '[0]', B + '[0]', heat], { ok: [0, 1] })
|
|
212
|
+
const changed = Number(String(ae.stderr).trim().split(/\s/)[0]) || 0
|
|
213
|
+
const rmse = await run('compare', ['-metric', 'RMSE', A + '[0]', B + '[0]', 'null:'], { ok: [0, 1] })
|
|
214
|
+
const sim = 1 - (Number((String(rmse.stderr).match(/\(([\d.e-]+)\)/) || [])[1]) || 0)
|
|
215
|
+
let where = 'nothing changed'
|
|
216
|
+
if (changed) { const bb = await run('convert', [A + '[0]', B + '[0]', '-compose', 'difference', '-composite', '-colorspace', 'Gray', '-threshold', `${fuzz}%`, '-format', '%@', 'info:']); where = `changes within ${bb.stdout.trim()}` }
|
|
217
|
+
return picture(heat, `${changed} of ${sa.w * sa.h} pixels differ by more than ${fuzz}% (${(100 * changed / (sa.w * sa.h)).toFixed(2)}%) · similarity ${sim.toFixed(4)} · ${where}${note}\nThe picture is A with the differing pixels in red.`)
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// ── PDFs ───────────────────────────────────────────────────────────────────────────────────────────────────────────────
|
|
221
|
+
async function pdfView (a) {
|
|
222
|
+
const src = inputFile(a.path, 'path')
|
|
223
|
+
const info = (await run('pdfinfo', [src])).stdout
|
|
224
|
+
const pages = Number((info.match(/^Pages:\s+(\d+)/m) || [])[1]) || 1
|
|
225
|
+
const meta = info.split('\n').filter((l) => /^(Title|Author|Producer|Pages|Page size|Encrypted|PDF version):/.test(l)).map((l) => l.replace(/\s+/g, ' ')).join(' · ')
|
|
226
|
+
if (a.overview) {
|
|
227
|
+
const n = Math.min(pages, 24), dir = outFile('d'); mkdirSync(dir, { recursive: true })
|
|
228
|
+
await run('pdftoppm', ['-f', '1', '-l', String(n), '-r', '40', '-png', src, join(dir, 'p')])
|
|
229
|
+
const files = readdirSync(dir).filter((f) => f.endsWith('.png')).sort().map((f) => join(dir, f))
|
|
230
|
+
const out = outFile('png')
|
|
231
|
+
await run('montage', [...files.flatMap((f, i) => ['-label', `p${i + 1}`, f]), '-tile', '6x', '-geometry', '+6+6', '-background', '#eee', out])
|
|
232
|
+
rmSync(dir, { recursive: true, force: true })
|
|
233
|
+
return picture(out, `${basename(src)}: ${meta}\nOverview of pages 1–${n}${pages > n ? ` of ${pages}` : ''}. Look closer with page (+ box, dpi).`)
|
|
234
|
+
}
|
|
235
|
+
const page = int(a.page, 1, pages, 'page') ?? 1
|
|
236
|
+
const dpi = int(a.dpi, 30, 600, 'dpi') ?? 110
|
|
237
|
+
const box = boxOf(a.box)
|
|
238
|
+
const base = outFile('x')
|
|
239
|
+
await run('pdftoppm', ['-f', String(page), '-l', String(page), '-r', String(dpi), '-png', '-singlefile', ...(box ? ['-x', String(box.x), '-y', String(box.y), '-W', String(box.w), '-H', String(box.h)] : []), src, base])
|
|
240
|
+
const text = a.text === false ? '' : (await run('pdftotext', ['-f', String(page), '-l', String(page), '-layout', src, '-'])).stdout.trim().slice(0, 4000)
|
|
241
|
+
return picture(base + '.png', `${basename(src)} page ${page}/${pages} at ${dpi} dpi${box ? ` · region ${geom(box)} (pixels at this dpi)` : ''} · ${meta}${text ? `\nEmbedded text of the page (compare with what you see):\n${text}` : '\n(no embedded text — scanned? use image_ocr on the saved page)'}`)
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// ── video ──────────────────────────────────────────────────────────────────────────────────────────────────────────────
|
|
245
|
+
async function probe (src) {
|
|
246
|
+
const j = JSON.parse((await run('ffprobe', ['-v', 'error', '-print_format', 'json', '-show_format', '-show_streams', src])).stdout)
|
|
247
|
+
const v = (j.streams || []).find((s) => s.codec_type === 'video'), au = (j.streams || []).filter((s) => s.codec_type === 'audio'), sub = (j.streams || []).filter((s) => s.codec_type === 'subtitle')
|
|
248
|
+
const dur = Number(j.format && j.format.duration) || Number(v && v.duration) || 0
|
|
249
|
+
const fps = v && v.avg_frame_rate && v.avg_frame_rate.includes('/') ? (([n, d]) => (+d ? +n / +d : 0))(v.avg_frame_rate.split('/')) : 0
|
|
250
|
+
const summary = [`duration ${dur.toFixed(2)} s`, v ? `video ${v.codec_name} ${v.width}×${v.height} @ ${fps.toFixed(2)} fps` : 'no video', ...au.map((x) => `audio ${x.codec_name} ${x.channels}ch ${x.sample_rate} Hz${x.tags && x.tags.language ? ' ' + x.tags.language : ''}`), ...sub.map((x, i) => `subtitles #${i} ${x.codec_name}${x.tags && x.tags.language ? ' ' + x.tags.language : ''}`), `${Math.round((Number(j.format && j.format.size) || 0) / 1024)} KB`]
|
|
251
|
+
return { dur, fps, v, au, sub, summary: summary.join(' · ') }
|
|
252
|
+
}
|
|
253
|
+
async function frameAt (src, t, box, scale) {
|
|
254
|
+
const f = outFile('png')
|
|
255
|
+
const vf = [...(box ? [`crop=${box.w}:${box.h}:${box.x}:${box.y}`] : []), ...(scale && scale !== 1 ? [`scale=iw*${scale}:-2:flags=lanczos`] : [])]
|
|
256
|
+
await run('ffmpeg', ['-v', 'error', '-ss', String(t), '-i', src, '-frames:v', '1', ...(vf.length ? ['-vf', vf.join(',')] : []), '-y', f])
|
|
257
|
+
if (!existsSync(f)) fail(`no frame at ${ts(t)}`)
|
|
258
|
+
return f
|
|
259
|
+
}
|
|
260
|
+
async function sheet (frames, cols, label) {
|
|
261
|
+
const out = outFile('png')
|
|
262
|
+
await run('montage', [...frames.flatMap(([f, t]) => ['-label', label(t), f]), '-tile', `${cols}x`, '-geometry', '+4+4', '-pointsize', '14', '-background', '#eee', out])
|
|
263
|
+
return out
|
|
264
|
+
}
|
|
265
|
+
async function videoOverview (a) {
|
|
266
|
+
const src = inputFile(a.path, 'path')
|
|
267
|
+
const p = await probe(src)
|
|
268
|
+
if (!p.v) return [{ type: 'text', text: `${basename(src)}: ${p.summary}` }]
|
|
269
|
+
const n = int(a.frames, 4, MAX_FRAMES, 'frames') ?? 16
|
|
270
|
+
const times = Array.from({ length: n }, (_, i) => (p.dur * (i + 0.5)) / n)
|
|
271
|
+
const frames = []; for (const t of times) frames.push([await frameAt(src, t, null, 320 / Math.max(p.v.width, 1)), t])
|
|
272
|
+
let scenes = ''
|
|
273
|
+
if (a.scenes !== false && p.dur <= 1800) {
|
|
274
|
+
const thr = num(a.scene_threshold, 0.05, 0.9, 'scene_threshold') ?? 0.3
|
|
275
|
+
const r = await run('ffmpeg', ['-v', 'info', '-i', src, '-vf', `select='gt(scene,${thr})',showinfo`, '-an', '-f', 'null', '-'], { ok: [0, 1] })
|
|
276
|
+
const cuts = [...r.stderr.matchAll(/pts_time:([\d.]+)/g)].map((m) => Number(m[1])).slice(0, 60)
|
|
277
|
+
const bounds = [0, ...cuts, p.dur]
|
|
278
|
+
scenes = `\n${cuts.length + 1} scene(s): ` + bounds.slice(0, -1).map((s, i) => `${ts(s)}–${ts(bounds[i + 1])}`).join(' · ')
|
|
279
|
+
}
|
|
280
|
+
const out = await sheet(frames, int(a.columns, 2, 8, 'columns') ?? 4, ts)
|
|
281
|
+
frames.forEach(([f]) => rmSync(f, { force: true }))
|
|
282
|
+
return picture(out, `${basename(src)}: ${p.summary}${scenes}\nContact sheet: ${n} frames, evenly spaced, labelled m:ss. Look closer with video_frames.`)
|
|
283
|
+
}
|
|
284
|
+
async function videoFrames (a) {
|
|
285
|
+
const src = inputFile(a.path, 'path')
|
|
286
|
+
const p = await probe(src)
|
|
287
|
+
if (!p.v) fail('no video stream')
|
|
288
|
+
const box = boxOf(a.box), scale = num(a.scale, 0.05, 8, 'scale')
|
|
289
|
+
if (a.at !== undefined) {
|
|
290
|
+
const t = num(a.at, 0, p.dur, 'at')
|
|
291
|
+
const f = await frameAt(src, t, box, scale)
|
|
292
|
+
return picture(f, `${basename(src)} at ${ts(t)} (${t.toFixed(3)} s)${box ? ` · region ${geom(box)}` : ''}${scale ? ` · ×${scale}` : ''}`)
|
|
293
|
+
}
|
|
294
|
+
const start = num(a.start, 0, p.dur, 'start') ?? 0, end = num(a.end, 0, p.dur, 'end') ?? p.dur
|
|
295
|
+
if (end <= start) fail('end must be after start')
|
|
296
|
+
const fps = num(a.fps, 0.01, 60, 'fps') ?? Math.min(MAX_FRAMES / (end - start), Math.max(p.fps, 1))
|
|
297
|
+
const n = Math.min(MAX_FRAMES, Math.max(1, Math.floor((end - start) * fps)))
|
|
298
|
+
const frames = []; for (let i = 0; i < n; i++) { const t = start + i / fps; frames.push([await frameAt(src, t, box, scale || (box ? 1 : 320 / Math.max(p.v.width, 1))), t]) }
|
|
299
|
+
let clip = ''
|
|
300
|
+
if (a.save_clip) { const c = outFile('mp4'); await run('ffmpeg', ['-v', 'error', '-ss', String(start), '-to', String(end), '-i', src, ...(box ? ['-vf', `crop=${box.w}:${box.h}:${box.x}:${box.y}`] : []), '-c:a', 'copy', '-y', c]); clip = `\nClip saved at ${c}` }
|
|
301
|
+
const out = await sheet(frames, int(a.columns, 2, 10, 'columns') ?? Math.min(6, n), (t) => `${t.toFixed(2)}s`)
|
|
302
|
+
frames.forEach(([f]) => rmSync(f, { force: true }))
|
|
303
|
+
return picture(out, `${basename(src)} ${ts(start)}–${ts(end)}: ${n} frame(s) at ${fps.toFixed(2)} fps${box ? ` · region ${geom(box)}` : ''}, labelled in seconds${clip}`)
|
|
304
|
+
}
|
|
305
|
+
async function videoDiff (a) {
|
|
306
|
+
const src = inputFile(a.path, 'path')
|
|
307
|
+
const p = await probe(src)
|
|
308
|
+
const t1 = num(a.t1, 0, p.dur, 't1'), t2 = num(a.t2, 0, p.dur, 't2')
|
|
309
|
+
if (t1 === undefined || t2 === undefined) fail('give t1 and t2 in seconds')
|
|
310
|
+
const box = boxOf(a.box)
|
|
311
|
+
const f1 = await frameAt(src, t1, box), f2 = await frameAt(src, t2, box)
|
|
312
|
+
const r = await imageDiff({ a: f1, b: f2, fuzz: a.fuzz })
|
|
313
|
+
r[1].text = `${basename(src)}: ${ts(t1)} vs ${ts(t2)}${box ? ` · region ${geom(box)}` : ''}\n` + r[1].text
|
|
314
|
+
return r
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
// ── audio ──────────────────────────────────────────────────────────────────────────────────────────────────────────────
|
|
318
|
+
async function audio (a) {
|
|
319
|
+
const src = inputFile(a.path, 'path')
|
|
320
|
+
const p = await probe(src)
|
|
321
|
+
const start = num(a.start, 0, p.dur || 1e9, 'start'), end = num(a.end, 0, p.dur || 1e9, 'end')
|
|
322
|
+
const range = [...(start !== undefined ? ['-ss', String(start)] : []), ...(end !== undefined ? ['-to', String(end)] : [])]
|
|
323
|
+
if (a.op === 'extract') {
|
|
324
|
+
if (!p.au.length) fail('no audio stream')
|
|
325
|
+
const out = outFile('wav'); await run('ffmpeg', ['-v', 'error', ...range, '-i', src, '-vn', '-ac', '1', '-ar', '16000', '-y', out])
|
|
326
|
+
return [{ type: 'text', text: `${basename(src)}: ${p.summary}\nAudio (16 kHz mono WAV) saved at ${out}` }]
|
|
327
|
+
}
|
|
328
|
+
if (a.op === 'silence') {
|
|
329
|
+
if (!p.au.length) fail('no audio stream')
|
|
330
|
+
const db = num(a.noise_db, -90, 0, 'noise_db') ?? -35, d = num(a.min_seconds, 0.1, 60, 'min_seconds') ?? 0.5
|
|
331
|
+
const r = await run('ffmpeg', ['-v', 'info', ...range, '-i', src, '-af', `silencedetect=noise=${db}dB:d=${d}`, '-f', 'null', '-'], { ok: [0, 1] })
|
|
332
|
+
const st = [...r.stderr.matchAll(/silence_start: ([\d.]+)/g)].map((m) => +m[1]), en = [...r.stderr.matchAll(/silence_end: ([\d.]+)/g)].map((m) => +m[1])
|
|
333
|
+
return [{ type: 'text', text: `${basename(src)}: ${st.length} silence(s) quieter than ${db} dB for ≥ ${d} s\n` + (st.map((s, i) => `${ts(s)}–${en[i] !== undefined ? ts(en[i]) : 'end'}`).join(' · ') || '(none)') }]
|
|
334
|
+
}
|
|
335
|
+
if (a.op === 'subtitles') {
|
|
336
|
+
if (!p.sub.length) return [{ type: 'text', text: `${basename(src)}: no subtitle streams` }]
|
|
337
|
+
const i = int(a.stream, 0, p.sub.length - 1, 'stream') ?? 0
|
|
338
|
+
const r = await run('ffmpeg', ['-v', 'error', '-i', src, '-map', `0:s:${i}`, '-f', 'srt', '-'])
|
|
339
|
+
return [{ type: 'text', text: `${basename(src)} subtitles #${i}:\n${String(r.stdout).slice(0, 20000)}` }]
|
|
340
|
+
}
|
|
341
|
+
if (a.op === 'transcribe') {
|
|
342
|
+
if (!p.au.length) fail('no audio stream')
|
|
343
|
+
return transcribe(src, p, a, range, start || 0)
|
|
344
|
+
}
|
|
345
|
+
if (a.op === 'info' || a.op === undefined) return [{ type: 'text', text: `${basename(src)}: ${p.summary}` }]
|
|
346
|
+
fail('op is info, extract, silence, subtitles or transcribe')
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
// Speech to text ON THIS COMPUTER with whisper.cpp — the audio never leaves the machine (docs/media-mcp.md § Transcription).
|
|
350
|
+
// The program: WITBITZ_WHISPER_BIN, else whisper-cli on PATH, else the pinned build fetched once into WITBITZ_WHISPER_DIR
|
|
351
|
+
// (default ~/.cache/witbitz/whisper). The model: WITBITZ_WHISPER_MODEL, else the first of ggml-{small,base,tiny}.bin in that
|
|
352
|
+
// dir, else ggml-base.bin fetched once. Every download is pinned by SHA-256 and refused on a mismatch; WITBITZ_WHISPER_FETCH=0
|
|
353
|
+
// turns fetching off. Measured 2026-10-06 (v1.9.4, 8 threads, x86): base reads 66 s of speech in 2.8 s, tiny in 1.9 s.
|
|
354
|
+
const WHISPER_DIR = process.env.WITBITZ_WHISPER_DIR || join(homedir(), '.cache', 'witbitz', 'whisper')
|
|
355
|
+
// whisper.cpp's own release builds of v1.9.4 (release b5130, 2026-09-11). macOS has no prebuilt program there.
|
|
356
|
+
export const WHISPER_BUILD = 'b5130'
|
|
357
|
+
export const WHISPER_ASSETS = {
|
|
358
|
+
'linux-x64': { name: 'whisper-bin-ubuntu-x64.tar.gz', sha256: '53e7fd8b5764edad916b8848dd0af6abb1ff1d3b86c899e79c78652412536c32' },
|
|
359
|
+
'linux-arm64': { name: 'whisper-bin-ubuntu-arm64.tar.gz', sha256: '93532a0e3777f26f041ffa358ee77dd88b1a33a86847c1990745327ff335a5d6' },
|
|
360
|
+
'win32-x64': { name: 'whisper-bin-x64.zip', sha256: 'f9ec6c52a2e949b62ab51fa21d0d497958f9e41c3010c157c4e42932d5316f3c' },
|
|
361
|
+
'win32-arm64': { name: 'whisper-bin-win-cpu-arm64.zip', sha256: '799543b926ab5b6c2d60cab269a2092e0ae8d27820e9e15429e59de3699546fc' }
|
|
362
|
+
}
|
|
363
|
+
// Two models, both whisper.cpp's own (Hugging Face ggerganov/whisper.cpp; the hashes are its x-linked-etag). Measured
|
|
364
|
+
// 2026-10-06 on 60 s of Hebrew (a spoken-Wikipedia article): base garbles about a third of the words; large-v3-turbo q5_0 reads
|
|
365
|
+
// it almost cleanly, with punctuation, at ~2.5× real time on 8 threads (base: ~9×). ivrit.ai's Hebrew-tuned turbo, quantized
|
|
366
|
+
// the same way, was no better on that clip. So English → base, any other language → turbo.
|
|
367
|
+
export const WHISPER_MODEL_PINS = {
|
|
368
|
+
fast: { name: 'ggml-base.bin', bytes: 147951465, sha256: '60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe' },
|
|
369
|
+
accurate: { name: 'ggml-large-v3-turbo-q5_0.bin', bytes: 574041195, sha256: '394221709cd5ad1f40c46e6031ca61bce88931e6e088c188294c6d5a55ffa7e2' }
|
|
370
|
+
}
|
|
371
|
+
export const WHISPER_MODEL_PIN = WHISPER_MODEL_PINS.fast
|
|
372
|
+
const WHISPER_ASSET_URL = (a) => `https://github.com/ggml-org/whisper.cpp/releases/download/${WHISPER_BUILD}/${a.name}`
|
|
373
|
+
const WHISPER_MODEL_URL = (pin) => `https://huggingface.co/ggerganov/whisper.cpp/resolve/main/${pin.name}`
|
|
374
|
+
const WHISPER_MANUAL = 'install whisper.cpp v1.9.4 (macOS: brew install whisper-cpp; elsewhere https://github.com/ggml-org/whisper.cpp) so whisper-cli is on PATH or in WITBITZ_WHISPER_BIN, and put a model (ggml-base.bin) in '
|
|
375
|
+
const MODEL_FILES = { fast: ['small', 'base', 'tiny'], accurate: ['large-v3-turbo-q5_0', 'large-v3-turbo', 'large-v3'] }
|
|
376
|
+
export function whisperModel (env = process.env, dir = WHISPER_DIR, exists = existsSync, quality = 'fast') {
|
|
377
|
+
if (env.WITBITZ_WHISPER_MODEL) return exists(env.WITBITZ_WHISPER_MODEL) ? env.WITBITZ_WHISPER_MODEL : null
|
|
378
|
+
for (const m of MODEL_FILES[quality] || MODEL_FILES.fast) { const f = join(dir, `ggml-${m}.bin`); if (exists(f)) return f }
|
|
379
|
+
return null
|
|
380
|
+
}
|
|
381
|
+
function onPath (cmd, env, platform) {
|
|
382
|
+
const exts = platform === 'win32' ? ['.exe', ''] : ['']
|
|
383
|
+
for (const d of String(env.PATH || '').split(platform === 'win32' ? ';' : ':')) for (const x of exts) { if (d && existsSync(join(d, cmd + x))) return join(d, cmd + x) }
|
|
384
|
+
return null
|
|
385
|
+
}
|
|
386
|
+
function findFile (dir, name, depth = 3) {
|
|
387
|
+
let ents; try { ents = readdirSync(dir, { withFileTypes: true }) } catch { return null }
|
|
388
|
+
for (const e of ents) if (e.isFile() && e.name === name) return join(dir, e.name)
|
|
389
|
+
if (depth > 0) for (const e of ents) if (e.isDirectory()) { const f = findFile(join(dir, e.name), name, depth - 1); if (f) return f }
|
|
390
|
+
return null
|
|
391
|
+
}
|
|
392
|
+
/** Download `url` to `dest`, refusing it unless its SHA-256 is `sha256`. Written beside it first and renamed, so a cut
|
|
393
|
+
* download never looks finished. */
|
|
394
|
+
async function fetchPinned (url, dest, sha256, fetchImpl) {
|
|
395
|
+
const res = await fetchImpl(url, { redirect: 'follow' })
|
|
396
|
+
if (!res.ok || !res.body) fail(`could not download ${url} (${res.status})`)
|
|
397
|
+
const part = `${dest}.${randomUUID().slice(0, 8)}.part`, h = createHash('sha256')
|
|
398
|
+
const out = createWriteStream(part)
|
|
399
|
+
try {
|
|
400
|
+
for await (const chunk of res.body) { h.update(chunk); if (!out.write(chunk)) await new Promise((r) => out.once('drain', r)) }
|
|
401
|
+
await new Promise((r, j) => out.end((e) => (e ? j(e) : r())))
|
|
402
|
+
} catch (e) { out.destroy(); rmSync(part, { force: true }); throw e }
|
|
403
|
+
const got = h.digest('hex')
|
|
404
|
+
if (got !== sha256) { rmSync(part, { force: true }); fail(`refused ${url}: its SHA-256 is ${got}, not the pinned ${sha256}`) }
|
|
405
|
+
renameSync(part, dest)
|
|
406
|
+
}
|
|
407
|
+
const fetching = new Map() // one download per target, shared by calls that arrive while it runs
|
|
408
|
+
const once = (key, fn) => { if (!fetching.has(key)) fetching.set(key, fn().finally(() => fetching.delete(key))); return fetching.get(key) }
|
|
409
|
+
/** → { bin, model, fetched: [] } or a ToolError that says what to install. Everything is injectable for the tests. */
|
|
410
|
+
export async function ensureWhisper ({ env = process.env, dir = WHISPER_DIR, platform = process.platform, arch = process.arch, fetchImpl = globalThis.fetch, assets = WHISPER_ASSETS, quality = 'fast', modelPin = WHISPER_MODEL_PINS[quality] || WHISPER_MODEL_PIN } = {}) {
|
|
411
|
+
const fetched = [], allow = env.WITBITZ_WHISPER_FETCH !== '0'
|
|
412
|
+
const exe = platform === 'win32' ? 'whisper-cli.exe' : 'whisper-cli'
|
|
413
|
+
const home = join(dir, 'bin', WHISPER_BUILD)
|
|
414
|
+
let bin = env.WITBITZ_WHISPER_BIN || onPath('whisper-cli', env, platform) || findFile(home, exe)
|
|
415
|
+
if (!bin) {
|
|
416
|
+
const asset = assets[`${platform}-${arch}`]
|
|
417
|
+
if (!asset || !allow) fail(`whisper-cli is not on this computer${asset ? ' and fetching is off (WITBITZ_WHISPER_FETCH=0)' : ` and whisper.cpp publishes no ${platform}-${arch} build`} — ${WHISPER_MANUAL}${dir}`)
|
|
418
|
+
bin = await once(home, async () => {
|
|
419
|
+
mkdirSync(dir, { recursive: true })
|
|
420
|
+
const arc = join(dir, asset.name), tmp = `${home}.${randomUUID().slice(0, 8)}`
|
|
421
|
+
await fetchPinned(WHISPER_ASSET_URL(asset), arc, asset.sha256, fetchImpl)
|
|
422
|
+
try {
|
|
423
|
+
mkdirSync(tmp, { recursive: true })
|
|
424
|
+
await run('tar', ['-xf', arc, '-C', tmp]) // GNU tar reads the .tar.gz; Windows' tar (bsdtar) reads the .zip
|
|
425
|
+
if (!findFile(tmp, exe)) fail(`${asset.name} has no ${exe}`)
|
|
426
|
+
rmSync(home, { recursive: true, force: true }); renameSync(tmp, home)
|
|
427
|
+
} finally { rmSync(tmp, { recursive: true, force: true }); rmSync(arc, { force: true }) }
|
|
428
|
+
return findFile(home, exe)
|
|
429
|
+
})
|
|
430
|
+
fetched.push(`whisper.cpp ${WHISPER_BUILD} (${asset.name})`)
|
|
431
|
+
}
|
|
432
|
+
let model = whisperModel(env, dir, existsSync, quality)
|
|
433
|
+
if (!model) {
|
|
434
|
+
if (env.WITBITZ_WHISPER_MODEL) fail(`no whisper model at ${env.WITBITZ_WHISPER_MODEL}`)
|
|
435
|
+
if (!allow) fail(`no whisper model in ${dir} and fetching is off (WITBITZ_WHISPER_FETCH=0) — ${WHISPER_MANUAL}${dir}`)
|
|
436
|
+
model = join(dir, modelPin.name)
|
|
437
|
+
await once(model, async () => { mkdirSync(dir, { recursive: true }); await fetchPinned(WHISPER_MODEL_URL(modelPin), model, modelPin.sha256, fetchImpl) })
|
|
438
|
+
fetched.push(`${modelPin.name} (${Math.round(modelPin.bytes / 1e6)} MB)`)
|
|
439
|
+
}
|
|
440
|
+
return { bin, model, fetched }
|
|
441
|
+
}
|
|
442
|
+
const LANG = /^(auto|[a-z]{2,3})$/
|
|
443
|
+
async function transcribe (src, p, a, range, offset) {
|
|
444
|
+
const lang = a.language === undefined ? 'auto' : String(a.language).toLowerCase()
|
|
445
|
+
if (!LANG.test(lang)) fail('language is a code like en, he, fr — or auto')
|
|
446
|
+
if (a.quality !== undefined && !['fast', 'accurate'].includes(a.quality)) fail('quality is fast or accurate')
|
|
447
|
+
// English → the fast model; a named other language → the accurate one; auto → fast first, and if what it heard is not
|
|
448
|
+
// English, once more with the accurate model (the fast pass costs a fraction of the accurate one).
|
|
449
|
+
let quality = a.quality || (lang === 'en' || lang === 'auto' ? 'fast' : 'accurate')
|
|
450
|
+
const wav = outFile('wav'); await run('ffmpeg', ['-v', 'error', ...range, '-i', src, '-vn', '-ac', '1', '-ar', '16000', '-y', wav])
|
|
451
|
+
const fetched = []
|
|
452
|
+
let r
|
|
453
|
+
try {
|
|
454
|
+
r = await whisperPass(wav, lang, quality, a.words, p, offset, fetched)
|
|
455
|
+
if (!a.quality && lang === 'auto' && r.language && r.language !== 'en' && !process.env.WITBITZ_WHISPER_MODEL) { quality = 'accurate'; r = await whisperPass(wav, r.language, quality, a.words, p, offset, fetched) }
|
|
456
|
+
} finally { rmSync(wav, { force: true }) }
|
|
457
|
+
const lines = r.segs.map((x) => `[${ts(x.from)}–${ts(x.to)}] ${x.text}`).join('\n')
|
|
458
|
+
const head = `${basename(src)}: transcribed on this computer (whisper.cpp, ${basename(r.model)}, language ${r.language || lang}) · ${r.segs.length} ${a.words ? 'word' : 'segment'}(s)` +
|
|
459
|
+
(fetched.length ? `\nFirst use: downloaded ${fetched.join(' and ')} into ${dirname(r.model)}, checked against pinned SHA-256s (the audio was not sent anywhere)` : '')
|
|
460
|
+
return [{ type: 'text', text: `${head}\nSubtitles (SRT) saved at ${r.of}.srt\n${lines.slice(0, 20000) || '(no speech)'}${lines.length > 20000 ? '\n… (cut; the full text is in the SRT)' : ''}` }]
|
|
461
|
+
}
|
|
462
|
+
async function whisperPass (wav, lang, quality, words, p, offset, fetched) {
|
|
463
|
+
const { bin, model, fetched: got } = await ensureWhisper({ quality })
|
|
464
|
+
fetched.push(...got)
|
|
465
|
+
const of = outFile('x').slice(0, -2)
|
|
466
|
+
const secs = Math.max(1, (p.dur || 0) - offset)
|
|
467
|
+
const threads = String(Math.max(1, Math.min(8, availableParallelism())))
|
|
468
|
+
await run(bin, ['-m', model, '-f', wav, '-l', lang, '-t', threads, '-np', '-oj', '-osrt', '-of', of, ...(words ? ['-ml', '1', '-sow'] : [])],
|
|
469
|
+
{ timeout: Math.round(Math.max(TIMEOUT, secs * (quality === 'accurate' ? 4000 : 1000))) }) // a slow machine still finishes
|
|
470
|
+
const j = JSON.parse(readFileSync(of + '.json', 'utf8')); rmSync(of + '.json', { force: true })
|
|
471
|
+
const segs = (j.transcription || []).map((x) => ({ from: offset + x.offsets.from / 1000, to: offset + x.offsets.to / 1000, text: String(x.text).trim() })).filter((x) => x.text)
|
|
472
|
+
return { segs, model, of, language: j.result && j.result.language }
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
// ── engines fetched once: ZXing (barcodes), OpenCV (matching, tracking, flow), exiftool (metadata) ─────────────────────
|
|
476
|
+
// Portable builds from npm, never system packages a laptop may not have: ZXing and OpenCV are WebAssembly run by this same
|
|
477
|
+
// node, exiftool is Perl (Windows: its own perl.exe). Each .tgz is pinned by SHA-256 (measured 2026-10-06), fetched the
|
|
478
|
+
// first time a tool needs it into WITBITZ_MEDIA_TOOLS_DIR (default ~/.cache/witbitz/media), and refused on a mismatch.
|
|
479
|
+
// WITBITZ_MEDIA_FETCH=0 turns fetching off; the cloud computer's image carries them pre-placed.
|
|
480
|
+
const ENGINE_DIR = process.env.WITBITZ_MEDIA_TOOLS_DIR || join(homedir(), '.cache', 'witbitz', 'media')
|
|
481
|
+
const npmTgz = (pkg, v) => `https://registry.npmjs.org/${pkg}/-/${pkg.split('/').pop()}-${v}.tgz`
|
|
482
|
+
export const ENGINE_PINS = {
|
|
483
|
+
zxing: { pkg: 'zxing-wasm', version: '3.1.5', sha256: '81e1cd4b01c98b35a30a3f1d16ef38bf338d6e3ada2a0591bd69595a94a07443' },
|
|
484
|
+
opencv: { pkg: '@techstark/opencv-js', version: '5.0.0-release.1', sha256: '5f2289421462489de42444d8ddc59ed035de13458fd8d9d390616f9c53b35434' },
|
|
485
|
+
exiftool: { pkg: 'exiftool-vendored.pl', version: '13.59.3', sha256: 'dea9f92a8748a75a43ff42aae017644a48982aead848f985c6a443ec11e0b8f5' },
|
|
486
|
+
'exiftool-win': { pkg: 'exiftool-vendored.exe', version: '13.59.3', sha256: '7eaf41b3b02ddb9c7bd32bea2417c783d6364a6725cb57d1a0163617aff4b577' },
|
|
487
|
+
}
|
|
488
|
+
/** → the unpacked package directory of engine `key`, fetching it once. */
|
|
489
|
+
export async function ensureEngine (key, { dir = ENGINE_DIR, env = process.env, fetchImpl = globalThis.fetch, pins = ENGINE_PINS } = {}) {
|
|
490
|
+
const pin = pins[key]
|
|
491
|
+
const home = join(dir, `${key}-${pin.version}`), pkg = join(home, 'package')
|
|
492
|
+
if (existsSync(join(pkg, 'package.json'))) return pkg
|
|
493
|
+
if (env.WITBITZ_MEDIA_FETCH === '0') fail(`${pin.pkg} is not on this computer and fetching is off (WITBITZ_MEDIA_FETCH=0) — unpack ${npmTgz(pin.pkg, pin.version)} into ${home}`)
|
|
494
|
+
return once(home, async () => {
|
|
495
|
+
mkdirSync(dir, { recursive: true })
|
|
496
|
+
const tmp = `${home}.${randomUUID().slice(0, 8)}`, tgz = `${tmp}.tgz`
|
|
497
|
+
try {
|
|
498
|
+
await fetchPinned(npmTgz(pin.pkg, pin.version), tgz, pin.sha256, fetchImpl)
|
|
499
|
+
mkdirSync(tmp, { recursive: true })
|
|
500
|
+
await run('tar', ['-xzf', tgz, '-C', tmp])
|
|
501
|
+
if (!existsSync(join(tmp, 'package', 'package.json'))) fail(`${pin.pkg} unpacked to nothing`)
|
|
502
|
+
rmSync(home, { recursive: true, force: true }); renameSync(tmp, home)
|
|
503
|
+
return pkg
|
|
504
|
+
} finally { rmSync(tmp, { recursive: true, force: true }); rmSync(tgz, { force: true }) }
|
|
505
|
+
})
|
|
506
|
+
}
|
|
507
|
+
let _zxing = null, _cv = null
|
|
508
|
+
async function zxing () {
|
|
509
|
+
if (!_zxing) _zxing = (async () => {
|
|
510
|
+
const p = await ensureEngine('zxing')
|
|
511
|
+
const z = await import(pathToFileURL(join(p, 'dist', 'es', 'reader', 'index.js')).href)
|
|
512
|
+
await z.prepareZXingModule({ overrides: { wasmBinary: readFileSync(join(p, 'dist', 'reader', 'zxing_reader.wasm')) }, fireImmediately: true })
|
|
513
|
+
return z
|
|
514
|
+
})().catch((e) => { _zxing = null; throw e })
|
|
515
|
+
return _zxing
|
|
516
|
+
}
|
|
517
|
+
async function opencv () {
|
|
518
|
+
if (!_cv) _cv = (async () => {
|
|
519
|
+
let cv = createRequire(import.meta.url)(join(await ensureEngine('opencv'), 'dist', 'opencv.js'))
|
|
520
|
+
if (cv instanceof Promise) cv = await cv
|
|
521
|
+
else if (!cv.Mat) await new Promise((r) => { cv.onRuntimeInitialized = r })
|
|
522
|
+
return cv
|
|
523
|
+
})().catch((e) => { _cv = null; throw e })
|
|
524
|
+
return _cv
|
|
525
|
+
}
|
|
526
|
+
async function exiftoolCmd () {
|
|
527
|
+
if (process.platform === 'win32') return [join(await ensureEngine('exiftool-win'), 'bin', 'exiftool.exe')]
|
|
528
|
+
const perl = String(process.env.PATH || '').split(':').map((d) => join(d, 'perl')).find((f) => existsSync(f))
|
|
529
|
+
if (!perl) fail('exiftool needs perl, which is not on this computer (install perl)')
|
|
530
|
+
return [perl, join(await ensureEngine('exiftool'), 'bin', 'exiftool')]
|
|
531
|
+
}
|
|
532
|
+
/** An image as RGBA pixels: { w, h, data } (ImageMagick decodes any format; the first frame, upright). */
|
|
533
|
+
async function rgba (file, box) {
|
|
534
|
+
const png = outFile('png'); await run('convert', [file + '[0]', '-auto-orient', ...(box ? ['-crop', geom(box), '+repage'] : []), png])
|
|
535
|
+
const { w, h } = await sizeOf(png)
|
|
536
|
+
const { stdout } = await run('convert', [png, '-depth', '8', 'rgba:-'], { encoding: 'buffer' })
|
|
537
|
+
rmSync(png, { force: true })
|
|
538
|
+
return { w, h, data: stdout }
|
|
539
|
+
}
|
|
540
|
+
const rgbaMat = (cv, img) => cv.matFromArray(img.h, img.w, cv.CV_8UC4, img.data)
|
|
541
|
+
async function drawBoxes (src, boxes, { color = 'red' } = {}) { // boxes: [{ x, y, w, h, label? }] in src's pixels
|
|
542
|
+
const out = outFile('png')
|
|
543
|
+
const rects = boxes.flatMap((b) => ['-draw', `rectangle ${b.x},${b.y} ${b.x + b.w},${b.y + b.h}`])
|
|
544
|
+
const labels = boxes.flatMap((b, i) => ['-draw', `text ${b.x + 2},${Math.max(14, b.y - 4)} '${String(b.label ?? i + 1).replace(/['\\]/g, '')}'`])
|
|
545
|
+
await run('convert', [src + '[0]', '-auto-orient', '-fill', 'none', '-stroke', color, '-strokewidth', '3', ...rects, '-stroke', 'none', '-fill', color, '-pointsize', '16', ...labels, out])
|
|
546
|
+
return out
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
// ── barcodes and QR codes ────────────────────────────────────────────────────────────────────────────────────────────────
|
|
550
|
+
async function imageCodes (a) {
|
|
551
|
+
const src = inputFile(a.path, 'path'), box = boxOf(a.box)
|
|
552
|
+
const png = outFile('png'); await run('convert', [src + '[0]', '-auto-orient', ...(box ? ['-crop', geom(box), '+repage'] : []), png])
|
|
553
|
+
const z = await zxing()
|
|
554
|
+
const formats = Array.isArray(a.formats) ? a.formats.filter((f) => typeof f === 'string' && /^[A-Za-z0-9-]{2,20}$/.test(f)).slice(0, 20) : []
|
|
555
|
+
const found = await z.readBarcodes(new Blob([readFileSync(png)]), { tryHarder: true, tryRotate: true, tryInvert: true, tryDownscale: true, maxNumberOfSymbols: 50, ...(formats.length ? { formats } : {}) })
|
|
556
|
+
rmSync(png, { force: true })
|
|
557
|
+
const ok = found.filter((f) => f.isValid)
|
|
558
|
+
if (!ok.length) return [{ type: 'text', text: `${basename(src)}: no barcode or QR code found${box ? ` in ${geom(box)}` : ''} — try a box around it, or image_view with enhance (normalize, sharpen) and scale 2, then read the saved file` }]
|
|
559
|
+
const dx = box ? box.x : 0, dy = box ? box.y : 0
|
|
560
|
+
const boxes = ok.map((f, i) => { const p = f.position; const xs = [p.topLeft.x, p.topRight.x, p.bottomRight.x, p.bottomLeft.x].map((v) => v + dx), ys = [p.topLeft.y, p.topRight.y, p.bottomRight.y, p.bottomLeft.y].map((v) => v + dy); const x = Math.min(...xs), y = Math.min(...ys); return { x, y, w: Math.max(...xs) - x, h: Math.max(...ys) - y, label: i + 1 } })
|
|
561
|
+
const lines = ok.map((f, i) => `${i + 1}. ${f.format} [${boxes[i].x},${boxes[i].y},${boxes[i].w},${boxes[i].h}]${f.orientation ? ` rotated ${f.orientation}°` : ''}: ${JSON.stringify(f.text).slice(0, 2000)}`)
|
|
562
|
+
return picture(await drawBoxes(src, boxes), `${basename(src)}: ${ok.length} code(s) — [x,y,w,h] in the original image's pixels\n${lines.join('\n')}`)
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
// ── metadata (exiftool) ──────────────────────────────────────────────────────────────────────────────────────────────────
|
|
566
|
+
const TAG = /^[A-Za-z][A-Za-z0-9_-]{0,63}(:[A-Za-z][A-Za-z0-9_-]{0,63})?$/
|
|
567
|
+
async function mediaMeta (a) {
|
|
568
|
+
const src = inputFile(a.path, 'path')
|
|
569
|
+
const ex = await exiftoolCmd(), op = a.op || 'read'
|
|
570
|
+
if (op === 'read') {
|
|
571
|
+
const want = Array.isArray(a.tags) ? a.tags.filter((t) => typeof t === 'string' && TAG.test(t.replace(/:all$/i, ':x'))).slice(0, 50) : []
|
|
572
|
+
const { stdout } = await run(ex[0], [...ex.slice(1), '-json', '-G1', '-s', '-a', '-api', 'LargeFileSupport=1', ...want.map((t) => `-${t}`), src])
|
|
573
|
+
const j = (JSON.parse(stdout)[0]) || {}; delete j.SourceFile
|
|
574
|
+
const rows = Object.entries(j).filter(([k]) => !/^(ExifTool|System):/.test(k) || want.length).map(([k, v]) => `${k}: ${typeof v === 'object' ? JSON.stringify(v) : String(v)}`.slice(0, 500))
|
|
575
|
+
const gps = Object.keys(j).some((k) => /^GPS|:GPS/.test(k))
|
|
576
|
+
return [{ type: 'text', text: `${basename(src)}: ${rows.length} metadata field(s)${gps ? ' · ⚠ contains GPS location' : ''}\n${rows.join('\n').slice(0, 20000) || '(none)'}` }]
|
|
577
|
+
}
|
|
578
|
+
if (op !== 'write' && op !== 'strip') fail('op is read, write or strip')
|
|
579
|
+
const args = []
|
|
580
|
+
if (op === 'write') {
|
|
581
|
+
const tags = a.tags && typeof a.tags === 'object' && !Array.isArray(a.tags) ? Object.entries(a.tags) : []
|
|
582
|
+
if (!tags.length || tags.length > 50) fail('write: give tags as { "Artist": "…", "XMP:Title": "…" } (at most 50)')
|
|
583
|
+
for (const [k, v] of tags) { if (!TAG.test(k)) fail(`write: "${k}" is not a tag name`); if (v !== null && typeof v !== 'string' && typeof v !== 'number') fail(`write: ${k} must be a string, a number or null (to delete it)`); args.push(`-${k}=${v === null ? '' : String(v).slice(0, 4000)}`) }
|
|
584
|
+
} else {
|
|
585
|
+
const what = a.what || 'all'
|
|
586
|
+
const map = { all: ['-all='], gps: ['-gps:all=', '-xmp:geotag='], camera: ['-exif:Make=', '-exif:Model=', '-exif:SerialNumber=', '-exif:LensModel=', '-makernotes:all='], personal: ['-gps:all=', '-xmp:geotag=', '-exif:Artist=', '-exif:Copyright=', '-iptc:all=', '-xmp:Creator=', '-exif:SerialNumber='] }
|
|
587
|
+
if (!map[what]) fail('strip: what is all, gps, camera or personal')
|
|
588
|
+
args.push(...map[what])
|
|
589
|
+
}
|
|
590
|
+
// A COPY by default: the agent's edit should not silently change a user's original photo.
|
|
591
|
+
const target = a.in_place === true ? src : outFile((src.match(/\.([A-Za-z0-9]{1,6})$/) || [, 'bin'])[1].toLowerCase())
|
|
592
|
+
if (target !== src) copyFileSync(src, target)
|
|
593
|
+
const r = await run(ex[0], [...ex.slice(1), '-overwrite_original', '-m', ...args, target], { ok: [0, 1] })
|
|
594
|
+
const said = `${r.stdout} ${r.stderr}`.replace(/\s+/g, ' ').trim()
|
|
595
|
+
if (r.code !== 0 && !/files? updated/.test(said)) fail(`exiftool: ${said.slice(0, 300)}`)
|
|
596
|
+
return [{ type: 'text', text: `${op === 'write' ? 'Wrote' : `Stripped (${a.what || 'all'})`} → ${target === src ? 'the ORIGINAL, in place' : `a copy at ${target} (the original is untouched; in_place: true writes it)`}\nexiftool: ${said.slice(0, 300)}\nCheck with op: read.` }]
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
// ── OpenCV: find an image inside an image ───────────────────────────────────────────────────────────────────────────────
|
|
600
|
+
async function imageMatch (a) {
|
|
601
|
+
const src = inputFile(a.path, 'path'), tplPath = inputFile(a.template, 'template'), box = boxOf(a.box)
|
|
602
|
+
const thr = num(a.threshold, 0.1, 1, 'threshold') ?? 0.8, max = int(a.max, 1, 100, 'max') ?? 10
|
|
603
|
+
const scales = Array.isArray(a.scales) ? a.scales.slice(0, 12).map((s) => num(s, 0.1, 10, 'scales')) : [1]
|
|
604
|
+
const cv = await opencv()
|
|
605
|
+
const img = await rgba(src, box), tpl = await rgba(tplPath)
|
|
606
|
+
const I = rgbaMat(cv, img), T = rgbaMat(cv, tpl), gi = new cv.Mat(), gt0 = new cv.Mat()
|
|
607
|
+
cv.cvtColor(I, gi, cv.COLOR_RGBA2GRAY); cv.cvtColor(T, gt0, cv.COLOR_RGBA2GRAY)
|
|
608
|
+
const hits = []
|
|
609
|
+
try {
|
|
610
|
+
for (const s of scales) {
|
|
611
|
+
const gt = new cv.Mat(); cv.resize(gt0, gt, new cv.Size(Math.max(4, Math.round(tpl.w * s)), Math.max(4, Math.round(tpl.h * s))), 0, 0, cv.INTER_AREA)
|
|
612
|
+
if (gt.cols > gi.cols || gt.rows > gi.rows) { gt.delete(); continue }
|
|
613
|
+
const res = new cv.Mat(); cv.matchTemplate(gi, gt, res, cv.TM_CCOEFF_NORMED)
|
|
614
|
+
for (let k = 0; k < max * 4; k++) { // the best match, then blank its neighbourhood and look again
|
|
615
|
+
const m = cv.minMaxLoc(res); if (m.maxVal < thr) break
|
|
616
|
+
hits.push({ x: m.maxLoc.x, y: m.maxLoc.y, w: gt.cols, h: gt.rows, score: m.maxVal, scale: s })
|
|
617
|
+
cv.rectangle(res, new cv.Point(m.maxLoc.x - gt.cols / 2, m.maxLoc.y - gt.rows / 2), new cv.Point(m.maxLoc.x + gt.cols / 2, m.maxLoc.y + gt.rows / 2), new cv.Scalar(-1), -1)
|
|
618
|
+
}
|
|
619
|
+
res.delete(); gt.delete()
|
|
620
|
+
}
|
|
621
|
+
} finally { I.delete(); T.delete(); gi.delete(); gt0.delete() }
|
|
622
|
+
// across scales, keep the best of overlapping hits
|
|
623
|
+
const iou = (p, q) => { const ix = Math.max(0, Math.min(p.x + p.w, q.x + q.w) - Math.max(p.x, q.x)), iy = Math.max(0, Math.min(p.y + p.h, q.y + q.h) - Math.max(p.y, q.y)); return (ix * iy) / (p.w * p.h + q.w * q.h - ix * iy) }
|
|
624
|
+
const keep = []; for (const h of hits.sort((p, q) => q.score - p.score)) if (keep.length < max && keep.every((k) => iou(k, h) < 0.3)) keep.push(h)
|
|
625
|
+
const dx = box ? box.x : 0, dy = box ? box.y : 0
|
|
626
|
+
const boxes = keep.map((h, i) => ({ x: h.x + dx, y: h.y + dy, w: h.w, h: h.h, label: i + 1 }))
|
|
627
|
+
const text = `${basename(tplPath)} (${tpl.w}×${tpl.h}) in ${basename(src)}${box ? ` region ${geom(box)}` : ''}: ${keep.length} match(es) with score ≥ ${thr}${scales.length > 1 ? ` across scales ${scales.join(', ')}` : ''} — [x,y,w,h] in the original image's pixels`
|
|
628
|
+
if (!keep.length) return [{ type: 'text', text: `${text}\n(none — lower the threshold, or try scales: [0.5, 0.75, 1.25, 1.5] if the size differs)` }]
|
|
629
|
+
return picture(await drawBoxes(src, boxes), `${text}\n${keep.map((h, i) => `${i + 1}. [${boxes[i].x},${boxes[i].y},${h.w},${h.h}] score ${h.score.toFixed(3)}${scales.length > 1 ? ` at ×${h.scale}` : ''}`).join('\n')}`)
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
// ── OpenCV on video: follow an object, or see where things move ────────────────────────────────────────────────────────
|
|
633
|
+
const MAX_TRACK_SIDE = 640
|
|
634
|
+
async function rawFrames (src, p, start, end, fps, W, H) { // RGB frames, W×H, from start to end at fps
|
|
635
|
+
const { stdout } = await run('ffmpeg', ['-v', 'error', '-ss', String(start), '-to', String(end), '-i', src, '-vf', `fps=${fps},scale=${W}:${H}`, '-f', 'rawvideo', '-pix_fmt', 'rgb24', '-'], { encoding: 'buffer', timeout: Math.max(TIMEOUT, (end - start) * 2000) })
|
|
636
|
+
const n = Math.floor(stdout.length / (W * H * 3)), out = []
|
|
637
|
+
for (let i = 0; i < n; i++) out.push(stdout.subarray(i * W * H * 3, (i + 1) * W * H * 3))
|
|
638
|
+
return out
|
|
639
|
+
}
|
|
640
|
+
async function videoTrack (a) {
|
|
641
|
+
const src = inputFile(a.path, 'path'), p = await probe(src)
|
|
642
|
+
if (!p.v) fail('no video stream')
|
|
643
|
+
const cv = await opencv()
|
|
644
|
+
const k = Math.min(1, MAX_TRACK_SIDE / Math.max(p.v.width, p.v.height)), W = 2 * Math.round(p.v.width * k / 2), H = 2 * Math.round(p.v.height * k / 2)
|
|
645
|
+
const sx = W / p.v.width, sy = H / p.v.height
|
|
646
|
+
if ((a.op || 'track') === 'flow') {
|
|
647
|
+
const t1 = num(a.t1, 0, p.dur, 't1') ?? 0, t2 = num(a.t2, 0, p.dur, 't2') ?? Math.min(p.dur, t1 + 1 / Math.max(1, p.fps || 10))
|
|
648
|
+
const f = async (t) => { const fr = await rawFrames(src, p, t, t + 0.5, 50, W, H); if (!fr.length) fail(`no frame at ${ts(t)}`); const m = cv.matFromArray(H, W, cv.CV_8UC3, fr[0]); const g = new cv.Mat(); cv.cvtColor(m, g, cv.COLOR_RGB2GRAY); m.delete(); return g }
|
|
649
|
+
const g1 = await f(t1), g2 = await f(t2), flow = new cv.Mat()
|
|
650
|
+
cv.calcOpticalFlowFarneback(g1, g2, flow, 0.5, 3, 15, 3, 5, 1.2, 0)
|
|
651
|
+
const d = flow.data32F; let moving = 0, sdx = 0, sdy = 0, maxm = 0, x0 = W, y0 = H, x1 = -1, y1 = -1
|
|
652
|
+
const mag = Buffer.alloc(W * H)
|
|
653
|
+
for (let i = 0; i < W * H; i++) { const dx = d[2 * i], dy = d[2 * i + 1], m = Math.hypot(dx, dy); if (m > maxm) maxm = m; if (m >= 1) { moving++; sdx += dx; sdy += dy; const x = i % W, y = (i / W) | 0; if (x < x0) x0 = x; if (x > x1) x1 = x; if (y < y0) y0 = y; if (y > y1) y1 = y } mag[i] = Math.min(255, Math.round(m * 32)) }
|
|
654
|
+
g1.delete(); g2.delete(); flow.delete()
|
|
655
|
+
const base = await frameAt(src, t2, null, sx), heat = outFile('png'), out = outFile('png')
|
|
656
|
+
const raw = outFile('gray'); writeFileSync(raw, mag)
|
|
657
|
+
await run('convert', ['-size', `${W}x${H}`, '-depth', '8', `gray:${raw}`, heat]); rmSync(raw, { force: true })
|
|
658
|
+
await run('convert', [base, '(', heat, '-threshold', '12%', '-fill', 'red', '-opaque', 'white', '-transparent', 'black', '-channel', 'A', '-evaluate', 'multiply', '0.6', '+channel', ')', '-composite', out]); rmSync(heat, { force: true }); rmSync(base, { force: true })
|
|
659
|
+
const dir = moving ? (Math.atan2(-sdy / moving, sdx / moving) * 180 / Math.PI).toFixed(0) : null
|
|
660
|
+
const bb = moving ? `[${Math.round(x0 / sx)},${Math.round(y0 / sy)},${Math.round((x1 - x0 + 1) / sx)},${Math.round((y1 - y0 + 1) / sy)}]` : ''
|
|
661
|
+
return picture(out, `${basename(src)}: motion from ${ts(t1)} to ${ts(t2)} (dense optical flow, at ${W}×${H})\n${moving ? `${(100 * moving / (W * H)).toFixed(1)}% of the picture moved ≥ 1 px · within ${bb} (original pixels) · mean direction ${dir}° (0° = right, 90° = up) · fastest ${(maxm / sx).toFixed(1)} px` : 'nothing moved ≥ 1 px'}\nThe picture is the frame at ${ts(t2)} with the moving parts in red.`)
|
|
662
|
+
}
|
|
663
|
+
const box = boxOf(a.box); if (!box) fail('track: give the box of the object in the first frame ([x, y, width, height], original pixels)')
|
|
664
|
+
const start = num(a.start, 0, p.dur, 'start') ?? 0, end = num(a.end, 0, p.dur, 'end') ?? p.dur
|
|
665
|
+
if (end <= start) fail('end must be after start')
|
|
666
|
+
const fps = Math.min(num(a.fps, 0.2, 30, 'fps') ?? 4, 120 / (end - start))
|
|
667
|
+
const frames = await rawFrames(src, p, start, end, fps, W, H)
|
|
668
|
+
if (frames.length < 2) fail('fewer than two frames in that range')
|
|
669
|
+
const mats = (i) => cv.matFromArray(H, W, cv.CV_8UC3, frames[i])
|
|
670
|
+
const tr = new cv.TrackerMIL(), path = []
|
|
671
|
+
let cur = { x: Math.round(box.x * sx), y: Math.round(box.y * sy), w: Math.max(4, Math.round(box.w * sx)), h: Math.max(4, Math.round(box.h * sy)) }
|
|
672
|
+
// MIL never says it lost the object (measured: on a frame with nothing to follow it drifted to a corner and reported
|
|
673
|
+
// success), so each box is also compared with the FIRST frame's patch; a weak likeness is reported as uncertain.
|
|
674
|
+
const gray = (m) => { const g = new cv.Mat(); cv.cvtColor(m, g, cv.COLOR_RGB2GRAY); return g }
|
|
675
|
+
const m0 = mats(0), g0 = gray(m0), patch = g0.roi(new cv.Rect(cur.x, cur.y, Math.min(cur.w, W - cur.x), Math.min(cur.h, H - cur.y))).clone()
|
|
676
|
+
tr.init(m0, new cv.Rect(cur.x, cur.y, cur.w, cur.h)); m0.delete(); g0.delete()
|
|
677
|
+
const likeness = (g, b) => { const x = Math.max(0, Math.min(W - 4, b.x)), y = Math.max(0, Math.min(H - 4, b.y)), w = Math.max(4, Math.min(b.w, W - x)), h = Math.max(4, Math.min(b.h, H - y)); const r = g.roi(new cv.Rect(x, y, w, h)), z = new cv.Mat(), o = new cv.Mat(); cv.resize(r, z, new cv.Size(patch.cols, patch.rows)); cv.matchTemplate(z, patch, o, cv.TM_CCOEFF_NORMED); const v = o.data32F[0]; r.delete(); z.delete(); o.delete(); return Number.isFinite(v) ? v : null }
|
|
678
|
+
path.push({ t: start, ...cur, ok: true, like: 1 })
|
|
679
|
+
for (let i = 1; i < frames.length; i++) { const m = mats(i); const [ok, r] = tr.update(m); if (ok) cur = { x: r.x, y: r.y, w: r.width, h: r.height }; const g = gray(m); const like = likeness(g, cur); g.delete(); m.delete(); path.push({ t: start + i / fps, ...cur, ok: ok && (like === null || like >= 0.4), like }) }
|
|
680
|
+
patch.delete()
|
|
681
|
+
const orig = path.map((q) => ({ t: q.t, ok: q.ok, like: q.like, x: Math.round(q.x / sx), y: Math.round(q.y / sy), w: Math.round(q.w / sx), h: Math.round(q.h / sy) }))
|
|
682
|
+
const pick = Array.from({ length: Math.min(12, orig.length) }, (_, i) => orig[Math.round(i * (orig.length - 1) / Math.max(1, Math.min(12, orig.length) - 1))])
|
|
683
|
+
const shots = []
|
|
684
|
+
for (const q of pick) { const f = await frameAt(src, q.t, null, sx); const b = { x: Math.round(q.x * sx), y: Math.round(q.y * sy), w: Math.round(q.w * sx), h: Math.round(q.h * sy), label: ts(q.t) }; const o = await drawBoxes(f, [b], { color: q.ok ? 'lime' : 'red' }); rmSync(f, { force: true }); shots.push({ f: o, t: q.t }) }
|
|
685
|
+
const out = await sheet(shots.map((s) => [s.f, s.t]), Math.min(4, shots.length), ts)
|
|
686
|
+
for (const s of shots) rmSync(s.f, { force: true })
|
|
687
|
+
const first = orig[0], last = orig[orig.length - 1], lost = orig.filter((q) => !q.ok).length
|
|
688
|
+
const lines = orig.map((q) => `${ts(q.t)} [${q.x},${q.y},${q.w},${q.h}]${q.like === null ? ' (likeness unknown: a flat patch)' : ` likeness ${q.like.toFixed(2)}`}${q.ok ? '' : ' UNCERTAIN'}`)
|
|
689
|
+
return picture(out, `${basename(src)}: tracked [${box.x},${box.y},${box.w},${box.h}] from ${ts(start)} to ${ts(end)} at ${fps.toFixed(2)} fps (OpenCV MIL tracker) — ${orig.length} positions, original pixels\nmoved ${Math.round(last.x + last.w / 2 - first.x - first.w / 2)}, ${Math.round(last.y + last.h / 2 - first.y - first.h / 2)} px (centre)${lost ? ` · UNCERTAIN in ${lost} frame(s) (likeness to the first frame's patch < 0.4 — check them with video_frames)` : ''}\n${lines.join('\n').slice(0, 6000)}\nThe sheet shows the box in green (red where uncertain).`)
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
// ── the MCP surface ────────────────────────────────────────────────────────────────────────────────────────────────────
|
|
693
|
+
// ── web_search: CONFIDENTIAL web search, the same one the Spaces assistant has (agent/tinfoilWebSearch.mjs) ─────────────
|
|
694
|
+
// OpenCode's own `websearch` tool is offered only with the OpenCode provider or with OPENCODE_ENABLE_EXA / _PARALLEL set,
|
|
695
|
+
// which hands every query to Exa or Parallel in the clear. This one asks Tinfoil's search agent, which runs INSIDE an
|
|
696
|
+
// SEV-SNP enclave and opens TLS from there to the search index — the query is never plaintext on a box anyone else
|
|
697
|
+
// operates, and it is the same boundary the connector's confidential models already use (TINFOIL_API_KEY is in
|
|
698
|
+
// ~/.opencode-server.env for them). Constants held equal to agent/tinfoilWebSearch.mjs by tools/media-mcp/webSearch.test.mjs.
|
|
699
|
+
export const SEARCH_HOST = 'inference.tinfoil.sh'
|
|
700
|
+
export const SEARCH_PATH = '/v1/chat/completions'
|
|
701
|
+
export const SEARCH_MODEL = 'gpt-oss-120b' // the SEARCH AGENT, small and fast — not the coding agent's brain
|
|
702
|
+
const SEARCH_TIMEOUT_MS = Number(process.env.WITBITZ_SEARCH_TIMEOUT_MS || 60_000)
|
|
703
|
+
const SEARCH_MAX_CHARS = 6000
|
|
704
|
+
export const SEARCH_MAX_TOKENS = 8000 // == agent/tinfoilWebSearch.mjs: the search model reasons inside this budget
|
|
705
|
+
export const SEARCH_REASONING_EFFORT = 'low' // == agent/tinfoilWebSearch.mjs: brief reasoning, so it answers in time
|
|
706
|
+
export function parseSearchAnswer (data) { // == agent/tinfoilWebSearch.mjs parseSearchAnswer
|
|
707
|
+
const msg = (data && data.choices && data.choices[0] && data.choices[0].message) || {}
|
|
708
|
+
const text = typeof msg.content === 'string' ? msg.content : ''
|
|
709
|
+
const citations = []; const seen = new Set()
|
|
710
|
+
for (const a of (Array.isArray(msg.annotations) ? msg.annotations : [])) {
|
|
711
|
+
if (!a || a.type !== 'url_citation') continue
|
|
712
|
+
const c = a.url_citation && typeof a.url_citation === 'object' ? a.url_citation : a
|
|
713
|
+
const url = typeof c.url === 'string' ? c.url : ''
|
|
714
|
+
if (!/^https?:\/\//i.test(url) || seen.has(url)) continue
|
|
715
|
+
seen.add(url)
|
|
716
|
+
let site = ''; try { site = new URL(url).hostname.replace(/^www\./, '') } catch { /* */ }
|
|
717
|
+
citations.push({ url, title: String(c.title || '').slice(0, 200), site })
|
|
718
|
+
if (citations.length >= 12) break
|
|
719
|
+
}
|
|
720
|
+
return { text, citations }
|
|
721
|
+
}
|
|
722
|
+
async function webSearch (a, { fetchImpl = fetch, env = process.env } = {}) {
|
|
723
|
+
const q = String((a && a.query) || '').replace(/\s+/g, ' ').trim().slice(0, 2000)
|
|
724
|
+
if (!q) fail('query is required')
|
|
725
|
+
const apiKey = env.TINFOIL_API_KEY
|
|
726
|
+
if (!apiKey) fail('web_search needs a Tinfoil key: TINFOIL_API_KEY is not set for this OpenCode (on a laptop it comes from ~/.opencode-server.env; a cloud computer gets it from its Sentinel)')
|
|
727
|
+
const url = env.WITBITZ_SEARCH_URL || `https://${SEARCH_HOST}${SEARCH_PATH}`
|
|
728
|
+
const body = {
|
|
729
|
+
model: SEARCH_MODEL,
|
|
730
|
+
messages: [
|
|
731
|
+
{ role: 'system', content: 'You are a web research assistant. Search the web and report what you find, briefly and factually, with dates. State plainly when sources disagree or when something has not happened yet. Do not speculate beyond what the sources say.' },
|
|
732
|
+
{ role: 'user', content: q },
|
|
733
|
+
],
|
|
734
|
+
max_tokens: SEARCH_MAX_TOKENS,
|
|
735
|
+
reasoning_effort: SEARCH_REASONING_EFFORT,
|
|
736
|
+
web_search_options: {},
|
|
737
|
+
...(a && a.pii_check === false ? {} : { pii_check_options: {} }), // a query carrying personal data is refused before it leaves the enclave for the index
|
|
738
|
+
}
|
|
739
|
+
let data
|
|
740
|
+
try {
|
|
741
|
+
const res = await fetchImpl(url, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${apiKey}` }, body: JSON.stringify(body), signal: AbortSignal.timeout(SEARCH_TIMEOUT_MS) })
|
|
742
|
+
if (!res.ok) fail(`the search endpoint answered ${res.status}`)
|
|
743
|
+
data = await res.json()
|
|
744
|
+
} catch (e) { fail(`could not search the web (${String((e && e.message) || e).slice(0, 160)})`) }
|
|
745
|
+
const { text, citations } = parseSearchAnswer(data)
|
|
746
|
+
const cutOff = ((data && data.choices && data.choices[0]) || {}).finish_reason === 'length'
|
|
747
|
+
if (!text && !citations.length) fail(cutOff ? 'the search ran out of room before it answered' : 'the search returned nothing')
|
|
748
|
+
const findings = text.length > SEARCH_MAX_CHARS ? text.slice(0, SEARCH_MAX_CHARS) + '\n…(truncated)' : text
|
|
749
|
+
const sources = citations.length ? '\n\nSources:\n' + citations.map((c, i) => `${i + 1}. ${c.title || c.site || c.url} — ${c.url}`).join('\n') : ''
|
|
750
|
+
return [{ type: 'text', text: findings + sources }]
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
const path = { type: 'string', description: 'Path to the file (absolute or relative to the working directory).' }
|
|
754
|
+
const box = { type: 'array', items: { type: 'number' }, minItems: 4, maxItems: 4, description: '[x, y, width, height] in pixels.' }
|
|
755
|
+
export const TOOLS = [
|
|
756
|
+
{ name: 'web_search', handler: webSearch, description: 'Search the web and get a short factual report with dated findings and numbered source links. Use it for anything current (versions, releases, prices, news, documentation you do not have) instead of guessing or scraping a search page with webfetch. Confidential: the query is answered by a search agent inside a hardware enclave (Tinfoil) and never stored in the clear anywhere else. Then open a source with webfetch if you need its full text.',
|
|
757
|
+
inputSchema: { type: 'object', required: ['query'], properties: { query: { type: 'string', description: 'What to find out — a plain question or search phrase, one topic per call.' }, pii_check: { type: 'boolean', description: 'Default true: a query that carries personal data (names with addresses, emails, phone numbers) is refused before it leaves the enclave. Set false only for a query about public figures that the check wrongly refuses.' } } } },
|
|
758
|
+
{ name: 'image_view', handler: imageView, description: 'Look at an image (any format, first frame) after transforming it, and SEE the result: crop a region (box), zoom (scale), resize, rotate, flip, flatten a photographed page (perspective: its four corners), deskew, pad, isolate a channel, enhance (normalize, brightness/contrast, gamma, sharpen, denoise, grayscale, threshold, adaptive_threshold, invert, edges), or overlay a labelled pixel grid. Re-inspect small details by cropping and scaling 2–4×.',
|
|
759
|
+
inputSchema: { type: 'object', required: ['path'], properties: { path, box, scale: { type: 'number', description: 'e.g. 3 = three times larger' }, width: { type: 'integer' }, height: { type: 'integer' }, rotate: { type: 'number', description: 'degrees clockwise' }, flip: { type: 'string', enum: ['horizontal', 'vertical'] }, perspective: { type: 'array', description: 'four [x, y] corners: top-left, top-right, bottom-right, bottom-left', items: { type: 'array', items: { type: 'number' } } }, deskew: { type: 'boolean' }, pad: { type: 'integer' }, channel: { type: 'string', enum: ['red', 'green', 'blue', 'alpha'] }, grid: { type: 'integer', description: 'grid spacing in pixels, labelled with coordinates' }, enhance: { type: 'object', properties: { normalize: { type: 'boolean' }, brightness: { type: 'number' }, contrast: { type: 'number' }, gamma: { type: 'number' }, sharpen: { type: 'boolean' }, denoise: { type: 'boolean' }, grayscale: { type: 'boolean' }, threshold: { type: 'number', description: 'percent' }, adaptive_threshold: { type: 'boolean' }, invert: { type: 'boolean' }, edges: { type: 'boolean' } } } } } },
|
|
760
|
+
{ name: 'image_info', handler: imageInfo, description: 'Facts about an image file without looking at it: format, size, color space, bit depth, alpha, frame count, mean colors and contrast, and EXIF (camera, date, orientation, GPS).', inputSchema: { type: 'object', required: ['path'], properties: { path } } },
|
|
761
|
+
{ name: 'image_measure', handler: imageMeasure, description: 'Measure instead of estimating: exact colors at points, distances and angles between consecutive points, area and mean color of a box, or count and locate distinct regions (connected components).', inputSchema: { type: 'object', required: ['path'], properties: { path, points: { type: 'array', items: { type: 'array', items: { type: 'number' } }, description: '[[x, y], …]' }, box, regions: { type: 'object', properties: { threshold: { type: 'number', description: 'percent, default 50' }, dark: { type: 'boolean', description: 'count dark regions on a light background (default true)' }, min_area: { type: 'integer' } } } } } },
|
|
762
|
+
{ name: 'image_ocr', handler: imageOcr, description: 'Read text with OCR (Tesseract) and get each line with its position — an independent check on what you read visually, and exact coordinates for it. Optionally only a box, scaled up for small print.', inputSchema: { type: 'object', required: ['path'], properties: { path, box, scale: { type: 'number' }, lang: { type: 'string', description: 'tesseract language(s), default eng' }, psm: { type: 'integer', description: 'page segmentation mode, default 3; 6 = one block, 7 = one line' } } } },
|
|
763
|
+
{ name: 'image_diff', handler: imageDiff, description: 'Compare two images: how many pixels differ, a similarity score, where the changes are, and a picture with the differences in red — for UI regressions and before/after checks.', inputSchema: { type: 'object', required: ['a', 'b'], properties: { a: path, b: path, fuzz: { type: 'number', description: 'percent of color difference to ignore, default 3' } } } },
|
|
764
|
+
{ name: 'pdf_view', handler: pdfView, description: 'See a PDF page as an image (with its embedded text and the document metadata), zoom into a region at high dpi, or get an overview of all pages at once.', inputSchema: { type: 'object', required: ['path'], properties: { path, page: { type: 'integer' }, dpi: { type: 'integer', description: 'default 110; 200–400 to zoom' }, box: { ...box, description: '[x, y, width, height] in pixels AT THIS DPI' }, overview: { type: 'boolean' }, text: { type: 'boolean' } } } },
|
|
765
|
+
{ name: 'video_overview', handler: videoOverview, description: 'Start here with a video: its metadata (duration, resolution, fps, codecs, audio, subtitles), the scene cuts, and a contact sheet of evenly spaced frames labelled with times.', inputSchema: { type: 'object', required: ['path'], properties: { path, frames: { type: 'integer', description: 'default 16' }, columns: { type: 'integer' }, scenes: { type: 'boolean' }, scene_threshold: { type: 'number' } } } },
|
|
766
|
+
{ name: 'video_frames', handler: videoFrames, description: 'Look at a video more closely: one exact frame (at), or a time range sampled densely (start, end, fps) as a labelled sheet — optionally only a region (box), enlarged (scale), and saved as a clip.', inputSchema: { type: 'object', required: ['path'], properties: { path, at: { type: 'number', description: 'seconds' }, start: { type: 'number' }, end: { type: 'number' }, fps: { type: 'number' }, box, scale: { type: 'number' }, columns: { type: 'integer' }, save_clip: { type: 'boolean' } } } },
|
|
767
|
+
{ name: 'video_diff', handler: videoDiff, description: 'What changed between two moments of a video: the differing pixels in red, how many, and where — for screen recordings and UI bugs.', inputSchema: { type: 'object', required: ['path', 't1', 't2'], properties: { path, t1: { type: 'number' }, t2: { type: 'number' }, box, fuzz: { type: 'number' } } } },
|
|
768
|
+
{ name: 'image_codes', handler: imageCodes, description: 'Read barcodes and QR codes in an image (ZXing: QR, Data Matrix, Aztec, PDF417, EAN/UPC, Code 128/39/93, ITF, Codabar…) — rotated, inverted or small ones too — with each one\'s text and position, outlined on the picture.', inputSchema: { type: 'object', required: ['path'], properties: { path, box, formats: { type: 'array', items: { type: 'string' }, description: 'only these, e.g. ["QRCode", "EAN13"]' } } } },
|
|
769
|
+
{ name: 'media_meta', handler: mediaMeta, description: 'Metadata of any file (exiftool: EXIF, GPS, XMP, IPTC, camera, PDF/video/audio tags): read it; write tags; or strip it (all, gps, camera, personal). Writes go to a COPY unless in_place is true.', inputSchema: { type: 'object', required: ['path'], properties: { path, op: { type: 'string', enum: ['read', 'write', 'strip'] }, tags: { description: 'read: names to show, e.g. ["GPS:all", "DateTimeOriginal"]; write: { "Artist": "…", "XMP:Title": "…", "Copyright": null }' }, what: { type: 'string', enum: ['all', 'gps', 'camera', 'personal'], description: 'strip: which (default all)' }, in_place: { type: 'boolean' } } } },
|
|
770
|
+
{ name: 'image_match', handler: imageMatch, description: 'Find where a smaller image (a template: an icon, a button, a logo) appears in an image (OpenCV template matching): every match above a score, at one or several scales, outlined on the picture with exact boxes.', inputSchema: { type: 'object', required: ['path', 'template'], properties: { path, template: { type: 'string', description: 'Path to the image to look for.' }, box, threshold: { type: 'number', description: 'minimum score 0–1, default 0.8' }, max: { type: 'integer', description: 'default 10' }, scales: { type: 'array', items: { type: 'number' }, description: 'template sizes to try, e.g. [0.5, 0.75, 1, 1.25, 1.5]' } } } },
|
|
771
|
+
{ name: 'video_track', handler: videoTrack, description: 'OpenCV on video. op "track": follow an object (its box in the first frame) through a time range — its box at each moment and how far it moved, on a sheet of frames. op "flow": what moved between two moments (dense optical flow) — the moving parts in red, how much, where and in which direction.', inputSchema: { type: 'object', required: ['path'], properties: { path, op: { type: 'string', enum: ['track', 'flow'] }, box, start: { type: 'number' }, end: { type: 'number' }, fps: { type: 'number', description: 'track: positions per second, default 4' }, t1: { type: 'number' }, t2: { type: 'number' } } } },
|
|
772
|
+
{ name: 'audio', handler: audio, description: 'The sound of a video or audio file: info, extract a range as WAV, find silences (natural section breaks), read embedded subtitles, or transcribe speech to timestamped text (on this computer, nothing is uploaded; words: true for one line per word).', inputSchema: { type: 'object', required: ['path'], properties: { path, op: { type: 'string', enum: ['info', 'extract', 'silence', 'subtitles', 'transcribe'] }, language: { type: 'string', description: 'transcribe: en, he, … or auto (default)' }, words: { type: 'boolean' }, quality: { type: 'string', enum: ['fast', 'accurate'], description: 'transcribe: default fast for English, accurate (a larger model) for other languages' }, start: { type: 'number' }, end: { type: 'number' }, noise_db: { type: 'number' }, min_seconds: { type: 'number' }, stream: { type: 'integer' } } } },
|
|
773
|
+
]
|
|
774
|
+
|
|
775
|
+
export async function callTool (name, args) {
|
|
776
|
+
const t = TOOLS.find((x) => x.name === name)
|
|
777
|
+
if (!t) return { content: [{ type: 'text', text: `unknown tool: ${name}` }], isError: true }
|
|
778
|
+
try { return { content: await t.handler(args && typeof args === 'object' ? args : {}) } }
|
|
779
|
+
catch (e) { return { content: [{ type: 'text', text: e instanceof ToolError ? e.message : `${name} failed: ${String((e && e.message) || e).slice(0, 300)}` }], isError: true } }
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
783
|
+
const send = (m) => process.stdout.write(JSON.stringify(m) + '\n')
|
|
784
|
+
createInterface({ input: process.stdin }).on('line', async (line) => {
|
|
785
|
+
let m; try { m = JSON.parse(line) } catch { return }
|
|
786
|
+
if (m.method === 'initialize') return send({ jsonrpc: '2.0', id: m.id, result: { protocolVersion: (m.params && m.params.protocolVersion) || PROTOCOL_VERSION, capabilities: { tools: {} }, serverInfo: { name: 'witbitz-media', version: '1.0.0' } } })
|
|
787
|
+
if (m.method === 'tools/list') return send({ jsonrpc: '2.0', id: m.id, result: { tools: TOOLS.map(({ name, description, inputSchema }) => ({ name, description, inputSchema })) } })
|
|
788
|
+
if (m.method === 'tools/call') return send({ jsonrpc: '2.0', id: m.id, result: await callTool(m.params && m.params.name, m.params && m.params.arguments) })
|
|
789
|
+
if (m.id !== undefined && m.method) send({ jsonrpc: '2.0', id: m.id, error: { code: -32601, message: 'method not found' } })
|
|
790
|
+
})
|
|
791
|
+
}
|