capturekit 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- capturekit-0.1.0/.claude/agents/architecture-security-reviewer.md +18 -0
- capturekit-0.1.0/.claude/agents/audio-systems-specialist.md +15 -0
- capturekit-0.1.0/.claude/agents/linux-x11-specialist.md +16 -0
- capturekit-0.1.0/.claude/agents/performance-reviewer.md +14 -0
- capturekit-0.1.0/.claude/agents/test-validation-reviewer.md +14 -0
- capturekit-0.1.0/.claude/agents/windows-api-researcher.md +17 -0
- capturekit-0.1.0/.claude/skills/audio-capture-routing/SKILL.md +64 -0
- capturekit-0.1.0/.claude/skills/av-sync/SKILL.md +40 -0
- capturekit-0.1.0/.claude/skills/bug-investigation/SKILL.md +43 -0
- capturekit-0.1.0/.claude/skills/docs-release/SKILL.md +39 -0
- capturekit-0.1.0/.claude/skills/ffmpeg-encoding/SKILL.md +65 -0
- capturekit-0.1.0/.claude/skills/linux-x11-backend/SKILL.md +67 -0
- capturekit-0.1.0/.claude/skills/native-api-research/SKILL.md +67 -0
- capturekit-0.1.0/.claude/skills/performance-profiling/SKILL.md +38 -0
- capturekit-0.1.0/.claude/skills/test-development/SKILL.md +47 -0
- capturekit-0.1.0/.claude/skills/windows-capture-backend/SKILL.md +71 -0
- capturekit-0.1.0/.claude/skills/windows-capture-backend/reference/quirks.md +40 -0
- capturekit-0.1.0/.claude/skills/windows-capture-backend/reference/vtables.md +52 -0
- capturekit-0.1.0/.gitattributes +4 -0
- capturekit-0.1.0/.github/workflows/publish.yml +75 -0
- capturekit-0.1.0/.gitignore +14 -0
- capturekit-0.1.0/CHANGELOG.md +24 -0
- capturekit-0.1.0/CLAUDE.md +80 -0
- capturekit-0.1.0/CONTRIBUTING.md +43 -0
- capturekit-0.1.0/PKG-INFO +112 -0
- capturekit-0.1.0/README.md +87 -0
- capturekit-0.1.0/docs/PROGRESS.md +100 -0
- capturekit-0.1.0/docs/architecture.md +148 -0
- capturekit-0.1.0/docs/capabilities.md +65 -0
- capturekit-0.1.0/docs/decisions.md +62 -0
- capturekit-0.1.0/docs/guide/api.md +90 -0
- capturekit-0.1.0/docs/guide/audio.md +57 -0
- capturekit-0.1.0/docs/guide/encoding.md +47 -0
- capturekit-0.1.0/docs/guide/linux.md +105 -0
- capturekit-0.1.0/docs/guide/performance.md +52 -0
- capturekit-0.1.0/docs/guide/troubleshooting.md +22 -0
- capturekit-0.1.0/docs/guide/windows.md +70 -0
- capturekit-0.1.0/docs/research/linux-x11-and-audio.md +429 -0
- capturekit-0.1.0/examples/grab_frames.py +12 -0
- capturekit-0.1.0/examples/list_sources.py +17 -0
- capturekit-0.1.0/examples/record_window.py +41 -0
- capturekit-0.1.0/pyproject.toml +66 -0
- capturekit-0.1.0/scripts/bench_capture.py +76 -0
- capturekit-0.1.0/scripts/bench_record.py +75 -0
- capturekit-0.1.0/scripts/linux_smoke.sh +36 -0
- capturekit-0.1.0/src/capturekit/__init__.py +164 -0
- capturekit-0.1.0/src/capturekit/_backends/__init__.py +0 -0
- capturekit-0.1.0/src/capturekit/_backends/base.py +89 -0
- capturekit-0.1.0/src/capturekit/_backends/linux/__init__.py +1 -0
- capturekit-0.1.0/src/capturekit/_backends/linux/_xlib.py +655 -0
- capturekit-0.1.0/src/capturekit/_backends/linux/api.py +160 -0
- capturekit-0.1.0/src/capturekit/_backends/linux/pulse.py +464 -0
- capturekit-0.1.0/src/capturekit/_backends/linux/x11.py +720 -0
- capturekit-0.1.0/src/capturekit/_backends/windows/__init__.py +0 -0
- capturekit-0.1.0/src/capturekit/_backends/windows/_com.py +350 -0
- capturekit-0.1.0/src/capturekit/_backends/windows/_d3d11.py +307 -0
- capturekit-0.1.0/src/capturekit/_backends/windows/_win32.py +420 -0
- capturekit-0.1.0/src/capturekit/_backends/windows/_winrt.py +162 -0
- capturekit-0.1.0/src/capturekit/_backends/windows/api.py +214 -0
- capturekit-0.1.0/src/capturekit/_backends/windows/cursor.py +66 -0
- capturekit-0.1.0/src/capturekit/_backends/windows/dxgi_duplication.py +310 -0
- capturekit-0.1.0/src/capturekit/_backends/windows/wasapi.py +570 -0
- capturekit-0.1.0/src/capturekit/_backends/windows/wgc.py +398 -0
- capturekit-0.1.0/src/capturekit/_pipeline/__init__.py +1 -0
- capturekit-0.1.0/src/capturekit/_pipeline/aligner.py +177 -0
- capturekit-0.1.0/src/capturekit/_pipeline/clock.py +47 -0
- capturekit-0.1.0/src/capturekit/_pipeline/frame_queue.py +120 -0
- capturekit-0.1.0/src/capturekit/_pipeline/mixer.py +80 -0
- capturekit-0.1.0/src/capturekit/_pipeline/timeline.py +133 -0
- capturekit-0.1.0/src/capturekit/_platform.py +24 -0
- capturekit-0.1.0/src/capturekit/_version.py +1 -0
- capturekit-0.1.0/src/capturekit/_worker.py +100 -0
- capturekit-0.1.0/src/capturekit/capabilities.py +79 -0
- capturekit-0.1.0/src/capturekit/capture.py +141 -0
- capturekit-0.1.0/src/capturekit/config.py +85 -0
- capturekit-0.1.0/src/capturekit/encoding/__init__.py +5 -0
- capturekit-0.1.0/src/capturekit/encoding/ffmpeg.py +356 -0
- capturekit-0.1.0/src/capturekit/encoding/pipes.py +212 -0
- capturekit-0.1.0/src/capturekit/errors.py +109 -0
- capturekit-0.1.0/src/capturekit/py.typed +0 -0
- capturekit-0.1.0/src/capturekit/recording.py +940 -0
- capturekit-0.1.0/src/capturekit/sources.py +124 -0
- capturekit-0.1.0/src/capturekit/types.py +211 -0
- capturekit-0.1.0/tests/__init__.py +0 -0
- capturekit-0.1.0/tests/conftest.py +138 -0
- capturekit-0.1.0/tests/fixtures/color_window.py +220 -0
- capturekit-0.1.0/tests/fixtures/tone_player.py +82 -0
- capturekit-0.1.0/tests/fixtures/x11_color_window.py +127 -0
- capturekit-0.1.0/tests/helpers.py +137 -0
- capturekit-0.1.0/tests/linux/__init__.py +0 -0
- capturekit-0.1.0/tests/linux/conftest.py +73 -0
- capturekit-0.1.0/tests/linux/test_pulse.py +164 -0
- capturekit-0.1.0/tests/linux/test_x11.py +205 -0
- capturekit-0.1.0/tests/unit/__init__.py +0 -0
- capturekit-0.1.0/tests/unit/test_api_types.py +100 -0
- capturekit-0.1.0/tests/unit/test_pipeline.py +184 -0
- capturekit-0.1.0/tests/unit/test_recorder_lifecycle.py +277 -0
- capturekit-0.1.0/tests/windows/__init__.py +0 -0
- capturekit-0.1.0/tests/windows/conftest.py +5 -0
- capturekit-0.1.0/tests/windows/test_audio.py +100 -0
- capturekit-0.1.0/tests/windows/test_recording.py +252 -0
- capturekit-0.1.0/tests/windows/test_recording_audio.py +193 -0
- capturekit-0.1.0/tests/windows/test_video.py +215 -0
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: architecture-security-reviewer
|
|
3
|
+
description: Architecture and security reviewer for capturekit — checks layering rules, public API typing and consistency, thread-safety, resource cleanup, native memory safety (ctypes pointers, buffer bounds, handle validation), subprocess safety, and absence of stealth/bypass behaviour. Use for reviewing significant changes or new public APIs.
|
|
4
|
+
tools: Read, Grep, Glob, Bash
|
|
5
|
+
model: inherit
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
You review capturekit for architecture and security. Rules come from `CLAUDE.md` and `docs/architecture.md`.
|
|
9
|
+
|
|
10
|
+
Check:
|
|
11
|
+
- Layering: public modules don't import backends at import time; platform specifics stay in `_backends/`.
|
|
12
|
+
- Thread safety: shared state guarded; COM objects used on their owning thread; user callbacks never run on capture threads; shutdown joins threads with timeouts and cannot deadlock.
|
|
13
|
+
- Native safety: argtypes/restype set, HRESULTs checked, no use-after-release, numpy views never outlive mapped memory, crop regions bounds-checked, handles validated.
|
|
14
|
+
- Subprocess: list args, no shell, untrusted paths never interpreted as options (e.g. a filename starting with '-').
|
|
15
|
+
- Ethics/security: recording only via explicit API calls, no persistence, no bypass of capture exclusion/DRM/OS permissions.
|
|
16
|
+
- Errors are never swallowed.
|
|
17
|
+
|
|
18
|
+
Read-only; do not edit or delegate. Output ranked findings with file:line, severity and a concrete fix.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: audio-systems-specialist
|
|
3
|
+
description: Audio systems specialist for capturekit — WASAPI (endpoint/process loopback, microphones, sessions), PulseAudio/PipeWire streams, sample formats, drift and resampling, device disconnection. Use for implementing or reviewing audio capture code and diagnosing audio glitches or isolation problems.
|
|
4
|
+
tools: Read, Grep, Glob, Edit, Write, Bash, PowerShell, WebSearch, WebFetch
|
|
5
|
+
model: inherit
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
You are the audio systems specialist for capturekit. Follow `.claude/skills/audio-capture-routing/SKILL.md` and `.claude/skills/av-sync/SKILL.md`.
|
|
9
|
+
|
|
10
|
+
Rules:
|
|
11
|
+
- Application audio capture must never silently fall back to system-wide audio. Unsupported → typed error.
|
|
12
|
+
- Never change default devices, volumes, or persistent routing. Temporary routing changes require the caller's explicit opt-in.
|
|
13
|
+
- Verify isolation with generated tones at distinct frequencies (e.g. 440 Hz target, 1000 Hz distractor) and spectral checks — not by listening.
|
|
14
|
+
- Run tests with the project venv; report real results including skips with reasons.
|
|
15
|
+
- Do not delegate to other agents. Keep final reports under 80 lines: changes, evidence, open issues.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: linux-x11-specialist
|
|
3
|
+
description: Linux X11/Xvfb and Linux audio (PulseAudio/PipeWire) specialist for capturekit — XShm, XRandR, XComposite/XDamage/XFixes window capture, EWMH enumeration, Xvfb test setups in WSL, Wayland detection, and per-application audio stream capture. Use for implementing/debugging/reviewing Linux backends or running Linux tests in WSL.
|
|
4
|
+
tools: Read, Grep, Glob, Edit, Write, Bash, WebSearch, WebFetch
|
|
5
|
+
model: inherit
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
You are the Linux/X11 specialist for capturekit. Follow `.claude/skills/linux-x11-backend/SKILL.md` and `.claude/skills/audio-capture-routing/SKILL.md`.
|
|
9
|
+
|
|
10
|
+
Rules:
|
|
11
|
+
- Window capture must use XComposite redirection — never root-window cropping presented as window capture. Document compositor/WM interactions you observe.
|
|
12
|
+
- Always target an explicit display string; never assume $DISPLAY is the desired server.
|
|
13
|
+
- Run Linux tests in WSL: `wsl -d Ubuntu -- bash -lc "..."`. Start private Xvfb servers on unused display numbers (e.g. :97-:99) with `-nolisten tcp` and kill them when done.
|
|
14
|
+
- Do NOT use sudo, install packages, or modify system audio routing/configuration. If a package is missing, report the exact apt package name instead.
|
|
15
|
+
- Do not delegate to other agents.
|
|
16
|
+
- Report: files changed, commands run with real outputs (pass/fail/skip counts), remaining limitations. Never claim untested behaviour works.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: performance-reviewer
|
|
3
|
+
description: Independent performance reviewer for capturekit pipelines — reviews code for unnecessary copies, allocations, per-frame Python overhead, unbounded buffers, blocking calls and timing errors, and runs the benchmark scripts to measure. Use after significant changes to capture, pacing, encoding or audio paths.
|
|
4
|
+
tools: Read, Grep, Glob, Bash, PowerShell
|
|
5
|
+
model: inherit
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
You review capturekit for performance. Follow `.claude/skills/performance-profiling/SKILL.md`.
|
|
9
|
+
|
|
10
|
+
- Read the changed code paths end to end (backend acquire → pacer → queue → encoder writer).
|
|
11
|
+
- Identify: extra GPU/CPU copies, per-frame allocations of large buffers, Python loops over pixels/samples, locks held across I/O, unbounded queues/lists, sleep-based pacing without absolute deadlines.
|
|
12
|
+
- Run benchmarks (`scripts/bench_capture.py`, `scripts/bench_record.py`) when hardware is available; report exact commands and measured output. Never estimate numbers and present them as measured.
|
|
13
|
+
- Read-only: do not edit files. Do not delegate.
|
|
14
|
+
- Output: ranked findings (impact, evidence, file:line, suggested fix), then measured results.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: test-validation-reviewer
|
|
3
|
+
description: Independent test and validation reviewer for capturekit — checks that tests actually verify the claimed behaviour (exclusive window capture under occlusion, process audio isolation, A/V sync), that skips are explicit, and that docs/capabilities claims match executed tests. Use before declaring a feature done.
|
|
4
|
+
tools: Read, Grep, Glob, Bash, PowerShell
|
|
5
|
+
model: inherit
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
You are a skeptical test reviewer for capturekit. Follow `.claude/skills/test-development/SKILL.md`.
|
|
9
|
+
|
|
10
|
+
- For each capability in `docs/capabilities.md` marked Implemented, find the test(s) proving it and check the assertions are meaningful (e.g. the occlusion test also proves the occluder was really on top; the audio isolation test checks the distractor tone is absent).
|
|
11
|
+
- Run the relevant tests with `-rs` and report pass/fail/skip counts with skip reasons. A skipped test is not evidence.
|
|
12
|
+
- Flag: tests that cannot fail, mocked-away native behaviour presented as integration coverage, missing cleanup, timing-dependent flakiness.
|
|
13
|
+
- Read-only: do not edit files. Do not delegate.
|
|
14
|
+
- Output: claims → evidence status (proven / weak / unproven), then concrete missing tests.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: windows-api-researcher
|
|
3
|
+
description: Researches and verifies Windows native APIs (DXGI, D3D11, Windows Graphics Capture/WinRT ABI, WASAPI, Application Loopback, DWM, Win32 windowing) for capturekit — signatures, vtable order, IIDs, minimum builds and undocumented behaviour — and runs small ctypes experiments to confirm. Use before implementing or changing a Windows binding, or when Windows behaviour is unclear.
|
|
4
|
+
tools: Read, Grep, Glob, WebSearch, WebFetch, Bash, PowerShell, Write
|
|
5
|
+
model: inherit
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
You are the Windows native API researcher for capturekit (Python, ctypes-only bindings).
|
|
9
|
+
|
|
10
|
+
Follow `.claude/skills/native-api-research/SKILL.md`. Responsibilities:
|
|
11
|
+
- Answer the specific question you were given with cited sources (Microsoft Learn, SDK headers, official samples). Mark anything unverified as UNVERIFIED.
|
|
12
|
+
- When a fact can be tested, write a minimal experiment script in the session scratchpad (never in src/) and run it with the project venv `.venv/Scripts/python -X faulthandler`. Report the exact output.
|
|
13
|
+
- For COM/WinRT interfaces give: IID, inheritance chain, full vtable slot list with indices, ctypes argtypes.
|
|
14
|
+
- You may only write files under the scratchpad or `docs/research/`. Do not edit src/, tests/, or skills — report findings so the main session can apply them.
|
|
15
|
+
- Never change system settings, install software, or disable security features. Do not delegate to other agents.
|
|
16
|
+
|
|
17
|
+
Final report: findings (with sources), experiment commands + outputs, and recommended code-level actions. Keep it under 80 lines.
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: audio-capture-routing
|
|
3
|
+
description: Procedures for capturekit audio sources — WASAPI microphone, endpoint loopback and process loopback on Windows; PulseAudio/PipeWire devices, monitor sources and per-application streams on Linux; window-to-process-to-stream mapping. Use when touching audio backends or audio source selection.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Audio capture and stream routing
|
|
7
|
+
|
|
8
|
+
## Uniform internal format
|
|
9
|
+
Every backend delivers `AudioPacket(data: float32[frames, 2], timestamp_ns,
|
|
10
|
+
flags)` at 48 kHz stereo. Conversion happens in the OS where possible
|
|
11
|
+
(WASAPI `AUTOCONVERTPCM | SRC_DEFAULT_QUALITY`; Pulse/PipeWire stream format).
|
|
12
|
+
Never resample in Python except the tiny drift correction in `av-sync`.
|
|
13
|
+
|
|
14
|
+
## Windows (WASAPI, `_backends/windows/wasapi.py`)
|
|
15
|
+
| Source | Activation | Flags |
|
|
16
|
+
| --- | --- | --- |
|
|
17
|
+
| Microphone | `IMMDevice::Activate(IAudioClient)` on eCapture endpoint | `EVENTCALLBACK` |
|
|
18
|
+
| System audio | eRender endpoint | `LOOPBACK | EVENTCALLBACK` |
|
|
19
|
+
| Process audio | `ActivateAudioInterfaceAsync("VAD\\Process_Loopback", IAudioClient, VT_BLOB AUDIOCLIENT_ACTIVATION_PARAMS)` | `LOOPBACK | EVENTCALLBACK` |
|
|
20
|
+
|
|
21
|
+
Quirks (verified — keep this list current):
|
|
22
|
+
- Endpoint loopback delivers **no packets while nothing is playing**. The
|
|
23
|
+
aligner heartbeat must fill silence; never treat the gap as a stall.
|
|
24
|
+
- Process loopback `IAudioClient::GetMixFormat` returns `E_NOTIMPL`; pass an
|
|
25
|
+
explicit WAVEFORMATEX (float32, 48 kHz, 2 ch). Requires build ≥ 20348.
|
|
26
|
+
- Process loopback completion handler must implement `IAgileObject`
|
|
27
|
+
(it is called on an MTA worker thread).
|
|
28
|
+
- `INCLUDE_TARGET_PROCESS_TREE` captures the target and its descendants —
|
|
29
|
+
needed for browsers whose audio comes from a child process. A PID with no
|
|
30
|
+
audio produces silence, not an error; report "no active session" via
|
|
31
|
+
`list_audio_sessions()` so callers can detect it.
|
|
32
|
+
- Process loopback must never fall back to endpoint loopback on failure:
|
|
33
|
+
raise `CapabilityUnavailableError`.
|
|
34
|
+
- `AUDCLNT_BUFFERFLAGS_SILENT` (0x2): treat buffer as zeros.
|
|
35
|
+
`DATA_DISCONTINUITY` (0x1): count a glitch; aligner realigns by timestamp.
|
|
36
|
+
- `u64QPCPosition` is in 100 ns units of QPC → `* 100` to ns.
|
|
37
|
+
- Device invalidation: `AUDCLNT_E_DEVICE_INVALIDATED (0x88890004)` → raise
|
|
38
|
+
`AudioDeviceDisconnectedError` via the error channel.
|
|
39
|
+
|
|
40
|
+
- Sessions report the *rendering* PID. The venv `python.exe` launcher spawns
|
|
41
|
+
the real interpreter as a child, so a fixture's PID differs from its
|
|
42
|
+
session PID; use `_win32.process_tree(pid)` when matching (tested).
|
|
43
|
+
|
|
44
|
+
Window → audio: `Window.pid` → `ProcessAudioSource(pid, include_tree=True)`.
|
|
45
|
+
Ambiguity (launcher processes, shared audio hosts) is resolved by the caller
|
|
46
|
+
choosing a PID from `list_audio_sessions()` (IAudioSessionManager2 sessions
|
|
47
|
+
with their owning PIDs).
|
|
48
|
+
|
|
49
|
+
## Linux
|
|
50
|
+
See `docs/research/linux-x11-and-audio.md` for verified API details.
|
|
51
|
+
- Enumerate via `pactl -f json list sinks|sources|sink-inputs` (works on
|
|
52
|
+
PulseAudio ≥ 16 and pipewire-pulse) — fallback to text parsing.
|
|
53
|
+
- System audio: record the default sink's `.monitor` source.
|
|
54
|
+
- App audio: per sink-input capture (`parec --monitor-stream=<idx>`); multiple
|
|
55
|
+
sink-inputs from one PID → one capture per stream, mixed by `AudioMixer`.
|
|
56
|
+
New streams appearing mid-recording are polled and attached.
|
|
57
|
+
- Never move user streams or change default devices without explicit opt-in.
|
|
58
|
+
- Xvfb has no audio; audio availability is independent of the X display.
|
|
59
|
+
|
|
60
|
+
## Tests
|
|
61
|
+
`pytest -m audio`. Process-loopback test plays a generated tone from a child
|
|
62
|
+
process (`tests/fixtures/tone_player.py`) and asserts tone energy at the
|
|
63
|
+
expected frequency in the captured track and absence of a second tone played
|
|
64
|
+
by another process.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: av-sync
|
|
3
|
+
description: Audio/video synchronization design and verification for capturekit — shared clock, pause-aware timeline, CFR frame pacing, audio gap filling, overlap trimming, drift correction and how to measure A/V offset in recorded files. Use when changing _pipeline/ timing code or investigating sync drift.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Audio/video synchronisation
|
|
7
|
+
|
|
8
|
+
Code: `src/capturekit/_pipeline/` — `clock.py`, `timeline.py`, `pacer.py`,
|
|
9
|
+
`aligner.py`, `mixer.py`.
|
|
10
|
+
|
|
11
|
+
## Model
|
|
12
|
+
- One clock (`now_ns`). Every frame/packet timestamp is converted to it at the
|
|
13
|
+
backend boundary.
|
|
14
|
+
- `Timeline.media_ns(wall_ns)` maps wall time to media time, excluding paused
|
|
15
|
+
intervals; returns `None` while paused.
|
|
16
|
+
- Video: slot `n` covers media time `[n*P, (n+1)*P)`; pacer fires at
|
|
17
|
+
`wall(t0) + n*P` (absolute). FFmpeg rawvideo timestamps = slot index.
|
|
18
|
+
- Audio: sample `k` ↔ media time `k / 48000`. Aligner computes expected
|
|
19
|
+
position `p = media_ns(packet.ts) * 48000 / 1e9`, compares with samples
|
|
20
|
+
written `w`:
|
|
21
|
+
- `p - w > gap_threshold (20 ms)` → insert silence `p - w`.
|
|
22
|
+
- `w - p > overlap_threshold (20 ms)` → trim overlap from packet head.
|
|
23
|
+
- otherwise → drift correction: resample packet to `n + c` samples where
|
|
24
|
+
`c = clamp(k_p * (p - w), ±0.5% n)` (proportional controller).
|
|
25
|
+
- Heartbeat: if no packet for > 100 ms, fill silence up to
|
|
26
|
+
`media_now - 60 ms` (loopback silence, device stall).
|
|
27
|
+
|
|
28
|
+
## Invariants to test
|
|
29
|
+
1. Total audio samples ≈ total video slots × 48000 / fps (± one packet).
|
|
30
|
+
2. Pause of D seconds adds 0 to both durations.
|
|
31
|
+
3. A clock running 100 ppm fast for 1 h stays within ±20 ms
|
|
32
|
+
(`tests/unit/test_aligner.py` simulates).
|
|
33
|
+
|
|
34
|
+
## Measuring sync in a real file
|
|
35
|
+
1. Record a flash+beep fixture (`tests/fixtures/av_sync_beacon.py`: window
|
|
36
|
+
flips black→white and plays a 1 kHz beep at the same QPC instant).
|
|
37
|
+
2. Extract: `ffprobe -show_frames` for video luminance (or decode with FFmpeg to
|
|
38
|
+
rawvideo gray and find the step), audio onset by envelope threshold.
|
|
39
|
+
3. Offset = audio_onset − video_onset; report mean and drift over time.
|
|
40
|
+
Acceptable: |offset| ≤ 1 frame period + 10 ms, drift ≤ 5 ms/min.
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: bug-investigation
|
|
3
|
+
description: Root-cause analysis procedure for capturekit bugs — crashes in ctypes/COM code, hangs/deadlocks in pipelines, black or stale frames, A/V desync, encoder failures and resource leaks. Use when something fails and the cause is not immediately obvious.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Bug investigation and root-cause analysis
|
|
7
|
+
|
|
8
|
+
## Procedure
|
|
9
|
+
1. **Reproduce** minimally (script in scratchpad or a failing test). Record
|
|
10
|
+
exact command, environment (OS build, GPU, FFmpeg version).
|
|
11
|
+
2. **Classify** and follow the matching playbook below.
|
|
12
|
+
3. **Hypothesise → test one variable at a time.** Write down what each
|
|
13
|
+
experiment rules out.
|
|
14
|
+
4. **Fix the cause, not the symptom** (no retries/sleeps to hide races).
|
|
15
|
+
5. **Regression test** that fails before and passes after.
|
|
16
|
+
6. **Capture the lesson** in the owning skill's quirks section.
|
|
17
|
+
|
|
18
|
+
## Playbooks
|
|
19
|
+
### Interpreter crash (access violation, no traceback)
|
|
20
|
+
- Run `python -X faulthandler`; the native stack points at the call.
|
|
21
|
+
- Check: vtable index, `argtypes` (pointer vs value struct, 64-bit handles
|
|
22
|
+
as `c_void_p` not `c_int`), lifetime (released COM object, numpy view of
|
|
23
|
+
unmapped memory, ctypes callback garbage-collected while native code still
|
|
24
|
+
holds it — keep a strong reference on the owning object).
|
|
25
|
+
|
|
26
|
+
### Hang / deadlock
|
|
27
|
+
- `faulthandler.dump_traceback_later(30)` or `py-spy dump --pid`.
|
|
28
|
+
- Typical: FFmpeg waiting on an unconnected pipe; writer blocked on full pipe
|
|
29
|
+
while FFmpeg blocks on stderr (must be drained); queue `put()` without
|
|
30
|
+
timeout during shutdown; joining a thread from itself.
|
|
31
|
+
|
|
32
|
+
### Black / stale frames
|
|
33
|
+
- WGC: protected window (display affinity), minimized window, wrong HWND
|
|
34
|
+
(host vs content window), DRM. DXGI: wrong adapter, ACCESS_LOST not handled,
|
|
35
|
+
frame with `LastPresentTime == 0` (pointer-only update — no new image).
|
|
36
|
+
|
|
37
|
+
### Desync
|
|
38
|
+
- Compare video slot count vs audio sample count in `stats`; check
|
|
39
|
+
`audio_silence_inserted_ms` / `trimmed_ms`; see av-sync skill.
|
|
40
|
+
|
|
41
|
+
### Leaks
|
|
42
|
+
- Run the capture open/close loop test (`tests/windows/test_lifecycle.py`)
|
|
43
|
+
watching handle count (`psutil.Process().num_handles()`) and GPU memory.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: docs-release
|
|
3
|
+
description: Documentation conventions and release preparation for capturekit — which docs to update for which change, capability-matrix honesty rules, changelog, versioning, packaging checks (build, twine check, wheel contents). Use when updating docs or preparing a release. Publishing requires explicit user approval.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Documentation and release preparation
|
|
7
|
+
|
|
8
|
+
## Doc map
|
|
9
|
+
| File | Update when |
|
|
10
|
+
| --- | --- |
|
|
11
|
+
| `README.md` | install / quickstart / feature summary changes |
|
|
12
|
+
| `docs/capabilities.md` | any feature status change (implemented / experimental / untested / unsupported) |
|
|
13
|
+
| `docs/architecture.md` | design changes |
|
|
14
|
+
| `docs/decisions.md` | new decision or dependency |
|
|
15
|
+
| `docs/guide/*.md` | user-facing API or platform setup changes |
|
|
16
|
+
| `docs/PROGRESS.md` | every working session |
|
|
17
|
+
| `CHANGELOG.md` | every user-visible change (Keep a Changelog format) |
|
|
18
|
+
|
|
19
|
+
## Capability honesty rules
|
|
20
|
+
- **Implemented**: real backend + tests executed on that platform.
|
|
21
|
+
- **Experimental**: works in manual/limited tests; known gaps listed.
|
|
22
|
+
- **Untested**: code exists but not executed on that platform/config.
|
|
23
|
+
- **Unsupported**: not implemented or out of scope (raises a typed error).
|
|
24
|
+
A stub or interface alone is never "implemented".
|
|
25
|
+
|
|
26
|
+
## Examples
|
|
27
|
+
Every code example in docs must be runnable; keep them in `examples/` and
|
|
28
|
+
reference them from docs. Run `python examples/<x>.py --help` (or a smoke
|
|
29
|
+
mode) before release.
|
|
30
|
+
|
|
31
|
+
## Release checklist (do NOT publish without user approval)
|
|
32
|
+
1. `ruff check . && ruff format --check . && mypy src`
|
|
33
|
+
2. Full test run on Windows + Xvfb (WSL); paste the summary (passed/skipped/
|
|
34
|
+
failed counts and skip reasons) into `docs/PROGRESS.md`.
|
|
35
|
+
3. Bump `src/capturekit/_version.py`; update CHANGELOG.
|
|
36
|
+
4. `python -m build` → `twine check dist/*`; inspect wheel contents
|
|
37
|
+
(`python -m zipfile -l dist/*.whl`) — no tests, no scratch files.
|
|
38
|
+
5. Install wheel in a fresh venv, run `examples/list_sources.py`.
|
|
39
|
+
6. Ask the user before `twine upload` / tagging.
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: ffmpeg-encoding
|
|
3
|
+
description: FFmpeg discovery, hardware encoder probing (NVENC/QSV/AMF/x264/MediaFoundation), command construction, named-pipe audio inputs, process lifecycle and failure handling for capturekit recordings. Use when editing src/capturekit/encoding/ or diagnosing encoder failures.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# FFmpeg and hardware encoding
|
|
7
|
+
|
|
8
|
+
Code: `src/capturekit/encoding/ffmpeg.py` (discovery, probe, command builder,
|
|
9
|
+
`FFmpegProcess`), `encoding/pipes.py` (named pipes / FIFOs).
|
|
10
|
+
|
|
11
|
+
## Discovery order
|
|
12
|
+
explicit `EncoderConfig.ffmpeg_path` → `$CAPTUREKIT_FFMPEG` → `PATH` →
|
|
13
|
+
`imageio_ffmpeg.get_ffmpeg_exe()` (optional dep). Record version via
|
|
14
|
+
`ffmpeg -version`.
|
|
15
|
+
|
|
16
|
+
## Encoder validation (never trust `-encoders` alone)
|
|
17
|
+
```
|
|
18
|
+
ffmpeg -hide_banner -loglevel error -f lavfi -i color=c=black:s=256x256:r=30
|
|
19
|
+
-frames:v 3 -c:v <encoder> -f null -
|
|
20
|
+
```
|
|
21
|
+
Exit 0 → usable. Cache per (ffmpeg path, mtime). Measured 2026-10-11 on RTX
|
|
22
|
+
4050 laptop: `h264_nvenc` OK; `h264_qsv`, `h264_amf` listed but fail.
|
|
23
|
+
NVENC minimum size is ~145x49 on some generations — probe at 256x256 and
|
|
24
|
+
reject smaller recordings for NVENC (fall back to libx264).
|
|
25
|
+
|
|
26
|
+
## Command shape
|
|
27
|
+
```
|
|
28
|
+
ffmpeg -hide_banner -nostdin -loglevel warning -stats_period 1 -progress pipe:2?
|
|
29
|
+
-f rawvideo -pix_fmt bgra -video_size WxH -framerate FPS -i pipe:0
|
|
30
|
+
[-f f32le -ar 48000 -ac 2 -i \\.\pipe\capturekit-<uuid>-a0]...
|
|
31
|
+
-map 0:v [-map 1:a ...] -c:v <enc> <enc opts> -c:a aac -b:a 192k
|
|
32
|
+
-metadata:s:a:i title=<track name> <out.mkv>
|
|
33
|
+
```
|
|
34
|
+
- Hardware encoders take BGRA directly (GPU colour conversion). libx264 needs
|
|
35
|
+
`-pix_fmt yuv420p` output (swscale on CPU — measure cost).
|
|
36
|
+
- Always `-pix_fmt yuv420p` (or nv12 for hw) on output for player compatibility.
|
|
37
|
+
- Rate control: NVENC `-rc vbr -b:v B -maxrate 1.5B -bufsize 2B -preset p4`;
|
|
38
|
+
x264 `-preset veryfast -b:v B -maxrate -bufsize`; or CQP/CRF when bitrate=None.
|
|
39
|
+
- MKV is the default (survives crashes). MP4 gets `-movflags +faststart`? No:
|
|
40
|
+
faststart rewrites the whole file at the end — omit by default for long
|
|
41
|
+
recordings; offer fragmented MP4 (`+frag_keyframe+empty_moov`) as option.
|
|
42
|
+
|
|
43
|
+
## Process handling
|
|
44
|
+
- `subprocess.Popen(list_args, stdin=PIPE, stderr=PIPE, stdout=DEVNULL,
|
|
45
|
+
creationflags=CREATE_NO_WINDOW)` — never `shell=True`.
|
|
46
|
+
- Drain stderr on a thread into a bounded deque (last 200 lines).
|
|
47
|
+
- Shutdown: close stdin and audio pipes → wait(timeout) → on timeout send
|
|
48
|
+
`q`? (stdin closed so no) → `terminate()` → `kill()`. Report exit code and
|
|
49
|
+
stderr tail in `EncoderError`.
|
|
50
|
+
- `BrokenPipeError` on write = FFmpeg died: collect stderr, raise EncoderError.
|
|
51
|
+
|
|
52
|
+
## Named pipes (Windows)
|
|
53
|
+
`CreateNamedPipeW(r"\\.\pipe\capturekit-<uuid4>", PIPE_ACCESS_OUTBOUND,
|
|
54
|
+
PIPE_TYPE_BYTE|PIPE_WAIT, 1, 1 MiB, 1 MiB, 0, NULL)` → start FFmpeg →
|
|
55
|
+
`ConnectNamedPipe` in the writer thread (blocks until FFmpeg opens it). FFmpeg
|
|
56
|
+
opens inputs in order, so every writer thread must be running before FFmpeg
|
|
57
|
+
starts probing. Linux: `os.mkfifo` in a private temp dir (mode 0600).
|
|
58
|
+
|
|
59
|
+
## Troubleshooting
|
|
60
|
+
| Symptom | Cause |
|
|
61
|
+
| --- | --- |
|
|
62
|
+
| hangs at start | an input pipe never connected/written; check writer threads started |
|
|
63
|
+
| `Invalid argument` with nvenc | odd width/height → we round crop to even dimensions |
|
|
64
|
+
| A/V offset constant | wrong timeline start for audio; see av-sync skill |
|
|
65
|
+
| file unplayable after crash | MP4 without fragmentation; use MKV |
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: linux-x11-backend
|
|
3
|
+
description: How to develop and test capturekit's Linux X11/Xvfb video backend — explicit display selection, XRandR monitor discovery with fallbacks, XShm capture, XComposite exclusive window capture, EWMH enumeration, X error handling, Wayland/XWayland detection, and running tests against private Xvfb servers in WSL. Use when editing src/capturekit/_backends/linux/.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Linux X11 / Xvfb backend
|
|
7
|
+
|
|
8
|
+
Reference (sources + API sequences): `docs/research/linux-x11-and-audio.md`.
|
|
9
|
+
|
|
10
|
+
## Display selection rules
|
|
11
|
+
- Every public call accepts `display: str | None`. `None` → `$DISPLAY`, but
|
|
12
|
+
only after Wayland checks; an explicit string (`":99"`) bypasses the host
|
|
13
|
+
session check and targets that server directly.
|
|
14
|
+
- Wayland-only (`WAYLAND_DISPLAY` set, no usable X server requested) →
|
|
15
|
+
`UnsupportedDisplayServerError`. If the default `$DISPLAY` is XWayland
|
|
16
|
+
(server has the `XWAYLAND` extension) → also unsupported for desktop
|
|
17
|
+
capture; say so in the message. WSLg sets `WAYLAND_DISPLAY=wayland-0` and
|
|
18
|
+
`DISPLAY=:0` (XWayland) — tests must always pass explicit Xvfb displays.
|
|
19
|
+
|
|
20
|
+
## Monitors
|
|
21
|
+
RRGetMonitors (RandR ≥ 1.5) → CRTC enumeration (RandR ≥ 1.2) → Xinerama →
|
|
22
|
+
root window geometry. Xvfb exposes one CRTC/output named "screen"; with no
|
|
23
|
+
CRTCs GetMonitors returns empty — fall back, never crash.
|
|
24
|
+
|
|
25
|
+
## Capture
|
|
26
|
+
- Monitor/region: `XShmGetImage` on the root window when MIT-SHM is present
|
|
27
|
+
(local connections only), else `XGetImage`. Mark the SHM segment
|
|
28
|
+
`IPC_RMID` immediately after attach; set `image.data = NULL` before
|
|
29
|
+
`XDestroyImage`.
|
|
30
|
+
- Window: `XCompositeRedirectWindow(dpy, w, CompositeRedirectAutomatic)`
|
|
31
|
+
(never Manual — a compositor may own it), then `XCompositeNameWindowPixmap`
|
|
32
|
+
→ capture from the pixmap. Re-name the pixmap after every ConfigureNotify
|
|
33
|
+
(resize) and MapNotify. Use the top-level *frame* or the client window per
|
|
34
|
+
`client_area_only`.
|
|
35
|
+
- Unmapped/iconified windows: pixmap is not available (BadMatch). Report
|
|
36
|
+
`TargetUnavailable`-style state; never substitute desktop pixels.
|
|
37
|
+
- FFmpeg x11grab `-window_id` is NOT exclusive capture (plain GetImage on the
|
|
38
|
+
window; obscured areas undefined). Do not use it as a window backend.
|
|
39
|
+
|
|
40
|
+
## Verified quirks (Xvfb 21.x / libX11 1.8.13, Ubuntu 26.04, 2026-10-11)
|
|
41
|
+
- `XShmGetImage`/`XGetImage` on a **pixmap** return an XImage with zero
|
|
42
|
+
colour masks (the reply has no visual for pixmaps). Validate the pixel
|
|
43
|
+
layout from the `Visual` struct at grabber creation, never from the image.
|
|
44
|
+
- Xvfb (stock Ubuntu build) offers Composite, DAMAGE, MIT-SHM, RANDR (one
|
|
45
|
+
monitor named `screen`), X-Resource (PIDs work), XFIXES, XINERAMA.
|
|
46
|
+
- WSLg PulseAudio 17 (`PULSE_SERVER=unix:/mnt/wslg/PulseServer`) supports
|
|
47
|
+
`pactl -f json`; its sink is `RDPSink` (audible on the Windows host).
|
|
48
|
+
|
|
49
|
+
## Xlib safety
|
|
50
|
+
- Install `XSetErrorHandler` (default handler calls `exit()`); handler is
|
|
51
|
+
process-global — record errors per-display, check after `XSync`.
|
|
52
|
+
- One `Display*` per capture thread; libX11 ≥ 1.8 auto-calls `XInitThreads`.
|
|
53
|
+
- 32-bit format properties come back as C `long` (8 bytes on x86-64).
|
|
54
|
+
|
|
55
|
+
## Enumeration
|
|
56
|
+
`_NET_CLIENT_LIST` (EWMH) → title `_NET_WM_NAME` (UTF8_STRING) / `WM_NAME`,
|
|
57
|
+
pid `_NET_WM_PID` or XRes `XResQueryClientIds`, `WM_CLASS`. No WM (bare
|
|
58
|
+
Xvfb): `XQueryTree(root)` children that are mapped InputOutput windows.
|
|
59
|
+
|
|
60
|
+
## Testing in WSL
|
|
61
|
+
Packages needed (ask user before installing):
|
|
62
|
+
`xvfb x11-utils x11-apps ffmpeg pulseaudio-utils libpulse0`.
|
|
63
|
+
```
|
|
64
|
+
wsl -d Ubuntu -- bash -lc "Xvfb :99 -screen 0 1280x720x24 -nolisten tcp & sleep 1; cd '/mnt/d/User/Documents/Projects/Python Projects/Cross-Platform Python Screen and Audio Capture Library' && python3 -m pytest -m xvfb -rs; kill %1"
|
|
65
|
+
```
|
|
66
|
+
Tests start their own Xvfb via the `xvfb_display` fixture on a free display
|
|
67
|
+
number; never reuse :0 in WSL.
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: native-api-research
|
|
3
|
+
description: Procedure for researching and verifying Windows/X11/PipeWire native API behaviour (signatures, vtable order, GUIDs, minimum OS builds, undocumented behaviour) before implementing it in capturekit. Use before writing any new ctypes binding or relying on a platform behaviour.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Native API research and verification
|
|
7
|
+
|
|
8
|
+
Goal: every native fact the code relies on is either (a) cited from an
|
|
9
|
+
authoritative source or (b) verified by an experiment on this machine — and
|
|
10
|
+
recorded so nobody re-derives it.
|
|
11
|
+
|
|
12
|
+
## Source priority
|
|
13
|
+
1. Microsoft Learn API reference (`learn.microsoft.com/windows/win32/api/...`,
|
|
14
|
+
`learn.microsoft.com/uwp/api/...`) — signatures, minimum builds, contracts.
|
|
15
|
+
2. Windows SDK headers (vtable order is the declaration order in the header,
|
|
16
|
+
including inherited interfaces). Mirror: `github.com/tpn/winsdk-10`, or
|
|
17
|
+
the `windows-rs`/`win32metadata` repos.
|
|
18
|
+
3. Official samples: `microsoft/Windows-classic-samples`
|
|
19
|
+
(ApplicationLoopback), `robmikh/Win32CaptureSample` (WGC, by the WGC
|
|
20
|
+
owner), `microsoft/Windows-universal-samples` (ScreenCaptureforHWND).
|
|
21
|
+
4. X.org protocol specs (`gitlab.freedesktop.org/xorg/proto/xorgproto`),
|
|
22
|
+
libX11/libXcomposite man pages, PipeWire/PulseAudio docs.
|
|
23
|
+
5. OBS Studio sources (libobs-winrt, win-capture, linux-capture) — reference
|
|
24
|
+
for real-world quirks, not authority.
|
|
25
|
+
|
|
26
|
+
Blog posts / StackOverflow are leads only. Mark unverified claims
|
|
27
|
+
`UNVERIFIED` in docs until tested.
|
|
28
|
+
|
|
29
|
+
## Procedure for a new COM / WinRT binding
|
|
30
|
+
1. Find the interface IID and full inheritance chain in the header.
|
|
31
|
+
2. Count vtable slots: IUnknown = 0..2; IInspectable adds 3..5; then the
|
|
32
|
+
interface's own methods in declaration order (WinRT property getters
|
|
33
|
+
`get_X`, setters `put_X`, events `add_X`/`remove_X` each take a slot).
|
|
34
|
+
3. Write the binding in `_backends/windows/` using `ComPtr.call(index,
|
|
35
|
+
restype, argtypes, *args)` and add the index to
|
|
36
|
+
`windows-capture-backend/reference/vtables.md`.
|
|
37
|
+
4. Experiment: minimal script in the scratchpad that calls the method and
|
|
38
|
+
checks HRESULT == 0 and plausible output. Never trust an index untested —
|
|
39
|
+
a wrong index usually returns garbage or crashes the interpreter.
|
|
40
|
+
5. Parameterised WinRT interfaces (e.g. `TypedEventHandler<A,B>`): compute
|
|
41
|
+
the IID with `capturekit._backends.windows._winrt.pinterface_iid`
|
|
42
|
+
(UUIDv5 of the signature string, namespace
|
|
43
|
+
`11f47ad5-7b73-42c0-abae-878b1e16adee`) and cross-check with a known value.
|
|
44
|
+
|
|
45
|
+
## Procedure for an OS-version-dependent feature
|
|
46
|
+
1. Record the minimum build from Learn's Requirements table.
|
|
47
|
+
2. Implement runtime detection by *trying* the API (QI for the interface,
|
|
48
|
+
`ApiInformation`-style checks) — build numbers are a hint only.
|
|
49
|
+
3. Expose the result in `get_capabilities()` with a human-readable reason.
|
|
50
|
+
|
|
51
|
+
## Recording findings
|
|
52
|
+
- Verified facts and quirks → the relevant backend skill's reference file.
|
|
53
|
+
- Design-affecting facts → `docs/decisions.md`.
|
|
54
|
+
- Unknown/unverifiable behaviour → `docs/capabilities.md` "untested" column.
|
|
55
|
+
|
|
56
|
+
## Verified facts log (append; include date + build)
|
|
57
|
+
- 2026-10-11, Win11 26200: `time.perf_counter_ns()` == QPC ns (delta ~1 µs);
|
|
58
|
+
QPC frequency 10 MHz.
|
|
59
|
+
- `AUDIOCLIENT_PROCESS_LOOPBACK_PARAMS` min client: Windows 10 build 20348
|
|
60
|
+
(Learn). Mode 0 = include target process tree, 1 = exclude.
|
|
61
|
+
- `GraphicsCaptureSession` 1803; `IsCursorCaptureEnabled` 2004 (19041);
|
|
62
|
+
`CreateForWindow` interop 1903. `IsBorderRequired`: 20348/Win11.
|
|
63
|
+
- Microsoft does not document WGC behaviour for minimized windows → measured
|
|
64
|
+
2026-10-11: no frames while minimized, resume on restore
|
|
65
|
+
(`tests/windows/test_video.py::test_window_minimized_behaviour`).
|
|
66
|
+
- Guessing IIDs from memory failed once (IDXGIOutput1). Always copy IIDs from
|
|
67
|
+
headers (MinGW-w64 `mingw-w64-headers/include/*.idl` is fetchable as raw text).
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: performance-profiling
|
|
3
|
+
description: How to measure and optimise capturekit capture/encode throughput — benchmark scripts, per-stage timing, py-spy/cProfile usage, known hot spots (GPU readback, memcpy, pipe bandwidth, swscale), and rules for reporting numbers honestly. Use when evaluating FPS targets (24/30/60+) or optimising a pipeline stage.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Performance profiling and optimisation
|
|
7
|
+
|
|
8
|
+
## Rules
|
|
9
|
+
- Numbers in docs must include: command, git commit, machine (CPU/GPU),
|
|
10
|
+
resolution, fps, encoder, date. Never estimate and present as measured.
|
|
11
|
+
- Measure first with `scripts/bench_capture.py` and the `Recorder.stats`
|
|
12
|
+
counters before changing code. Optimise the dominant stage only.
|
|
13
|
+
|
|
14
|
+
## Commands
|
|
15
|
+
```
|
|
16
|
+
.venv/Scripts/python scripts/bench_capture.py --source monitor --fps 60 --seconds 10
|
|
17
|
+
.venv/Scripts/python scripts/bench_capture.py --source window --hwnd <id> --fps 60
|
|
18
|
+
.venv/Scripts/python scripts/bench_record.py --fps 60 --encoder auto --seconds 20
|
|
19
|
+
python -X importtime -c "import capturekit"
|
|
20
|
+
py-spy record -o profile.svg -- python scripts/bench_record.py ... (if installed)
|
|
21
|
+
```
|
|
22
|
+
`stats` fields to watch: `capture_ms_p50/p99` (acquire+copy), `pacer_jitter_ms`,
|
|
23
|
+
`frames_late`, `frames_dropped`, `queue_depth_max`, `encoder_write_ms`.
|
|
24
|
+
|
|
25
|
+
## Known cost centres (1080p BGRA = 8.3 MB/frame)
|
|
26
|
+
| Stage | Notes |
|
|
27
|
+
| --- | --- |
|
|
28
|
+
| GPU→staging copy + Map | Map stalls until copy done; staging ring of 2–3 textures hides latency |
|
|
29
|
+
| numpy row copy | ~1–2 ms/frame; unavoidable single copy |
|
|
30
|
+
| pipe to FFmpeg | 60 fps × 8.3 MB ≈ 500 MB/s; use large writes (whole frame) |
|
|
31
|
+
| colour conversion | hw encoders take BGRA (GPU CSC); libx264 needs swscale (CPU) |
|
|
32
|
+
| Python per-frame overhead | keep per-frame Python ops O(1), no per-pixel loops |
|
|
33
|
+
|
|
34
|
+
## Optimisation backlog (evaluate with measurements before doing)
|
|
35
|
+
1. Staging texture ring to avoid Map stalls.
|
|
36
|
+
2. GPU BGRA→NV12 via D3D11 VideoProcessor (halves pipe bandwidth).
|
|
37
|
+
3. Shared-memory transport instead of pipe.
|
|
38
|
+
4. Skip readback entirely when the frame is unchanged (duplicate slot).
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: test-development
|
|
3
|
+
description: How to write, organise, run and interpret capturekit tests — unit vs platform integration vs performance tests, pytest markers and skip rules, deterministic fixture windows and tone players, pixel verification for exclusive window capture, and regression triage. Use when adding tests or analysing test failures.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Test development and regression analysis
|
|
7
|
+
|
|
8
|
+
## Layout
|
|
9
|
+
```
|
|
10
|
+
tests/unit/ pure Python, run everywhere (timeline, aligner, config,
|
|
11
|
+
ffmpeg command builder, region validation)
|
|
12
|
+
tests/windows/ marker windows (+gpu/audio/ffmpeg as needed)
|
|
13
|
+
tests/linux/ markers x11 / xvfb
|
|
14
|
+
tests/perf/ marker slow; prints measured numbers, asserts loose floors
|
|
15
|
+
tests/fixtures/ helper programs run as subprocesses
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Markers (pyproject `[tool.pytest.ini_options]`)
|
|
19
|
+
`windows, x11, xvfb, gpu, audio, ffmpeg, slow, interactive`.
|
|
20
|
+
`conftest.py` auto-skips when requirements are missing **with a reason**.
|
|
21
|
+
Report skipped counts explicitly: a skip is not a pass.
|
|
22
|
+
|
|
23
|
+
## Commands
|
|
24
|
+
```
|
|
25
|
+
.venv/Scripts/python -m pytest # all available
|
|
26
|
+
.venv/Scripts/python -m pytest tests/unit -q # fast
|
|
27
|
+
.venv/Scripts/python -m pytest -m "windows and not slow" -rs
|
|
28
|
+
.venv/Scripts/python -m pytest -m slow -s # perf, shows numbers
|
|
29
|
+
wsl -d Ubuntu -- bash -lc 'cd /mnt/d/... && python3 -m pytest -m xvfb -rs'
|
|
30
|
+
```
|
|
31
|
+
Always use `-rs` when reporting so skip reasons are visible.
|
|
32
|
+
|
|
33
|
+
## Exclusive-window-capture verification pattern
|
|
34
|
+
1. Spawn `tests/fixtures/color_window.py --color 0,200,0 --title <uuid>` (target).
|
|
35
|
+
2. Spawn an occluder window (`--color 200,0,0 --topmost`) overlapping it.
|
|
36
|
+
3. Find target by title, capture via `WindowCapture`.
|
|
37
|
+
4. Assert ≥ 99% of client-area pixels equal the target colour (±2 per channel)
|
|
38
|
+
and 0% match the occluder colour.
|
|
39
|
+
5. Also verify a *desktop* capture of the same area does show the occluder —
|
|
40
|
+
proves the test is meaningful (the occluder really covers the target).
|
|
41
|
+
|
|
42
|
+
## Regression analysis procedure
|
|
43
|
+
1. Re-run the failing test alone with `-x -vv -s`.
|
|
44
|
+
2. Determine flakiness: `pytest --count`-style loop (`for i in 1..10`).
|
|
45
|
+
3. Bisect with `git bisect run python -m pytest <test>` when history exists.
|
|
46
|
+
4. Write the regression test first, then fix (bug-investigation skill).
|
|
47
|
+
5. Record root cause in `docs/PROGRESS.md` test log if non-obvious.
|