percept-vision 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- percept_vision-0.1.0/PKG-INFO +235 -0
- percept_vision-0.1.0/README.md +198 -0
- percept_vision-0.1.0/percept/__init__.py +113 -0
- percept_vision-0.1.0/percept/adapters.py +369 -0
- percept_vision-0.1.0/percept/agent.py +982 -0
- percept_vision-0.1.0/percept/backends/__init__.py +21 -0
- percept_vision-0.1.0/percept/backends/_resilience.py +77 -0
- percept_vision-0.1.0/percept/backends/anthropic.py +291 -0
- percept_vision-0.1.0/percept/backends/anthropic_text.py +260 -0
- percept_vision-0.1.0/percept/backends/deepgram.py +130 -0
- percept_vision-0.1.0/percept/backends/fake.py +146 -0
- percept_vision-0.1.0/percept/backends/gemini.py +263 -0
- percept_vision-0.1.0/percept/behavior.py +543 -0
- percept_vision-0.1.0/percept/cache/__init__.py +15 -0
- percept_vision-0.1.0/percept/cache/cascade.py +62 -0
- percept_vision-0.1.0/percept/cache/framehash.py +338 -0
- percept_vision-0.1.0/percept/concern.py +122 -0
- percept_vision-0.1.0/percept/config.py +192 -0
- percept_vision-0.1.0/percept/consent.py +72 -0
- percept_vision-0.1.0/percept/entity.py +220 -0
- percept_vision-0.1.0/percept/episodes.py +86 -0
- percept_vision-0.1.0/percept/escalation.py +69 -0
- percept_vision-0.1.0/percept/eval/__init__.py +108 -0
- percept_vision-0.1.0/percept/events.py +117 -0
- percept_vision-0.1.0/percept/gate.py +83 -0
- percept_vision-0.1.0/percept/goals.py +93 -0
- percept_vision-0.1.0/percept/identity.py +89 -0
- percept_vision-0.1.0/percept/observability.py +70 -0
- percept_vision-0.1.0/percept/persistence.py +61 -0
- percept_vision-0.1.0/percept/registry.py +139 -0
- percept_vision-0.1.0/percept/routing/__init__.py +23 -0
- percept_vision-0.1.0/percept/routing/engine.py +156 -0
- percept_vision-0.1.0/percept/routing/rules.py +216 -0
- percept_vision-0.1.0/percept/scheduler.py +89 -0
- percept_vision-0.1.0/percept_vision.egg-info/PKG-INFO +235 -0
- percept_vision-0.1.0/percept_vision.egg-info/SOURCES.txt +81 -0
- percept_vision-0.1.0/percept_vision.egg-info/dependency_links.txt +1 -0
- percept_vision-0.1.0/percept_vision.egg-info/requires.txt +24 -0
- percept_vision-0.1.0/percept_vision.egg-info/top_level.txt +1 -0
- percept_vision-0.1.0/pyproject.toml +44 -0
- percept_vision-0.1.0/setup.cfg +4 -0
- percept_vision-0.1.0/tests/test_acoustic.py +81 -0
- percept_vision-0.1.0/tests/test_action_verification.py +103 -0
- percept_vision-0.1.0/tests/test_audit.py +160 -0
- percept_vision-0.1.0/tests/test_box_iou.py +42 -0
- percept_vision-0.1.0/tests/test_caffeine_trace.py +97 -0
- percept_vision-0.1.0/tests/test_cascade.py +72 -0
- percept_vision-0.1.0/tests/test_concern.py +111 -0
- percept_vision-0.1.0/tests/test_conductor_authoring.py +106 -0
- percept_vision-0.1.0/tests/test_conductor_parse.py +56 -0
- percept_vision-0.1.0/tests/test_consent.py +74 -0
- percept_vision-0.1.0/tests/test_consequence.py +92 -0
- percept_vision-0.1.0/tests/test_deadline.py +68 -0
- percept_vision-0.1.0/tests/test_entity.py +75 -0
- percept_vision-0.1.0/tests/test_episodes.py +75 -0
- percept_vision-0.1.0/tests/test_escalation.py +167 -0
- percept_vision-0.1.0/tests/test_events.py +114 -0
- percept_vision-0.1.0/tests/test_firing_cr.py +88 -0
- percept_vision-0.1.0/tests/test_framehash.py +192 -0
- percept_vision-0.1.0/tests/test_frames.py +57 -0
- percept_vision-0.1.0/tests/test_gate.py +63 -0
- percept_vision-0.1.0/tests/test_gemini_backend.py +63 -0
- percept_vision-0.1.0/tests/test_goals.py +58 -0
- percept_vision-0.1.0/tests/test_golden_loader.py +63 -0
- percept_vision-0.1.0/tests/test_identity.py +103 -0
- percept_vision-0.1.0/tests/test_novel_tasks.py +111 -0
- percept_vision-0.1.0/tests/test_nth_distinct.py +85 -0
- percept_vision-0.1.0/tests/test_observability.py +83 -0
- percept_vision-0.1.0/tests/test_perception_assembly.py +91 -0
- percept_vision-0.1.0/tests/test_perception_judge_wiring.py +63 -0
- percept_vision-0.1.0/tests/test_perception_spec.py +133 -0
- percept_vision-0.1.0/tests/test_persistence.py +88 -0
- percept_vision-0.1.0/tests/test_program_equivalence.py +258 -0
- percept_vision-0.1.0/tests/test_rearm_equivalence.py +223 -0
- percept_vision-0.1.0/tests/test_registry.py +81 -0
- percept_vision-0.1.0/tests/test_reliability.py +94 -0
- percept_vision-0.1.0/tests/test_replay_align.py +157 -0
- percept_vision-0.1.0/tests/test_routing.py +126 -0
- percept_vision-0.1.0/tests/test_scheduler.py +73 -0
- percept_vision-0.1.0/tests/test_shapes.py +55 -0
- percept_vision-0.1.0/tests/test_transition.py +113 -0
- percept_vision-0.1.0/tests/test_turn.py +163 -0
- percept_vision-0.1.0/tests/test_voice_backends.py +66 -0
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: percept-vision
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Research preview — the open-source cognition layer for goal-driven, proactive vision agents.
|
|
5
|
+
Author-email: Divi <divi@velvee.ai>
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Keywords: vision,agents,proactive,perception,cognition,vlm,video,real-time
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
17
|
+
Classifier: Topic :: Multimedia :: Video
|
|
18
|
+
Requires-Python: >=3.10
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
Provides-Extra: claude
|
|
21
|
+
Requires-Dist: anthropic>=0.40; extra == "claude"
|
|
22
|
+
Provides-Extra: gemini
|
|
23
|
+
Requires-Dist: google-genai>=1.0; extra == "gemini"
|
|
24
|
+
Provides-Extra: deepgram
|
|
25
|
+
Requires-Dist: deepgram-sdk>=3; extra == "deepgram"
|
|
26
|
+
Provides-Extra: audio
|
|
27
|
+
Requires-Dist: numpy>=1.24; extra == "audio"
|
|
28
|
+
Provides-Extra: ui
|
|
29
|
+
Provides-Extra: all
|
|
30
|
+
Requires-Dist: anthropic>=0.40; extra == "all"
|
|
31
|
+
Requires-Dist: google-genai>=1.0; extra == "all"
|
|
32
|
+
Requires-Dist: deepgram-sdk>=3; extra == "all"
|
|
33
|
+
Requires-Dist: numpy>=1.24; extra == "all"
|
|
34
|
+
Provides-Extra: dev
|
|
35
|
+
Requires-Dist: pytest>=8; extra == "dev"
|
|
36
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
37
|
+
|
|
38
|
+
# percept
|
|
39
|
+
|
|
40
|
+
The open-source **cognition layer** for goal-driven, proactive vision agents.
|
|
41
|
+
|
|
42
|
+
You state a goal in plain language — *"nudge me when I drink a coffee"*, *"tell me when the kettle
|
|
43
|
+
boils"* — and percept turns a live audio-visual stream into an agent that:
|
|
44
|
+
|
|
45
|
+
- **reasons over entities across time** — tracked things with stable ids, attributes carrying
|
|
46
|
+
provenance (*observed* vs *you-told-me*) and freshness;
|
|
47
|
+
- **fires proactively**, but only on the **rising edge** of a condition becoming true;
|
|
48
|
+
- **refuses to guess** — a three-state gate maps *known → act · not → silent · unknown → ask*, so it
|
|
49
|
+
never delivers a confidently-wrong nag.
|
|
50
|
+
|
|
51
|
+
The wedge is **temporal cognition** (entity memory + the three-state gate + reasoning over events),
|
|
52
|
+
which a raw VLM-in-a-loop and a per-frame pipeline both lack. percept builds on frontier models for
|
|
53
|
+
perception behind vendor-neutral seams.
|
|
54
|
+
|
|
55
|
+
> **⚠️ Research preview — v0.1.0.** Published for real-life testing and feedback, **not** for
|
|
56
|
+
> production. The cognition core (gate, entity graph, executor, events, scheduler, consent) runs and is
|
|
57
|
+
> tested; the benchmark card below is a **v0.5 DRAFT**. APIs may change between `0.1.x` releases —
|
|
58
|
+
> pin a version. Issues and feedback welcome. ([Status](#status).)
|
|
59
|
+
|
|
60
|
+
## The envelope (refusals, stated proudly)
|
|
61
|
+
|
|
62
|
+
- **Assistant-class, never the safety mechanism.** percept informs a human who stays responsible. It
|
|
63
|
+
is never the thing standing between a person and harm on a clock it can't guarantee. *Out:*
|
|
64
|
+
driver-drowsiness intervention, turn detection. *In:* a post-hoc driving debrief.
|
|
65
|
+
- **Watches the user's own world, with the user as beneficiary** — never a non-consenting third party
|
|
66
|
+
to the wearer's advantage. *Out:* covert analysis of someone you're negotiating with; card-counting
|
|
67
|
+
against a live casino. *In:* a play-money practice trainer.
|
|
68
|
+
|
|
69
|
+
These refusals are a feature. **Do not lead with surveillance demos.**
|
|
70
|
+
|
|
71
|
+
## Install
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
pip install percept-vision # core: PURE STDLIB — runs offline with fake backends, no keys
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
The **core has zero dependencies** and runs with deterministic fake backends, so `pip install` then
|
|
78
|
+
run works with no API keys. Frontier backends are opt-in extras:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
pip install "percept-vision[gemini]" # GeminiVision
|
|
82
|
+
pip install "percept-vision[claude]" # AnthropicVision (Claude)
|
|
83
|
+
pip install "percept-vision[deepgram]" # Deepgram STT + TTS (voice)
|
|
84
|
+
pip install "percept-vision[audio]" # numpy fast-path for acoustic/edge
|
|
85
|
+
pip install "percept-vision[all]" # everything
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Python **>= 3.10**. The package name is `percept-vision`; the import is `import percept`.
|
|
89
|
+
|
|
90
|
+
## 60-second quickstart (no keys)
|
|
91
|
+
|
|
92
|
+
Fully offline — fake backends, deterministic, nothing to configure. This script runs as-is:
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
import asyncio
|
|
96
|
+
from percept import Percept, Goal
|
|
97
|
+
|
|
98
|
+
async def main():
|
|
99
|
+
# Fake backends by default — offline, no keys. discover_plugins=False skips plugin lookup.
|
|
100
|
+
agent = Percept.create(discover_plugins=False)
|
|
101
|
+
|
|
102
|
+
agent.add_goal(Goal(
|
|
103
|
+
id="caffeine",
|
|
104
|
+
condition="the user is drinking coffee",
|
|
105
|
+
say="Heads up — stepping back from caffeine?",
|
|
106
|
+
))
|
|
107
|
+
|
|
108
|
+
# Each frame is judged; the gate fires ONCE on the rising edge.
|
|
109
|
+
# ("sip-coffee" is a token the fake vision backend scripts as a confident YES.)
|
|
110
|
+
fires = await agent.perceive_judged("sip-coffee")
|
|
111
|
+
for ev in fires: # ev is a FireEvent
|
|
112
|
+
print(ev.action, ev.goal_id, ev.text) # -> fire caffeine Heads up — stepping back from caffeine?
|
|
113
|
+
|
|
114
|
+
# The same frame again does NOT re-fire — rising-edge, not level-triggered.
|
|
115
|
+
print(await agent.perceive_judged("sip-coffee")) # -> []
|
|
116
|
+
|
|
117
|
+
asyncio.run(main())
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
`perceive_judged(frame)` returns a list of `FireEvent(goal_id, action, text, entity_id, verdict)`,
|
|
121
|
+
where `action` is `"fire"` or `"ask"`. With real eyes, swap the fake for a frontier vision backend —
|
|
122
|
+
the cognition layer above is unchanged:
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
agent = Percept.create(vision="gemini") # needs percept-vision[gemini] + GEMINI_API_KEY
|
|
126
|
+
# frame is now real image bytes; everything downstream (gate, graph, executor) is identical.
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Backends are selected by name (`"fake"` · `"gemini"` · `"claude"`) or by passing an adapter instance,
|
|
130
|
+
and can also be set via env (`PERCEPT_VISION_BACKEND`, etc.).
|
|
131
|
+
|
|
132
|
+
## Architecture — two layers
|
|
133
|
+
|
|
134
|
+
percept cleanly splits the **eyes** from the **brain**. Perception is one stateless seam;
|
|
135
|
+
everything stateful and proactive is cognition.
|
|
136
|
+
|
|
137
|
+
```
|
|
138
|
+
┌───────────────────────── COGNITION — the "brain" ──────────────────────────┐
|
|
139
|
+
│ three-state GATE fire (known-yes) · ask (unknown) · silent (known-no) │
|
|
140
|
+
│ ▲ rising edge: fires only on false→true, with a refractory (no nag) │
|
|
141
|
+
│ EXECUTOR one firing path (sense · key · accumulate); │
|
|
142
|
+
│ transitions A→B, counting, deadlines/absence, verify │
|
|
143
|
+
│ ENTITY GRAPH stable ids; attributes with provenance + freshness │
|
|
144
|
+
│ (observed vs you-told-me) │
|
|
145
|
+
│ EDGE DETECTOR REGISTRY opt-in cheap signals propose; the brain counts/gates │
|
|
146
|
+
│ general · deterministic · cheap · STATEFUL (has memory) │
|
|
147
|
+
└──────────────────────────────────────────────────────────────────────────────┘
|
|
148
|
+
▲ feeds on Verdict{satisfied, confidence}
|
|
149
|
+
┌───────────────────────── PERCEPTION — the "eyes" ──────────────────────────┐
|
|
150
|
+
│ vision.judge(condition, frame) -> Verdict ★ ONE seam │
|
|
151
|
+
│ GeminiVision / AnthropicVision · stateless · one frame · ~1s/call cloud │
|
|
152
|
+
└──────────────────────────────────────────────────────────────────────────────┘
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
The **gate** turns a noisy verdict stream into at most one alert on the rising edge, and falls back
|
|
156
|
+
to **ASK** rather than guess when confidence is unreliable. The **executor** is the single firing
|
|
157
|
+
path for every concern shape (watch, transition, count, deadline/absence, verification). The
|
|
158
|
+
**entity graph** carries memory: stable ids and attributes that know whether they were *observed* or
|
|
159
|
+
*asserted by the user*, and whether they're still fresh.
|
|
160
|
+
|
|
161
|
+
The **edge detector registry** is the *edge-proposes · brain-counts · VLM-confirms* split: cheap,
|
|
162
|
+
opt-in detectors emit timed boundaries that the brain accumulates — counting reps without spending a
|
|
163
|
+
vision token per frame. Three reference skills ship as registered detectors:
|
|
164
|
+
|
|
165
|
+
- **`motion-periodicity`** — frame-diff motion peaks (rep boundaries);
|
|
166
|
+
- **`acoustic-onset`** — energy spikes on the existing mic stream (no new capture path);
|
|
167
|
+
- **`pose-openness`** — BlazePose-based rep peaks (opt-in: `percept-harness[pose]`, MediaPipe).
|
|
168
|
+
|
|
169
|
+
A full request trace — *"tell me when the milk is about to boil over"* — is in
|
|
170
|
+
[`docs/e2e-flow-milk-boilover.md`](docs/e2e-flow-milk-boilover.md), including an honest map of where
|
|
171
|
+
the perception ceiling bites.
|
|
172
|
+
|
|
173
|
+
## The three packages
|
|
174
|
+
|
|
175
|
+
| package | what | install |
|
|
176
|
+
|---|---|---|
|
|
177
|
+
| **percept-vision** | the cognition core — gate, entity graph, executor, events, scheduler, consent, fakes. Pure stdlib. | `pip install percept-vision` (`packages/percept-vision/`) |
|
|
178
|
+
| **percept-harness** | server-side transport shell + tier-0 salience gate (WatchSpec down / Tier0Signal up); the reference home of the edge detector skills. | `packages/percept-harness/` |
|
|
179
|
+
| **@percept/edge** | on-device reactive edge in JS/WASM: VAD + motion gate over the same WatchSpec/Tier0Signal wire-contract. | `packages/percept-edge/` (npm `@percept/edge`) |
|
|
180
|
+
|
|
181
|
+
## Benchmark — Percept Benchmark v1 (v0.5 DRAFT)
|
|
182
|
+
|
|
183
|
+
The benchmark holds the **backbone fixed and measures the orchestration delta** across a `raw → core
|
|
184
|
+
→ e2e` config ladder (no composite score — a vector of headlines). The current card is a **v0.5
|
|
185
|
+
DRAFT** (sampled, N≈12/track, gemini-2.5-flash) on the DeepMind **Perception Test** (CC-BY-4.0), the
|
|
186
|
+
private **golden** ambiguity corpus, and **RepCount-A**; the e2e-relational accuracy cell is modeled
|
|
187
|
+
(flagged `~`), so the card is stamped `DRAFT`.
|
|
188
|
+
|
|
189
|
+
| track | measured (DRAFT, N≈12) | reading |
|
|
190
|
+
|---|---|---|
|
|
191
|
+
| **counting** (RepCount-A) | pose OBO **0.33** vs **0.29** (TransRAC, CVPR'22); nMAE 0.80 vs 0.44 | training-free pose ties a *trained* baseline on OBO, but its failures on low-amplitude actions cost it on MAE |
|
|
192
|
+
| **acoustic** (PT Sound Loc., unseen source) | vision-only recall **0.00 → 0.25–0.33** fused; fusion-FP 0–0.17 | the **recall-flip**: fusion rescues ~a third of sound events vision-only never hears (honest at scale, not the single-clip 1.0) |
|
|
193
|
+
| **relational** (golden) | confidence **AUROC ≈ 0.51** (≈ chance); ASK-rate ≈ 0.4 | the VLM's confidence does **not** separate right from wrong → justifies the **ASK** discipline over threshold-tuning |
|
|
194
|
+
| **timeliness** (PT action onsets) | **P-PAUC ≈ 60**; reaction λ p50 ≈ 1s | first real proactive-timeliness numbers (adapted PAUC) |
|
|
195
|
+
| **edge event-recall** (PT onsets) | **0.00** @ thr 0.10 | ⚠️ the motion-gate escalates on **none** of the subtle-action onsets (motion ~0.03–0.06 < 0.10) → e2e ≠ core there; a real **calibration** finding |
|
|
196
|
+
|
|
197
|
+
A v0.5 DRAFT, not a RELEASE claim. Two findings the benchmark surfaced that a self-congratulatory
|
|
198
|
+
eval would hide: the **flat confidence AUROC** (*why* the gate refuses to guess) and the **edge
|
|
199
|
+
event-recall of 0** on subtle actions (the motion-gate is mis-calibrated for them). Full plan, data,
|
|
200
|
+
and references:
|
|
201
|
+
[`packages/percept-vision/eval/BENCHMARK_PLAN.md`](packages/percept-vision/eval/BENCHMARK_PLAN.md).
|
|
202
|
+
|
|
203
|
+
## Layout
|
|
204
|
+
|
|
205
|
+
```
|
|
206
|
+
packages/percept-vision/ the SDK (pip install percept-vision; import percept)
|
|
207
|
+
packages/percept-harness/ server-side tier-0 edge reference + detector skills
|
|
208
|
+
packages/percept-edge/ @percept/edge — on-device JS/WASM edge
|
|
209
|
+
docs/ architecture & flow traces
|
|
210
|
+
eval/ golden corpus (benchmark plan: packages/percept-vision/eval/)
|
|
211
|
+
Spec/ the spec + implementation plan
|
|
212
|
+
Makefile test · eval-live · e2e · bench · check · reproduce
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
## Docs
|
|
216
|
+
|
|
217
|
+
- **Quickstart** — the 60-second offline snippet above; `make test` runs the fake-only unit lane.
|
|
218
|
+
- **Architecture / concepts** — [`docs/e2e-flow-milk-boilover.md`](docs/e2e-flow-milk-boilover.md)
|
|
219
|
+
(the two layers, the firing path, the perception ceiling) and [`Spec/`](Spec/) (the full spec +
|
|
220
|
+
phase plan).
|
|
221
|
+
- **Backends** — select by name (`fake` · `gemini` · `claude` · `deepgram`) or env
|
|
222
|
+
(`PERCEPT_*_BACKEND`); add via the `percept.backends` entry-point group. Extras:
|
|
223
|
+
`[gemini]` · `[claude]` · `[deepgram]` · `[audio]`.
|
|
224
|
+
- **Benchmark** — [`eval/BENCHMARK_PLAN.md`](packages/percept-vision/eval/BENCHMARK_PLAN.md).
|
|
225
|
+
|
|
226
|
+
## Status
|
|
227
|
+
|
|
228
|
+
**v0.1.0 — early / first public release.** The cognition core runs and is tested offline with fakes
|
|
229
|
+
(the L1 lane, no keys); the frontier backends (Gemini, Claude, Deepgram) and the edge packages are
|
|
230
|
+
wired behind their seams. The benchmark is a **v0.5 DRAFT** card. We are now in **real-life
|
|
231
|
+
testing** — APIs and numbers may change. Issues and contributions welcome.
|
|
232
|
+
|
|
233
|
+
## License
|
|
234
|
+
|
|
235
|
+
Apache-2.0.
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
# percept
|
|
2
|
+
|
|
3
|
+
The open-source **cognition layer** for goal-driven, proactive vision agents.
|
|
4
|
+
|
|
5
|
+
You state a goal in plain language — *"nudge me when I drink a coffee"*, *"tell me when the kettle
|
|
6
|
+
boils"* — and percept turns a live audio-visual stream into an agent that:
|
|
7
|
+
|
|
8
|
+
- **reasons over entities across time** — tracked things with stable ids, attributes carrying
|
|
9
|
+
provenance (*observed* vs *you-told-me*) and freshness;
|
|
10
|
+
- **fires proactively**, but only on the **rising edge** of a condition becoming true;
|
|
11
|
+
- **refuses to guess** — a three-state gate maps *known → act · not → silent · unknown → ask*, so it
|
|
12
|
+
never delivers a confidently-wrong nag.
|
|
13
|
+
|
|
14
|
+
The wedge is **temporal cognition** (entity memory + the three-state gate + reasoning over events),
|
|
15
|
+
which a raw VLM-in-a-loop and a per-frame pipeline both lack. percept builds on frontier models for
|
|
16
|
+
perception behind vendor-neutral seams.
|
|
17
|
+
|
|
18
|
+
> **⚠️ Research preview — v0.1.0.** Published for real-life testing and feedback, **not** for
|
|
19
|
+
> production. The cognition core (gate, entity graph, executor, events, scheduler, consent) runs and is
|
|
20
|
+
> tested; the benchmark card below is a **v0.5 DRAFT**. APIs may change between `0.1.x` releases —
|
|
21
|
+
> pin a version. Issues and feedback welcome. ([Status](#status).)
|
|
22
|
+
|
|
23
|
+
## The envelope (refusals, stated proudly)
|
|
24
|
+
|
|
25
|
+
- **Assistant-class, never the safety mechanism.** percept informs a human who stays responsible. It
|
|
26
|
+
is never the thing standing between a person and harm on a clock it can't guarantee. *Out:*
|
|
27
|
+
driver-drowsiness intervention, turn detection. *In:* a post-hoc driving debrief.
|
|
28
|
+
- **Watches the user's own world, with the user as beneficiary** — never a non-consenting third party
|
|
29
|
+
to the wearer's advantage. *Out:* covert analysis of someone you're negotiating with; card-counting
|
|
30
|
+
against a live casino. *In:* a play-money practice trainer.
|
|
31
|
+
|
|
32
|
+
These refusals are a feature. **Do not lead with surveillance demos.**
|
|
33
|
+
|
|
34
|
+
## Install
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
pip install percept-vision # core: PURE STDLIB — runs offline with fake backends, no keys
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
The **core has zero dependencies** and runs with deterministic fake backends, so `pip install` then
|
|
41
|
+
run works with no API keys. Frontier backends are opt-in extras:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
pip install "percept-vision[gemini]" # GeminiVision
|
|
45
|
+
pip install "percept-vision[claude]" # AnthropicVision (Claude)
|
|
46
|
+
pip install "percept-vision[deepgram]" # Deepgram STT + TTS (voice)
|
|
47
|
+
pip install "percept-vision[audio]" # numpy fast-path for acoustic/edge
|
|
48
|
+
pip install "percept-vision[all]" # everything
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Python **>= 3.10**. The package name is `percept-vision`; the import is `import percept`.
|
|
52
|
+
|
|
53
|
+
## 60-second quickstart (no keys)
|
|
54
|
+
|
|
55
|
+
Fully offline — fake backends, deterministic, nothing to configure. This script runs as-is:
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
import asyncio
|
|
59
|
+
from percept import Percept, Goal
|
|
60
|
+
|
|
61
|
+
async def main():
|
|
62
|
+
# Fake backends by default — offline, no keys. discover_plugins=False skips plugin lookup.
|
|
63
|
+
agent = Percept.create(discover_plugins=False)
|
|
64
|
+
|
|
65
|
+
agent.add_goal(Goal(
|
|
66
|
+
id="caffeine",
|
|
67
|
+
condition="the user is drinking coffee",
|
|
68
|
+
say="Heads up — stepping back from caffeine?",
|
|
69
|
+
))
|
|
70
|
+
|
|
71
|
+
# Each frame is judged; the gate fires ONCE on the rising edge.
|
|
72
|
+
# ("sip-coffee" is a token the fake vision backend scripts as a confident YES.)
|
|
73
|
+
fires = await agent.perceive_judged("sip-coffee")
|
|
74
|
+
for ev in fires: # ev is a FireEvent
|
|
75
|
+
print(ev.action, ev.goal_id, ev.text) # -> fire caffeine Heads up — stepping back from caffeine?
|
|
76
|
+
|
|
77
|
+
# The same frame again does NOT re-fire — rising-edge, not level-triggered.
|
|
78
|
+
print(await agent.perceive_judged("sip-coffee")) # -> []
|
|
79
|
+
|
|
80
|
+
asyncio.run(main())
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
`perceive_judged(frame)` returns a list of `FireEvent(goal_id, action, text, entity_id, verdict)`,
|
|
84
|
+
where `action` is `"fire"` or `"ask"`. With real eyes, swap the fake for a frontier vision backend —
|
|
85
|
+
the cognition layer above is unchanged:
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
agent = Percept.create(vision="gemini") # needs percept-vision[gemini] + GEMINI_API_KEY
|
|
89
|
+
# frame is now real image bytes; everything downstream (gate, graph, executor) is identical.
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Backends are selected by name (`"fake"` · `"gemini"` · `"claude"`) or by passing an adapter instance,
|
|
93
|
+
and can also be set via env (`PERCEPT_VISION_BACKEND`, etc.).
|
|
94
|
+
|
|
95
|
+
## Architecture — two layers
|
|
96
|
+
|
|
97
|
+
percept cleanly splits the **eyes** from the **brain**. Perception is one stateless seam;
|
|
98
|
+
everything stateful and proactive is cognition.
|
|
99
|
+
|
|
100
|
+
```
|
|
101
|
+
┌───────────────────────── COGNITION — the "brain" ──────────────────────────┐
|
|
102
|
+
│ three-state GATE fire (known-yes) · ask (unknown) · silent (known-no) │
|
|
103
|
+
│ ▲ rising edge: fires only on false→true, with a refractory (no nag) │
|
|
104
|
+
│ EXECUTOR one firing path (sense · key · accumulate); │
|
|
105
|
+
│ transitions A→B, counting, deadlines/absence, verify │
|
|
106
|
+
│ ENTITY GRAPH stable ids; attributes with provenance + freshness │
|
|
107
|
+
│ (observed vs you-told-me) │
|
|
108
|
+
│ EDGE DETECTOR REGISTRY opt-in cheap signals propose; the brain counts/gates │
|
|
109
|
+
│ general · deterministic · cheap · STATEFUL (has memory) │
|
|
110
|
+
└──────────────────────────────────────────────────────────────────────────────┘
|
|
111
|
+
▲ feeds on Verdict{satisfied, confidence}
|
|
112
|
+
┌───────────────────────── PERCEPTION — the "eyes" ──────────────────────────┐
|
|
113
|
+
│ vision.judge(condition, frame) -> Verdict ★ ONE seam │
|
|
114
|
+
│ GeminiVision / AnthropicVision · stateless · one frame · ~1s/call cloud │
|
|
115
|
+
└──────────────────────────────────────────────────────────────────────────────┘
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
The **gate** turns a noisy verdict stream into at most one alert on the rising edge, and falls back
|
|
119
|
+
to **ASK** rather than guess when confidence is unreliable. The **executor** is the single firing
|
|
120
|
+
path for every concern shape (watch, transition, count, deadline/absence, verification). The
|
|
121
|
+
**entity graph** carries memory: stable ids and attributes that know whether they were *observed* or
|
|
122
|
+
*asserted by the user*, and whether they're still fresh.
|
|
123
|
+
|
|
124
|
+
The **edge detector registry** is the *edge-proposes · brain-counts · VLM-confirms* split: cheap,
|
|
125
|
+
opt-in detectors emit timed boundaries that the brain accumulates — counting reps without spending a
|
|
126
|
+
vision token per frame. Three reference skills ship as registered detectors:
|
|
127
|
+
|
|
128
|
+
- **`motion-periodicity`** — frame-diff motion peaks (rep boundaries);
|
|
129
|
+
- **`acoustic-onset`** — energy spikes on the existing mic stream (no new capture path);
|
|
130
|
+
- **`pose-openness`** — BlazePose-based rep peaks (opt-in: `percept-harness[pose]`, MediaPipe).
|
|
131
|
+
|
|
132
|
+
A full request trace — *"tell me when the milk is about to boil over"* — is in
|
|
133
|
+
[`docs/e2e-flow-milk-boilover.md`](docs/e2e-flow-milk-boilover.md), including an honest map of where
|
|
134
|
+
the perception ceiling bites.
|
|
135
|
+
|
|
136
|
+
## The three packages
|
|
137
|
+
|
|
138
|
+
| package | what | install |
|
|
139
|
+
|---|---|---|
|
|
140
|
+
| **percept-vision** | the cognition core — gate, entity graph, executor, events, scheduler, consent, fakes. Pure stdlib. | `pip install percept-vision` (`packages/percept-vision/`) |
|
|
141
|
+
| **percept-harness** | server-side transport shell + tier-0 salience gate (WatchSpec down / Tier0Signal up); the reference home of the edge detector skills. | `packages/percept-harness/` |
|
|
142
|
+
| **@percept/edge** | on-device reactive edge in JS/WASM: VAD + motion gate over the same WatchSpec/Tier0Signal wire-contract. | `packages/percept-edge/` (npm `@percept/edge`) |
|
|
143
|
+
|
|
144
|
+
## Benchmark — Percept Benchmark v1 (v0.5 DRAFT)
|
|
145
|
+
|
|
146
|
+
The benchmark holds the **backbone fixed and measures the orchestration delta** across a `raw → core
|
|
147
|
+
→ e2e` config ladder (no composite score — a vector of headlines). The current card is a **v0.5
|
|
148
|
+
DRAFT** (sampled, N≈12/track, gemini-2.5-flash) on the DeepMind **Perception Test** (CC-BY-4.0), the
|
|
149
|
+
private **golden** ambiguity corpus, and **RepCount-A**; the e2e-relational accuracy cell is modeled
|
|
150
|
+
(flagged `~`), so the card is stamped `DRAFT`.
|
|
151
|
+
|
|
152
|
+
| track | measured (DRAFT, N≈12) | reading |
|
|
153
|
+
|---|---|---|
|
|
154
|
+
| **counting** (RepCount-A) | pose OBO **0.33** vs **0.29** (TransRAC, CVPR'22); nMAE 0.80 vs 0.44 | training-free pose ties a *trained* baseline on OBO, but its failures on low-amplitude actions cost it on MAE |
|
|
155
|
+
| **acoustic** (PT Sound Loc., unseen source) | vision-only recall **0.00 → 0.25–0.33** fused; fusion-FP 0–0.17 | the **recall-flip**: fusion rescues ~a third of sound events vision-only never hears (honest at scale, not the single-clip 1.0) |
|
|
156
|
+
| **relational** (golden) | confidence **AUROC ≈ 0.51** (≈ chance); ASK-rate ≈ 0.4 | the VLM's confidence does **not** separate right from wrong → justifies the **ASK** discipline over threshold-tuning |
|
|
157
|
+
| **timeliness** (PT action onsets) | **P-PAUC ≈ 60**; reaction λ p50 ≈ 1s | first real proactive-timeliness numbers (adapted PAUC) |
|
|
158
|
+
| **edge event-recall** (PT onsets) | **0.00** @ thr 0.10 | ⚠️ the motion-gate escalates on **none** of the subtle-action onsets (motion ~0.03–0.06 < 0.10) → e2e ≠ core there; a real **calibration** finding |
|
|
159
|
+
|
|
160
|
+
A v0.5 DRAFT, not a RELEASE claim. Two findings the benchmark surfaced that a self-congratulatory
|
|
161
|
+
eval would hide: the **flat confidence AUROC** (*why* the gate refuses to guess) and the **edge
|
|
162
|
+
event-recall of 0** on subtle actions (the motion-gate is mis-calibrated for them). Full plan, data,
|
|
163
|
+
and references:
|
|
164
|
+
[`packages/percept-vision/eval/BENCHMARK_PLAN.md`](packages/percept-vision/eval/BENCHMARK_PLAN.md).
|
|
165
|
+
|
|
166
|
+
## Layout
|
|
167
|
+
|
|
168
|
+
```
|
|
169
|
+
packages/percept-vision/ the SDK (pip install percept-vision; import percept)
|
|
170
|
+
packages/percept-harness/ server-side tier-0 edge reference + detector skills
|
|
171
|
+
packages/percept-edge/ @percept/edge — on-device JS/WASM edge
|
|
172
|
+
docs/ architecture & flow traces
|
|
173
|
+
eval/ golden corpus (benchmark plan: packages/percept-vision/eval/)
|
|
174
|
+
Spec/ the spec + implementation plan
|
|
175
|
+
Makefile test · eval-live · e2e · bench · check · reproduce
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
## Docs
|
|
179
|
+
|
|
180
|
+
- **Quickstart** — the 60-second offline snippet above; `make test` runs the fake-only unit lane.
|
|
181
|
+
- **Architecture / concepts** — [`docs/e2e-flow-milk-boilover.md`](docs/e2e-flow-milk-boilover.md)
|
|
182
|
+
(the two layers, the firing path, the perception ceiling) and [`Spec/`](Spec/) (the full spec +
|
|
183
|
+
phase plan).
|
|
184
|
+
- **Backends** — select by name (`fake` · `gemini` · `claude` · `deepgram`) or env
|
|
185
|
+
(`PERCEPT_*_BACKEND`); add via the `percept.backends` entry-point group. Extras:
|
|
186
|
+
`[gemini]` · `[claude]` · `[deepgram]` · `[audio]`.
|
|
187
|
+
- **Benchmark** — [`eval/BENCHMARK_PLAN.md`](packages/percept-vision/eval/BENCHMARK_PLAN.md).
|
|
188
|
+
|
|
189
|
+
## Status
|
|
190
|
+
|
|
191
|
+
**v0.1.0 — early / first public release.** The cognition core runs and is tested offline with fakes
|
|
192
|
+
(the L1 lane, no keys); the frontier backends (Gemini, Claude, Deepgram) and the edge packages are
|
|
193
|
+
wired behind their seams. The benchmark is a **v0.5 DRAFT** card. We are now in **real-life
|
|
194
|
+
testing** — APIs and numbers may change. Issues and contributions welcome.
|
|
195
|
+
|
|
196
|
+
## License
|
|
197
|
+
|
|
198
|
+
Apache-2.0.
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"""percept — the open-source cognition layer for goal-driven, proactive vision agents.
|
|
2
|
+
|
|
3
|
+
Public surface. The reactive watch loop, the three-state gate, the entity graph, and the goal
|
|
4
|
+
registry are wired (Phase 1); identity/episodes, events/scheduler, the edge, and persistence land
|
|
5
|
+
in later phases *beside* this core. See ``Spec/`` for the what/why and the implementation plan.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from . import registry
|
|
11
|
+
from .adapters import (
|
|
12
|
+
Attribute,
|
|
13
|
+
Box,
|
|
14
|
+
Decision,
|
|
15
|
+
Goal,
|
|
16
|
+
Observation,
|
|
17
|
+
STTEvent,
|
|
18
|
+
Tier0Signal,
|
|
19
|
+
TurnResult,
|
|
20
|
+
Verdict,
|
|
21
|
+
WatchCondition,
|
|
22
|
+
WatchSpec,
|
|
23
|
+
)
|
|
24
|
+
from .cache import CacheCascade, CacheKey, CacheResult, FrameHashLayer
|
|
25
|
+
from .config import Settings, load_settings
|
|
26
|
+
from .consent import ActionLog, ActionRecord, ConsentDenied, ConsentProfile
|
|
27
|
+
from .entity import Entity, EntityGraph
|
|
28
|
+
from .episodes import Episode, EpisodeLog
|
|
29
|
+
from .events import Event, EventLog, FrameWindow
|
|
30
|
+
from .gate import ASK, FIRE, SILENT, Gate, GateAction
|
|
31
|
+
from .goals import GoalOp, GoalRegistry
|
|
32
|
+
from .identity import AppearanceIdentity, ContinuityIdentity
|
|
33
|
+
from .scheduler import Scheduler, Timer
|
|
34
|
+
from .observability import Metrics, get_logger, health
|
|
35
|
+
from .persistence import InMemoryPersistence, SqlitePersistence
|
|
36
|
+
from .routing import (
|
|
37
|
+
DEFAULT_RULES,
|
|
38
|
+
RegexRouteLayer,
|
|
39
|
+
RouteEngine,
|
|
40
|
+
RouteLayer,
|
|
41
|
+
RouteMatch,
|
|
42
|
+
Rule,
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
__version__ = "0.1.0"
|
|
46
|
+
|
|
47
|
+
# Imported after __version__ so agent.health() can read it lazily without a cycle.
|
|
48
|
+
from .agent import FireEvent, Percept # noqa: E402
|
|
49
|
+
|
|
50
|
+
__all__ = [
|
|
51
|
+
"__version__",
|
|
52
|
+
"Percept",
|
|
53
|
+
"FireEvent",
|
|
54
|
+
"registry",
|
|
55
|
+
# DTOs
|
|
56
|
+
"Box",
|
|
57
|
+
"Observation",
|
|
58
|
+
"Verdict",
|
|
59
|
+
"Attribute",
|
|
60
|
+
"Goal",
|
|
61
|
+
"Decision",
|
|
62
|
+
"STTEvent",
|
|
63
|
+
"TurnResult",
|
|
64
|
+
"WatchCondition",
|
|
65
|
+
"WatchSpec",
|
|
66
|
+
"Tier0Signal",
|
|
67
|
+
# cognition core
|
|
68
|
+
"Gate",
|
|
69
|
+
"GateAction",
|
|
70
|
+
"FIRE",
|
|
71
|
+
"ASK",
|
|
72
|
+
"SILENT",
|
|
73
|
+
"EntityGraph",
|
|
74
|
+
"Entity",
|
|
75
|
+
"GoalRegistry",
|
|
76
|
+
"GoalOp",
|
|
77
|
+
# identity + episodes (Phase 4 — re-id across gaps; the cup problem)
|
|
78
|
+
"ContinuityIdentity",
|
|
79
|
+
"AppearanceIdentity",
|
|
80
|
+
"Episode",
|
|
81
|
+
"EpisodeLog",
|
|
82
|
+
# events + scheduler (Phase 5 — temporal perception; time-triggered fires)
|
|
83
|
+
"Event",
|
|
84
|
+
"EventLog",
|
|
85
|
+
"FrameWindow",
|
|
86
|
+
"Scheduler",
|
|
87
|
+
"Timer",
|
|
88
|
+
# persistence + consent (Phase 7 — endurance + safe act modality)
|
|
89
|
+
"InMemoryPersistence",
|
|
90
|
+
"SqlitePersistence",
|
|
91
|
+
"ConsentProfile",
|
|
92
|
+
"ConsentDenied",
|
|
93
|
+
"ActionLog",
|
|
94
|
+
"ActionRecord",
|
|
95
|
+
# routing (Phase 3 — local fast-path; learned tier slots behind RouteLayer in V0.1)
|
|
96
|
+
"RouteEngine",
|
|
97
|
+
"RouteLayer",
|
|
98
|
+
"RouteMatch",
|
|
99
|
+
"RegexRouteLayer",
|
|
100
|
+
"Rule",
|
|
101
|
+
"DEFAULT_RULES",
|
|
102
|
+
# cache
|
|
103
|
+
"CacheCascade",
|
|
104
|
+
"CacheResult",
|
|
105
|
+
"CacheKey",
|
|
106
|
+
"FrameHashLayer",
|
|
107
|
+
# infra
|
|
108
|
+
"Settings",
|
|
109
|
+
"load_settings",
|
|
110
|
+
"Metrics",
|
|
111
|
+
"get_logger",
|
|
112
|
+
"health",
|
|
113
|
+
]
|