percept-vision 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. percept_vision-0.1.0/PKG-INFO +235 -0
  2. percept_vision-0.1.0/README.md +198 -0
  3. percept_vision-0.1.0/percept/__init__.py +113 -0
  4. percept_vision-0.1.0/percept/adapters.py +369 -0
  5. percept_vision-0.1.0/percept/agent.py +982 -0
  6. percept_vision-0.1.0/percept/backends/__init__.py +21 -0
  7. percept_vision-0.1.0/percept/backends/_resilience.py +77 -0
  8. percept_vision-0.1.0/percept/backends/anthropic.py +291 -0
  9. percept_vision-0.1.0/percept/backends/anthropic_text.py +260 -0
  10. percept_vision-0.1.0/percept/backends/deepgram.py +130 -0
  11. percept_vision-0.1.0/percept/backends/fake.py +146 -0
  12. percept_vision-0.1.0/percept/backends/gemini.py +263 -0
  13. percept_vision-0.1.0/percept/behavior.py +543 -0
  14. percept_vision-0.1.0/percept/cache/__init__.py +15 -0
  15. percept_vision-0.1.0/percept/cache/cascade.py +62 -0
  16. percept_vision-0.1.0/percept/cache/framehash.py +338 -0
  17. percept_vision-0.1.0/percept/concern.py +122 -0
  18. percept_vision-0.1.0/percept/config.py +192 -0
  19. percept_vision-0.1.0/percept/consent.py +72 -0
  20. percept_vision-0.1.0/percept/entity.py +220 -0
  21. percept_vision-0.1.0/percept/episodes.py +86 -0
  22. percept_vision-0.1.0/percept/escalation.py +69 -0
  23. percept_vision-0.1.0/percept/eval/__init__.py +108 -0
  24. percept_vision-0.1.0/percept/events.py +117 -0
  25. percept_vision-0.1.0/percept/gate.py +83 -0
  26. percept_vision-0.1.0/percept/goals.py +93 -0
  27. percept_vision-0.1.0/percept/identity.py +89 -0
  28. percept_vision-0.1.0/percept/observability.py +70 -0
  29. percept_vision-0.1.0/percept/persistence.py +61 -0
  30. percept_vision-0.1.0/percept/registry.py +139 -0
  31. percept_vision-0.1.0/percept/routing/__init__.py +23 -0
  32. percept_vision-0.1.0/percept/routing/engine.py +156 -0
  33. percept_vision-0.1.0/percept/routing/rules.py +216 -0
  34. percept_vision-0.1.0/percept/scheduler.py +89 -0
  35. percept_vision-0.1.0/percept_vision.egg-info/PKG-INFO +235 -0
  36. percept_vision-0.1.0/percept_vision.egg-info/SOURCES.txt +81 -0
  37. percept_vision-0.1.0/percept_vision.egg-info/dependency_links.txt +1 -0
  38. percept_vision-0.1.0/percept_vision.egg-info/requires.txt +24 -0
  39. percept_vision-0.1.0/percept_vision.egg-info/top_level.txt +1 -0
  40. percept_vision-0.1.0/pyproject.toml +44 -0
  41. percept_vision-0.1.0/setup.cfg +4 -0
  42. percept_vision-0.1.0/tests/test_acoustic.py +81 -0
  43. percept_vision-0.1.0/tests/test_action_verification.py +103 -0
  44. percept_vision-0.1.0/tests/test_audit.py +160 -0
  45. percept_vision-0.1.0/tests/test_box_iou.py +42 -0
  46. percept_vision-0.1.0/tests/test_caffeine_trace.py +97 -0
  47. percept_vision-0.1.0/tests/test_cascade.py +72 -0
  48. percept_vision-0.1.0/tests/test_concern.py +111 -0
  49. percept_vision-0.1.0/tests/test_conductor_authoring.py +106 -0
  50. percept_vision-0.1.0/tests/test_conductor_parse.py +56 -0
  51. percept_vision-0.1.0/tests/test_consent.py +74 -0
  52. percept_vision-0.1.0/tests/test_consequence.py +92 -0
  53. percept_vision-0.1.0/tests/test_deadline.py +68 -0
  54. percept_vision-0.1.0/tests/test_entity.py +75 -0
  55. percept_vision-0.1.0/tests/test_episodes.py +75 -0
  56. percept_vision-0.1.0/tests/test_escalation.py +167 -0
  57. percept_vision-0.1.0/tests/test_events.py +114 -0
  58. percept_vision-0.1.0/tests/test_firing_cr.py +88 -0
  59. percept_vision-0.1.0/tests/test_framehash.py +192 -0
  60. percept_vision-0.1.0/tests/test_frames.py +57 -0
  61. percept_vision-0.1.0/tests/test_gate.py +63 -0
  62. percept_vision-0.1.0/tests/test_gemini_backend.py +63 -0
  63. percept_vision-0.1.0/tests/test_goals.py +58 -0
  64. percept_vision-0.1.0/tests/test_golden_loader.py +63 -0
  65. percept_vision-0.1.0/tests/test_identity.py +103 -0
  66. percept_vision-0.1.0/tests/test_novel_tasks.py +111 -0
  67. percept_vision-0.1.0/tests/test_nth_distinct.py +85 -0
  68. percept_vision-0.1.0/tests/test_observability.py +83 -0
  69. percept_vision-0.1.0/tests/test_perception_assembly.py +91 -0
  70. percept_vision-0.1.0/tests/test_perception_judge_wiring.py +63 -0
  71. percept_vision-0.1.0/tests/test_perception_spec.py +133 -0
  72. percept_vision-0.1.0/tests/test_persistence.py +88 -0
  73. percept_vision-0.1.0/tests/test_program_equivalence.py +258 -0
  74. percept_vision-0.1.0/tests/test_rearm_equivalence.py +223 -0
  75. percept_vision-0.1.0/tests/test_registry.py +81 -0
  76. percept_vision-0.1.0/tests/test_reliability.py +94 -0
  77. percept_vision-0.1.0/tests/test_replay_align.py +157 -0
  78. percept_vision-0.1.0/tests/test_routing.py +126 -0
  79. percept_vision-0.1.0/tests/test_scheduler.py +73 -0
  80. percept_vision-0.1.0/tests/test_shapes.py +55 -0
  81. percept_vision-0.1.0/tests/test_transition.py +113 -0
  82. percept_vision-0.1.0/tests/test_turn.py +163 -0
  83. percept_vision-0.1.0/tests/test_voice_backends.py +66 -0
@@ -0,0 +1,235 @@
1
+ Metadata-Version: 2.4
2
+ Name: percept-vision
3
+ Version: 0.1.0
4
+ Summary: Research preview — the open-source cognition layer for goal-driven, proactive vision agents.
5
+ Author-email: Divi <divi@velvee.ai>
6
+ License: Apache-2.0
7
+ Keywords: vision,agents,proactive,perception,cognition,vlm,video,real-time
8
+ Classifier: Development Status :: 3 - Alpha
9
+ Classifier: Intended Audience :: Developers
10
+ Classifier: License :: OSI Approved :: Apache Software License
11
+ Classifier: Operating System :: OS Independent
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3.10
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
17
+ Classifier: Topic :: Multimedia :: Video
18
+ Requires-Python: >=3.10
19
+ Description-Content-Type: text/markdown
20
+ Provides-Extra: claude
21
+ Requires-Dist: anthropic>=0.40; extra == "claude"
22
+ Provides-Extra: gemini
23
+ Requires-Dist: google-genai>=1.0; extra == "gemini"
24
+ Provides-Extra: deepgram
25
+ Requires-Dist: deepgram-sdk>=3; extra == "deepgram"
26
+ Provides-Extra: audio
27
+ Requires-Dist: numpy>=1.24; extra == "audio"
28
+ Provides-Extra: ui
29
+ Provides-Extra: all
30
+ Requires-Dist: anthropic>=0.40; extra == "all"
31
+ Requires-Dist: google-genai>=1.0; extra == "all"
32
+ Requires-Dist: deepgram-sdk>=3; extra == "all"
33
+ Requires-Dist: numpy>=1.24; extra == "all"
34
+ Provides-Extra: dev
35
+ Requires-Dist: pytest>=8; extra == "dev"
36
+ Requires-Dist: ruff>=0.6; extra == "dev"
37
+
38
+ # percept
39
+
40
+ The open-source **cognition layer** for goal-driven, proactive vision agents.
41
+
42
+ You state a goal in plain language — *"nudge me when I drink a coffee"*, *"tell me when the kettle
43
+ boils"* — and percept turns a live audio-visual stream into an agent that:
44
+
45
+ - **reasons over entities across time** — tracked things with stable ids, attributes carrying
46
+ provenance (*observed* vs *you-told-me*) and freshness;
47
+ - **fires proactively**, but only on the **rising edge** of a condition becoming true;
48
+ - **refuses to guess** — a three-state gate maps *known → act · not → silent · unknown → ask*, so it
49
+ never delivers a confidently-wrong nag.
50
+
51
+ The wedge is **temporal cognition** (entity memory + the three-state gate + reasoning over events),
52
+ which a raw VLM-in-a-loop and a per-frame pipeline both lack. percept builds on frontier models for
53
+ perception behind vendor-neutral seams.
54
+
55
+ > **⚠️ Research preview — v0.1.0.** Published for real-life testing and feedback, **not** for
56
+ > production. The cognition core (gate, entity graph, executor, events, scheduler, consent) runs and is
57
+ > tested; the benchmark card below is a **v0.5 DRAFT**. APIs may change between `0.1.x` releases —
58
+ > pin a version. Issues and feedback welcome. ([Status](#status).)
59
+
60
+ ## The envelope (refusals, stated proudly)
61
+
62
+ - **Assistant-class, never the safety mechanism.** percept informs a human who stays responsible. It
63
+ is never the thing standing between a person and harm on a clock it can't guarantee. *Out:*
64
+ driver-drowsiness intervention, turn detection. *In:* a post-hoc driving debrief.
65
+ - **Watches the user's own world, with the user as beneficiary** — never a non-consenting third party
66
+ to the wearer's advantage. *Out:* covert analysis of someone you're negotiating with; card-counting
67
+ against a live casino. *In:* a play-money practice trainer.
68
+
69
+ These refusals are a feature. **Do not lead with surveillance demos.**
70
+
71
+ ## Install
72
+
73
+ ```bash
74
+ pip install percept-vision # core: PURE STDLIB — runs offline with fake backends, no keys
75
+ ```
76
+
77
+ The **core has zero dependencies** and runs with deterministic fake backends, so `pip install` then
78
+ run works with no API keys. Frontier backends are opt-in extras:
79
+
80
+ ```bash
81
+ pip install "percept-vision[gemini]" # GeminiVision
82
+ pip install "percept-vision[claude]" # AnthropicVision (Claude)
83
+ pip install "percept-vision[deepgram]" # Deepgram STT + TTS (voice)
84
+ pip install "percept-vision[audio]" # numpy fast-path for acoustic/edge
85
+ pip install "percept-vision[all]" # everything
86
+ ```
87
+
88
+ Python **>= 3.10**. The package name is `percept-vision`; the import is `import percept`.
89
+
90
+ ## 60-second quickstart (no keys)
91
+
92
+ Fully offline — fake backends, deterministic, nothing to configure. This script runs as-is:
93
+
94
+ ```python
95
+ import asyncio
96
+ from percept import Percept, Goal
97
+
98
+ async def main():
99
+ # Fake backends by default — offline, no keys. discover_plugins=False skips plugin lookup.
100
+ agent = Percept.create(discover_plugins=False)
101
+
102
+ agent.add_goal(Goal(
103
+ id="caffeine",
104
+ condition="the user is drinking coffee",
105
+ say="Heads up — stepping back from caffeine?",
106
+ ))
107
+
108
+ # Each frame is judged; the gate fires ONCE on the rising edge.
109
+ # ("sip-coffee" is a token the fake vision backend scripts as a confident YES.)
110
+ fires = await agent.perceive_judged("sip-coffee")
111
+ for ev in fires: # ev is a FireEvent
112
+ print(ev.action, ev.goal_id, ev.text) # -> fire caffeine Heads up — stepping back from caffeine?
113
+
114
+ # The same frame again does NOT re-fire — rising-edge, not level-triggered.
115
+ print(await agent.perceive_judged("sip-coffee")) # -> []
116
+
117
+ asyncio.run(main())
118
+ ```
119
+
120
+ `perceive_judged(frame)` returns a list of `FireEvent(goal_id, action, text, entity_id, verdict)`,
121
+ where `action` is `"fire"` or `"ask"`. With real eyes, swap the fake for a frontier vision backend —
122
+ the cognition layer above is unchanged:
123
+
124
+ ```python
125
+ agent = Percept.create(vision="gemini") # needs percept-vision[gemini] + GEMINI_API_KEY
126
+ # frame is now real image bytes; everything downstream (gate, graph, executor) is identical.
127
+ ```
128
+
129
+ Backends are selected by name (`"fake"` · `"gemini"` · `"claude"`) or by passing an adapter instance,
130
+ and can also be set via env (`PERCEPT_VISION_BACKEND`, etc.).
131
+
132
+ ## Architecture — two layers
133
+
134
+ percept cleanly splits the **eyes** from the **brain**. Perception is one stateless seam;
135
+ everything stateful and proactive is cognition.
136
+
137
+ ```
138
+ ┌───────────────────────── COGNITION — the "brain" ──────────────────────────┐
139
+ │ three-state GATE fire (known-yes) · ask (unknown) · silent (known-no) │
140
+ │ ▲ rising edge: fires only on false→true, with a refractory (no nag) │
141
+ │ EXECUTOR one firing path (sense · key · accumulate); │
142
+ │ transitions A→B, counting, deadlines/absence, verify │
143
+ │ ENTITY GRAPH stable ids; attributes with provenance + freshness │
144
+ │ (observed vs you-told-me) │
145
+ │ EDGE DETECTOR REGISTRY opt-in cheap signals propose; the brain counts/gates │
146
+ │ general · deterministic · cheap · STATEFUL (has memory) │
147
+ └──────────────────────────────────────────────────────────────────────────────┘
148
+ ▲ feeds on Verdict{satisfied, confidence}
149
+ ┌───────────────────────── PERCEPTION — the "eyes" ──────────────────────────┐
150
+ │ vision.judge(condition, frame) -> Verdict ★ ONE seam │
151
+ │ GeminiVision / AnthropicVision · stateless · one frame · ~1s/call cloud │
152
+ └──────────────────────────────────────────────────────────────────────────────┘
153
+ ```
154
+
155
+ The **gate** turns a noisy verdict stream into at most one alert on the rising edge, and falls back
156
+ to **ASK** rather than guess when confidence is unreliable. The **executor** is the single firing
157
+ path for every concern shape (watch, transition, count, deadline/absence, verification). The
158
+ **entity graph** carries memory: stable ids and attributes that know whether they were *observed* or
159
+ *asserted by the user*, and whether they're still fresh.
160
+
161
+ The **edge detector registry** is the *edge-proposes · brain-counts · VLM-confirms* split: cheap,
162
+ opt-in detectors emit timed boundaries that the brain accumulates — counting reps without spending a
163
+ vision token per frame. Three reference skills ship as registered detectors:
164
+
165
+ - **`motion-periodicity`** — frame-diff motion peaks (rep boundaries);
166
+ - **`acoustic-onset`** — energy spikes on the existing mic stream (no new capture path);
167
+ - **`pose-openness`** — BlazePose-based rep peaks (opt-in: `percept-harness[pose]`, MediaPipe).
168
+
169
+ A full request trace — *"tell me when the milk is about to boil over"* — is in
170
+ [`docs/e2e-flow-milk-boilover.md`](docs/e2e-flow-milk-boilover.md), including an honest map of where
171
+ the perception ceiling bites.
172
+
173
+ ## The three packages
174
+
175
+ | package | what | install |
176
+ |---|---|---|
177
+ | **percept-vision** | the cognition core — gate, entity graph, executor, events, scheduler, consent, fakes. Pure stdlib. | `pip install percept-vision` (`packages/percept-vision/`) |
178
+ | **percept-harness** | server-side transport shell + tier-0 salience gate (WatchSpec down / Tier0Signal up); the reference home of the edge detector skills. | `packages/percept-harness/` |
179
+ | **@percept/edge** | on-device reactive edge in JS/WASM: VAD + motion gate over the same WatchSpec/Tier0Signal wire-contract. | `packages/percept-edge/` (npm `@percept/edge`) |
180
+
181
+ ## Benchmark — Percept Benchmark v1 (v0.5 DRAFT)
182
+
183
+ The benchmark holds the **backbone fixed and measures the orchestration delta** across a `raw → core
184
+ → e2e` config ladder (no composite score — a vector of headlines). The current card is a **v0.5
185
+ DRAFT** (sampled, N≈12/track, gemini-2.5-flash) on the DeepMind **Perception Test** (CC-BY-4.0), the
186
+ private **golden** ambiguity corpus, and **RepCount-A**; the e2e-relational accuracy cell is modeled
187
+ (flagged `~`), so the card is stamped `DRAFT`.
188
+
189
+ | track | measured (DRAFT, N≈12) | reading |
190
+ |---|---|---|
191
+ | **counting** (RepCount-A) | pose OBO **0.33** vs **0.29** (TransRAC, CVPR'22); nMAE 0.80 vs 0.44 | training-free pose ties a *trained* baseline on OBO, but its failures on low-amplitude actions cost it on MAE |
192
+ | **acoustic** (PT Sound Loc., unseen source) | vision-only recall **0.00 → 0.25–0.33** fused; fusion-FP 0–0.17 | the **recall-flip**: fusion rescues ~a third of sound events vision-only never hears (honest at scale, not the single-clip 1.0) |
193
+ | **relational** (golden) | confidence **AUROC ≈ 0.51** (≈ chance); ASK-rate ≈ 0.4 | the VLM's confidence does **not** separate right from wrong → justifies the **ASK** discipline over threshold-tuning |
194
+ | **timeliness** (PT action onsets) | **P-PAUC ≈ 60**; reaction λ p50 ≈ 1s | first real proactive-timeliness numbers (adapted PAUC) |
195
+ | **edge event-recall** (PT onsets) | **0.00** @ thr 0.10 | ⚠️ the motion-gate escalates on **none** of the subtle-action onsets (motion ~0.03–0.06 < 0.10) → e2e ≠ core there; a real **calibration** finding |
196
+
197
+ A v0.5 DRAFT, not a RELEASE claim. Two findings the benchmark surfaced that a self-congratulatory
198
+ eval would hide: the **flat confidence AUROC** (*why* the gate refuses to guess) and the **edge
199
+ event-recall of 0** on subtle actions (the motion-gate is mis-calibrated for them). Full plan, data,
200
+ and references:
201
+ [`packages/percept-vision/eval/BENCHMARK_PLAN.md`](packages/percept-vision/eval/BENCHMARK_PLAN.md).
202
+
203
+ ## Layout
204
+
205
+ ```
206
+ packages/percept-vision/ the SDK (pip install percept-vision; import percept)
207
+ packages/percept-harness/ server-side tier-0 edge reference + detector skills
208
+ packages/percept-edge/ @percept/edge — on-device JS/WASM edge
209
+ docs/ architecture & flow traces
210
+ eval/ golden corpus (benchmark plan: packages/percept-vision/eval/)
211
+ Spec/ the spec + implementation plan
212
+ Makefile test · eval-live · e2e · bench · check · reproduce
213
+ ```
214
+
215
+ ## Docs
216
+
217
+ - **Quickstart** — the 60-second offline snippet above; `make test` runs the fake-only unit lane.
218
+ - **Architecture / concepts** — [`docs/e2e-flow-milk-boilover.md`](docs/e2e-flow-milk-boilover.md)
219
+ (the two layers, the firing path, the perception ceiling) and [`Spec/`](Spec/) (the full spec +
220
+ phase plan).
221
+ - **Backends** — select by name (`fake` · `gemini` · `claude` · `deepgram`) or env
222
+ (`PERCEPT_*_BACKEND`); add via the `percept.backends` entry-point group. Extras:
223
+ `[gemini]` · `[claude]` · `[deepgram]` · `[audio]`.
224
+ - **Benchmark** — [`eval/BENCHMARK_PLAN.md`](packages/percept-vision/eval/BENCHMARK_PLAN.md).
225
+
226
+ ## Status
227
+
228
+ **v0.1.0 — early / first public release.** The cognition core runs and is tested offline with fakes
229
+ (the L1 lane, no keys); the frontier backends (Gemini, Claude, Deepgram) and the edge packages are
230
+ wired behind their seams. The benchmark is a **v0.5 DRAFT** card. We are now in **real-life
231
+ testing** — APIs and numbers may change. Issues and contributions welcome.
232
+
233
+ ## License
234
+
235
+ Apache-2.0.
@@ -0,0 +1,198 @@
1
+ # percept
2
+
3
+ The open-source **cognition layer** for goal-driven, proactive vision agents.
4
+
5
+ You state a goal in plain language — *"nudge me when I drink a coffee"*, *"tell me when the kettle
6
+ boils"* — and percept turns a live audio-visual stream into an agent that:
7
+
8
+ - **reasons over entities across time** — tracked things with stable ids, attributes carrying
9
+ provenance (*observed* vs *you-told-me*) and freshness;
10
+ - **fires proactively**, but only on the **rising edge** of a condition becoming true;
11
+ - **refuses to guess** — a three-state gate maps *known → act · not → silent · unknown → ask*, so it
12
+ never delivers a confidently-wrong nag.
13
+
14
+ The wedge is **temporal cognition** (entity memory + the three-state gate + reasoning over events),
15
+ which a raw VLM-in-a-loop and a per-frame pipeline both lack. percept builds on frontier models for
16
+ perception behind vendor-neutral seams.
17
+
18
+ > **⚠️ Research preview — v0.1.0.** Published for real-life testing and feedback, **not** for
19
+ > production. The cognition core (gate, entity graph, executor, events, scheduler, consent) runs and is
20
+ > tested; the benchmark card below is a **v0.5 DRAFT**. APIs may change between `0.1.x` releases —
21
+ > pin a version. Issues and feedback welcome. ([Status](#status).)
22
+
23
+ ## The envelope (refusals, stated proudly)
24
+
25
+ - **Assistant-class, never the safety mechanism.** percept informs a human who stays responsible. It
26
+ is never the thing standing between a person and harm on a clock it can't guarantee. *Out:*
27
+ driver-drowsiness intervention, turn detection. *In:* a post-hoc driving debrief.
28
+ - **Watches the user's own world, with the user as beneficiary** — never a non-consenting third party
29
+ to the wearer's advantage. *Out:* covert analysis of someone you're negotiating with; card-counting
30
+ against a live casino. *In:* a play-money practice trainer.
31
+
32
+ These refusals are a feature. **Do not lead with surveillance demos.**
33
+
34
+ ## Install
35
+
36
+ ```bash
37
+ pip install percept-vision # core: PURE STDLIB — runs offline with fake backends, no keys
38
+ ```
39
+
40
+ The **core has zero dependencies** and runs with deterministic fake backends, so `pip install` then
41
+ run works with no API keys. Frontier backends are opt-in extras:
42
+
43
+ ```bash
44
+ pip install "percept-vision[gemini]" # GeminiVision
45
+ pip install "percept-vision[claude]" # AnthropicVision (Claude)
46
+ pip install "percept-vision[deepgram]" # Deepgram STT + TTS (voice)
47
+ pip install "percept-vision[audio]" # numpy fast-path for acoustic/edge
48
+ pip install "percept-vision[all]" # everything
49
+ ```
50
+
51
+ Python **>= 3.10**. The package name is `percept-vision`; the import is `import percept`.
52
+
53
+ ## 60-second quickstart (no keys)
54
+
55
+ Fully offline — fake backends, deterministic, nothing to configure. This script runs as-is:
56
+
57
+ ```python
58
+ import asyncio
59
+ from percept import Percept, Goal
60
+
61
+ async def main():
62
+ # Fake backends by default — offline, no keys. discover_plugins=False skips plugin lookup.
63
+ agent = Percept.create(discover_plugins=False)
64
+
65
+ agent.add_goal(Goal(
66
+ id="caffeine",
67
+ condition="the user is drinking coffee",
68
+ say="Heads up — stepping back from caffeine?",
69
+ ))
70
+
71
+ # Each frame is judged; the gate fires ONCE on the rising edge.
72
+ # ("sip-coffee" is a token the fake vision backend scripts as a confident YES.)
73
+ fires = await agent.perceive_judged("sip-coffee")
74
+ for ev in fires: # ev is a FireEvent
75
+ print(ev.action, ev.goal_id, ev.text) # -> fire caffeine Heads up — stepping back from caffeine?
76
+
77
+ # The same frame again does NOT re-fire — rising-edge, not level-triggered.
78
+ print(await agent.perceive_judged("sip-coffee")) # -> []
79
+
80
+ asyncio.run(main())
81
+ ```
82
+
83
+ `perceive_judged(frame)` returns a list of `FireEvent(goal_id, action, text, entity_id, verdict)`,
84
+ where `action` is `"fire"` or `"ask"`. With real eyes, swap the fake for a frontier vision backend —
85
+ the cognition layer above is unchanged:
86
+
87
+ ```python
88
+ agent = Percept.create(vision="gemini") # needs percept-vision[gemini] + GEMINI_API_KEY
89
+ # frame is now real image bytes; everything downstream (gate, graph, executor) is identical.
90
+ ```
91
+
92
+ Backends are selected by name (`"fake"` · `"gemini"` · `"claude"`) or by passing an adapter instance,
93
+ and can also be set via env (`PERCEPT_VISION_BACKEND`, etc.).
94
+
95
+ ## Architecture — two layers
96
+
97
+ percept cleanly splits the **eyes** from the **brain**. Perception is one stateless seam;
98
+ everything stateful and proactive is cognition.
99
+
100
+ ```
101
+ ┌───────────────────────── COGNITION — the "brain" ──────────────────────────┐
102
+ │ three-state GATE fire (known-yes) · ask (unknown) · silent (known-no) │
103
+ │ ▲ rising edge: fires only on false→true, with a refractory (no nag) │
104
+ │ EXECUTOR one firing path (sense · key · accumulate); │
105
+ │ transitions A→B, counting, deadlines/absence, verify │
106
+ │ ENTITY GRAPH stable ids; attributes with provenance + freshness │
107
+ │ (observed vs you-told-me) │
108
+ │ EDGE DETECTOR REGISTRY opt-in cheap signals propose; the brain counts/gates │
109
+ │ general · deterministic · cheap · STATEFUL (has memory) │
110
+ └──────────────────────────────────────────────────────────────────────────────┘
111
+ ▲ feeds on Verdict{satisfied, confidence}
112
+ ┌───────────────────────── PERCEPTION — the "eyes" ──────────────────────────┐
113
+ │ vision.judge(condition, frame) -> Verdict ★ ONE seam │
114
+ │ GeminiVision / AnthropicVision · stateless · one frame · ~1s/call cloud │
115
+ └──────────────────────────────────────────────────────────────────────────────┘
116
+ ```
117
+
118
+ The **gate** turns a noisy verdict stream into at most one alert on the rising edge, and falls back
119
+ to **ASK** rather than guess when confidence is unreliable. The **executor** is the single firing
120
+ path for every concern shape (watch, transition, count, deadline/absence, verification). The
121
+ **entity graph** carries memory: stable ids and attributes that know whether they were *observed* or
122
+ *asserted by the user*, and whether they're still fresh.
123
+
124
+ The **edge detector registry** is the *edge-proposes · brain-counts · VLM-confirms* split: cheap,
125
+ opt-in detectors emit timed boundaries that the brain accumulates — counting reps without spending a
126
+ vision token per frame. Three reference skills ship as registered detectors:
127
+
128
+ - **`motion-periodicity`** — frame-diff motion peaks (rep boundaries);
129
+ - **`acoustic-onset`** — energy spikes on the existing mic stream (no new capture path);
130
+ - **`pose-openness`** — BlazePose-based rep peaks (opt-in: `percept-harness[pose]`, MediaPipe).
131
+
132
+ A full request trace — *"tell me when the milk is about to boil over"* — is in
133
+ [`docs/e2e-flow-milk-boilover.md`](docs/e2e-flow-milk-boilover.md), including an honest map of where
134
+ the perception ceiling bites.
135
+
136
+ ## The three packages
137
+
138
+ | package | what | install |
139
+ |---|---|---|
140
+ | **percept-vision** | the cognition core — gate, entity graph, executor, events, scheduler, consent, fakes. Pure stdlib. | `pip install percept-vision` (`packages/percept-vision/`) |
141
+ | **percept-harness** | server-side transport shell + tier-0 salience gate (WatchSpec down / Tier0Signal up); the reference home of the edge detector skills. | `packages/percept-harness/` |
142
+ | **@percept/edge** | on-device reactive edge in JS/WASM: VAD + motion gate over the same WatchSpec/Tier0Signal wire-contract. | `packages/percept-edge/` (npm `@percept/edge`) |
143
+
144
+ ## Benchmark — Percept Benchmark v1 (v0.5 DRAFT)
145
+
146
+ The benchmark holds the **backbone fixed and measures the orchestration delta** across a `raw → core
147
+ → e2e` config ladder (no composite score — a vector of headlines). The current card is a **v0.5
148
+ DRAFT** (sampled, N≈12/track, gemini-2.5-flash) on the DeepMind **Perception Test** (CC-BY-4.0), the
149
+ private **golden** ambiguity corpus, and **RepCount-A**; the e2e-relational accuracy cell is modeled
150
+ (flagged `~`), so the card is stamped `DRAFT`.
151
+
152
+ | track | measured (DRAFT, N≈12) | reading |
153
+ |---|---|---|
154
+ | **counting** (RepCount-A) | pose OBO **0.33** vs **0.29** (TransRAC, CVPR'22); nMAE 0.80 vs 0.44 | training-free pose ties a *trained* baseline on OBO, but its failures on low-amplitude actions cost it on MAE |
155
+ | **acoustic** (PT Sound Loc., unseen source) | vision-only recall **0.00 → 0.25–0.33** fused; fusion-FP 0–0.17 | the **recall-flip**: fusion rescues ~a third of sound events vision-only never hears (honest at scale, not the single-clip 1.0) |
156
+ | **relational** (golden) | confidence **AUROC ≈ 0.51** (≈ chance); ASK-rate ≈ 0.4 | the VLM's confidence does **not** separate right from wrong → justifies the **ASK** discipline over threshold-tuning |
157
+ | **timeliness** (PT action onsets) | **P-PAUC ≈ 60**; reaction λ p50 ≈ 1s | first real proactive-timeliness numbers (adapted PAUC) |
158
+ | **edge event-recall** (PT onsets) | **0.00** @ thr 0.10 | ⚠️ the motion-gate escalates on **none** of the subtle-action onsets (motion ~0.03–0.06 < 0.10) → e2e ≠ core there; a real **calibration** finding |
159
+
160
+ A v0.5 DRAFT, not a RELEASE claim. Two findings the benchmark surfaced that a self-congratulatory
161
+ eval would hide: the **flat confidence AUROC** (*why* the gate refuses to guess) and the **edge
162
+ event-recall of 0** on subtle actions (the motion-gate is mis-calibrated for them). Full plan, data,
163
+ and references:
164
+ [`packages/percept-vision/eval/BENCHMARK_PLAN.md`](packages/percept-vision/eval/BENCHMARK_PLAN.md).
165
+
166
+ ## Layout
167
+
168
+ ```
169
+ packages/percept-vision/ the SDK (pip install percept-vision; import percept)
170
+ packages/percept-harness/ server-side tier-0 edge reference + detector skills
171
+ packages/percept-edge/ @percept/edge — on-device JS/WASM edge
172
+ docs/ architecture & flow traces
173
+ eval/ golden corpus (benchmark plan: packages/percept-vision/eval/)
174
+ Spec/ the spec + implementation plan
175
+ Makefile test · eval-live · e2e · bench · check · reproduce
176
+ ```
177
+
178
+ ## Docs
179
+
180
+ - **Quickstart** — the 60-second offline snippet above; `make test` runs the fake-only unit lane.
181
+ - **Architecture / concepts** — [`docs/e2e-flow-milk-boilover.md`](docs/e2e-flow-milk-boilover.md)
182
+ (the two layers, the firing path, the perception ceiling) and [`Spec/`](Spec/) (the full spec +
183
+ phase plan).
184
+ - **Backends** — select by name (`fake` · `gemini` · `claude` · `deepgram`) or env
185
+ (`PERCEPT_*_BACKEND`); add via the `percept.backends` entry-point group. Extras:
186
+ `[gemini]` · `[claude]` · `[deepgram]` · `[audio]`.
187
+ - **Benchmark** — [`eval/BENCHMARK_PLAN.md`](packages/percept-vision/eval/BENCHMARK_PLAN.md).
188
+
189
+ ## Status
190
+
191
+ **v0.1.0 — early / first public release.** The cognition core runs and is tested offline with fakes
192
+ (the L1 lane, no keys); the frontier backends (Gemini, Claude, Deepgram) and the edge packages are
193
+ wired behind their seams. The benchmark is a **v0.5 DRAFT** card. We are now in **real-life
194
+ testing** — APIs and numbers may change. Issues and contributions welcome.
195
+
196
+ ## License
197
+
198
+ Apache-2.0.
@@ -0,0 +1,113 @@
1
+ """percept — the open-source cognition layer for goal-driven, proactive vision agents.
2
+
3
+ Public surface. The reactive watch loop, the three-state gate, the entity graph, and the goal
4
+ registry are wired (Phase 1); identity/episodes, events/scheduler, the edge, and persistence land
5
+ in later phases *beside* this core. See ``Spec/`` for the what/why and the implementation plan.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from . import registry
11
+ from .adapters import (
12
+ Attribute,
13
+ Box,
14
+ Decision,
15
+ Goal,
16
+ Observation,
17
+ STTEvent,
18
+ Tier0Signal,
19
+ TurnResult,
20
+ Verdict,
21
+ WatchCondition,
22
+ WatchSpec,
23
+ )
24
+ from .cache import CacheCascade, CacheKey, CacheResult, FrameHashLayer
25
+ from .config import Settings, load_settings
26
+ from .consent import ActionLog, ActionRecord, ConsentDenied, ConsentProfile
27
+ from .entity import Entity, EntityGraph
28
+ from .episodes import Episode, EpisodeLog
29
+ from .events import Event, EventLog, FrameWindow
30
+ from .gate import ASK, FIRE, SILENT, Gate, GateAction
31
+ from .goals import GoalOp, GoalRegistry
32
+ from .identity import AppearanceIdentity, ContinuityIdentity
33
+ from .scheduler import Scheduler, Timer
34
+ from .observability import Metrics, get_logger, health
35
+ from .persistence import InMemoryPersistence, SqlitePersistence
36
+ from .routing import (
37
+ DEFAULT_RULES,
38
+ RegexRouteLayer,
39
+ RouteEngine,
40
+ RouteLayer,
41
+ RouteMatch,
42
+ Rule,
43
+ )
44
+
45
+ __version__ = "0.1.0"
46
+
47
+ # Imported after __version__ so agent.health() can read it lazily without a cycle.
48
+ from .agent import FireEvent, Percept # noqa: E402
49
+
50
+ __all__ = [
51
+ "__version__",
52
+ "Percept",
53
+ "FireEvent",
54
+ "registry",
55
+ # DTOs
56
+ "Box",
57
+ "Observation",
58
+ "Verdict",
59
+ "Attribute",
60
+ "Goal",
61
+ "Decision",
62
+ "STTEvent",
63
+ "TurnResult",
64
+ "WatchCondition",
65
+ "WatchSpec",
66
+ "Tier0Signal",
67
+ # cognition core
68
+ "Gate",
69
+ "GateAction",
70
+ "FIRE",
71
+ "ASK",
72
+ "SILENT",
73
+ "EntityGraph",
74
+ "Entity",
75
+ "GoalRegistry",
76
+ "GoalOp",
77
+ # identity + episodes (Phase 4 — re-id across gaps; the cup problem)
78
+ "ContinuityIdentity",
79
+ "AppearanceIdentity",
80
+ "Episode",
81
+ "EpisodeLog",
82
+ # events + scheduler (Phase 5 — temporal perception; time-triggered fires)
83
+ "Event",
84
+ "EventLog",
85
+ "FrameWindow",
86
+ "Scheduler",
87
+ "Timer",
88
+ # persistence + consent (Phase 7 — endurance + safe act modality)
89
+ "InMemoryPersistence",
90
+ "SqlitePersistence",
91
+ "ConsentProfile",
92
+ "ConsentDenied",
93
+ "ActionLog",
94
+ "ActionRecord",
95
+ # routing (Phase 3 — local fast-path; learned tier slots behind RouteLayer in V0.1)
96
+ "RouteEngine",
97
+ "RouteLayer",
98
+ "RouteMatch",
99
+ "RegexRouteLayer",
100
+ "Rule",
101
+ "DEFAULT_RULES",
102
+ # cache
103
+ "CacheCascade",
104
+ "CacheResult",
105
+ "CacheKey",
106
+ "FrameHashLayer",
107
+ # infra
108
+ "Settings",
109
+ "load_settings",
110
+ "Metrics",
111
+ "get_logger",
112
+ "health",
113
+ ]