dv-pipecat-flows 0.0.0.dev2393__tar.gz → 0.0.0.dev2572__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/.gitignore +2 -0
  2. {dv_pipecat_flows-0.0.0.dev2393/src/dv_pipecat_flows.egg-info → dv_pipecat_flows-0.0.0.dev2572}/PKG-INFO +1 -2
  3. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/README.md +0 -1
  4. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/docs/CODE_MAP.md +9 -6
  5. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/docs/CONTRACTS.md +11 -0
  6. dv_pipecat_flows-0.0.0.dev2572/browser_copilot/docs/COPILOT_TODO.md +79 -0
  7. dv_pipecat_flows-0.0.0.dev2572/scripts/azure_latency_probe.py +220 -0
  8. dv_pipecat_flows-0.0.0.dev2572/scripts/langfuse_export.py +321 -0
  9. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572/src/dv_pipecat_flows.egg-info}/PKG-INFO +1 -2
  10. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/src/dv_pipecat_flows.egg-info/SOURCES.txt +3 -0
  11. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/src/dv_pipecat_flows.egg-info/scm_file_list.json +486 -468
  12. dv_pipecat_flows-0.0.0.dev2572/src/dv_pipecat_flows.egg-info/scm_version.json +8 -0
  13. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/src/pipecat_flows/actions.py +42 -7
  14. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/src/pipecat_flows/router_mode.py +12 -3
  15. dv_pipecat_flows-0.0.0.dev2393/src/dv_pipecat_flows.egg-info/scm_version.json +0 -8
  16. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/.agents/skills/loki-logs/SKILL.md +0 -0
  17. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/.agents/skills/loki-logs/query-reference.md +0 -0
  18. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/.claude/skills/loki-logs/SKILL.md +0 -0
  19. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/.claude/skills/loki-logs/query-reference.md +0 -0
  20. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/.gitattributes +0 -0
  21. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/.pre-commit-config.yaml +0 -0
  22. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/.python-version +0 -0
  23. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/.readthedocs.yaml +0 -0
  24. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/AGENTS.md +0 -0
  25. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/CHANGELOG.md +0 -0
  26. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/CLAUDE.md +0 -0
  27. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/CONTRIBUTING.md +0 -0
  28. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/LICENSE +0 -0
  29. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/MANIFEST.in +0 -0
  30. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/.env.example +0 -0
  31. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/.gitignore +0 -0
  32. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/README.md +0 -0
  33. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/docs/ARCHITECTURE.md +0 -0
  34. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/docs/CONFIG.md +0 -0
  35. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/docs/DISTRIBUTION.md +0 -0
  36. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/docs/HLD.md +0 -0
  37. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/docs/METHODS.md +0 -0
  38. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/docs/VOICE_E2E.md +0 -0
  39. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/eval/ANALYSIS_stagehand_vs_hybrid.md +0 -0
  40. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/eval/RESULTS.md +0 -0
  41. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/eval/_env.py +0 -0
  42. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/eval/bench.py +0 -0
  43. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/eval/bench_browseruse.py +0 -0
  44. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/eval/bench_stagehand.py +0 -0
  45. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/eval/inspect_browseruse.py +0 -0
  46. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/eval/inspect_browseruse_showcase.py +0 -0
  47. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/eval/inspect_stagehand_prompt.py +0 -0
  48. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/eval/pages/checkout.html +0 -0
  49. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/eval/pages/dashboard.html +0 -0
  50. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/eval/pages/signup.html +0 -0
  51. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/eval/results_browseruse.json +0 -0
  52. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/eval/results_perception.json +0 -0
  53. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/eval/results_stagehand.json +0 -0
  54. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/eval/tasks.py +0 -0
  55. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/eval/try_hybrid.py +0 -0
  56. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/eval/try_hybrid_multistep.py +0 -0
  57. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/eval/try_hybrid_oneshot.py +0 -0
  58. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/pyproject.toml +0 -0
  59. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/scripts/dev-stack.sh +0 -0
  60. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/browser_copilot/uv.lock +0 -0
  61. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/dev-requirements.txt +0 -0
  62. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/docker-compose.dev.yml +0 -0
  63. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/.eslintrc.json +0 -0
  64. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/.prettierrc +0 -0
  65. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/css/tailwind.css +0 -0
  66. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/examples/food_ordering.json +0 -0
  67. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/examples/movie_explorer.json +0 -0
  68. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/examples/patient_intake.json +0 -0
  69. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/examples/restaurant_reservation.json +0 -0
  70. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/examples/travel_planner.json +0 -0
  71. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/favicon.png +0 -0
  72. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/favicon.svg +0 -0
  73. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/index.html +0 -0
  74. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/js/editor/canvas.js +0 -0
  75. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/js/editor/editorState.js +0 -0
  76. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/js/editor/sidePanel.js +0 -0
  77. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/js/editor/toolbar.js +0 -0
  78. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/js/main.js +0 -0
  79. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/js/nodes/baseNode.js +0 -0
  80. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/js/nodes/endNode.js +0 -0
  81. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/js/nodes/flowNode.js +0 -0
  82. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/js/nodes/functionNode.js +0 -0
  83. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/js/nodes/index.js +0 -0
  84. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/js/nodes/mergeNode.js +0 -0
  85. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/js/nodes/startNode.js +0 -0
  86. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/js/types.js +0 -0
  87. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/js/utils/export.js +0 -0
  88. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/js/utils/helpers.js +0 -0
  89. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/js/utils/import.js +0 -0
  90. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/js/utils/validation.js +0 -0
  91. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/jsdoc.json +0 -0
  92. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/package-lock.json +0 -0
  93. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/package.json +0 -0
  94. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/postcss.config.cjs +0 -0
  95. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/public/favicon.png +0 -0
  96. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/public/favicon.svg +0 -0
  97. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/tailwind.config.cjs +0 -0
  98. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/vercel.json +0 -0
  99. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/editor/vite.config.js +0 -0
  100. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/engine_primitives_plan.md +0 -0
  101. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/env.example +0 -0
  102. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/images/food-ordering-flow.png +0 -0
  103. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/pipecat-flows.png +0 -0
  104. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/pipecat_upgrade.md +0 -0
  105. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/pyproject.toml +0 -0
  106. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/remote-asterisk-code/README.md +0 -0
  107. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/remote-asterisk-code/extensions.conf +0 -0
  108. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/remote-asterisk-code/rtp.conf +0 -0
  109. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/requirements.txt +0 -0
  110. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/scripts/check-pypi-package.py +0 -0
  111. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/scripts/fix-ruff.sh +0 -0
  112. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/scripts/pre-commit.sh +0 -0
  113. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/setup.cfg +0 -0
  114. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/src/dv_pipecat_flows.egg-info/dependency_links.txt +0 -0
  115. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/src/dv_pipecat_flows.egg-info/requires.txt +0 -0
  116. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/src/dv_pipecat_flows.egg-info/top_level.txt +0 -0
  117. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/src/pipecat_flows/__init__.py +0 -0
  118. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/src/pipecat_flows/adapters.py +0 -0
  119. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/src/pipecat_flows/condition_evaluator.py +0 -0
  120. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/src/pipecat_flows/exceptions.py +0 -0
  121. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/src/pipecat_flows/flow_validator.py +0 -0
  122. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/src/pipecat_flows/manager.py +0 -0
  123. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/src/pipecat_flows/processors/__init__.py +0 -0
  124. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/src/pipecat_flows/processors/router_mode_guard.py +0 -0
  125. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/src/pipecat_flows/processors/speak_interruption_guard.py +0 -0
  126. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/src/pipecat_flows/processors/user_turn_observer.py +0 -0
  127. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/src/pipecat_flows/types.py +0 -0
  128. {dv_pipecat_flows-0.0.0.dev2393 → dv_pipecat_flows-0.0.0.dev2572}/uv.lock +0 -0
@@ -187,3 +187,5 @@ MANIFEST
187
187
 
188
188
  # Asterisk configs
189
189
  remote-asterisk-code/pjsip.conf
190
+ .venvringg1
191
+ .venvringg2
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dv-pipecat-flows
3
- Version: 0.0.0.dev2393
3
+ Version: 0.0.0.dev2572
4
4
  Summary: Conversation Flow management for Pipecat AI applications
5
5
  License: BSD 2-Clause License
6
6
  Project-URL: Source, https://github.com/pipecat-ai/pipecat-flows
@@ -309,4 +309,3 @@ We aim to review all contributions promptly and provide constructive feedback to
309
309
  ➡️ [Reach us on X](https://x.com/pipecat_ai)
310
310
 
311
311
 
312
-
@@ -284,4 +284,3 @@ We aim to review all contributions promptly and provide constructive feedback to
284
284
  ➡️ [Reach us on X](https://x.com/pipecat_ai)
285
285
 
286
286
 
287
-
@@ -81,7 +81,7 @@ CHAT TEXT bot ──► widget: send_chat_text (topic "lk.chat", scrub_handles)
81
81
  - `perception.ts:buildDomSummary()` ↔ `dom_context_injector.py:_render_summary()` —
82
82
  the DomSummary v2 shape. All v2 keys optional; legacy `headings` kept for old injectors.
83
83
  - `copilot_support.py:BROWSER_COPILOT_ACTION_TOOLS` ↔
84
- `browser_actions.py:BROWSER_ACTION_SCHEMAS` — the 8 tool names the backend injects
84
+ `browser_actions.py:BROWSER_ACTION_SCHEMAS` — the 9 tool names the backend injects
85
85
  must resolve to schemas the bot registers.
86
86
  - Timeouts: widget `AUTO_DENY_MS=15s` (use-copilot.ts) must stay under the bot's
87
87
  `send_dom_action` 20s RPC timeout (livekit_rpc_bridge.py) — a stuck confirm bar
@@ -205,22 +205,25 @@ CHAT TEXT bot ──► widget: send_chat_text (topic "lk.chat", scrub_handles)
205
205
  - Legacy (pre-v2) summaries render byte-identical to before (`_action_flags` is empty for them); a raw input `value` key is never rendered on the summary path.
206
206
  - `_build_message_text()` = marker + URL/Title + summary + "User is currently: …"; hard tail-truncate at 12,000 chars (hostile summaries can't blow the context). `_sync_to_context()` filters old `[Browser context]` messages and appends the fresh one.
207
207
  - Also holds `_url` — read by `browser_actions._current_url` for the mutating-action origin guardrail.
208
+ - Auto-continue chain: `arm_continue` (mutating action) → next LLM step fires when the fresh summary lands (`_maybe_continue`, capped `MAX_CHAIN`). Stale-summary watchdog (`CONTINUE_FALLBACK_S=3.5`): if no summary lands (a DOM change the digest can't see, e.g. an option toggle), fire the continue anyway with a `[Browser context]` re-check note (`_stale_note`) so the chain can't starve. `_action_flags` renders `[checked]`/`[unchecked]` (selection state) alongside `[filled]`/`[sensitive]`; long `— options: …` lists truncate with `…`.
208
209
 
209
210
  ### `utils/flow_tools/browser_actions.py`
210
- - The LLM-callable hands: 8 handlers (`read_dom`, `find_elements`, `read_region`, `click_element`, `type_text`, `select_option`, `scroll_page`, `highlight_element`) + their `FlowsFunctionSchema`s in `BROWSER_ACTION_SCHEMAS`. Targeting: `cp` handle is the advertised, preferred key (`ref` deliberately NOT advertised — one handle namespace; strays still forwarded); `selector`/`text`/`role` fallbacks. Mutating schemas take an optional `say` ack.
211
- - `_run(flow_manager, action, params, say)` — shared guardrails, in order: bridge present (`state["livekit_rpc"]`) → kill-switch `copilot_config.disabled` → for `MUTATING_ACTIONS`, `_origin_allowed(injector._url, cfg.allowed_origins)` (empty allowlist = allow all) → confirm policy → `bridge.send_dom_action`.
211
+ - The LLM-callable hands: 9 handlers (`read_dom`, `find_elements`, `read_region`, `click_element`, `type_text`, `select_option`, `scroll_page`, `highlight_element`, `execute_plan`) + their `FlowsFunctionSchema`s in `BROWSER_ACTION_SCHEMAS`. Targeting: `cp` handle is the advertised, preferred key (`ref` deliberately NOT advertised — one handle namespace; strays still forwarded); `selector`/`text`/`role` fallbacks. Mutating schemas take an optional `say` ack.
212
+ - `_run_step(flow_manager, action, params) -> (ok, body)` — the shared guardrails, in order: bridge present (`state["livekit_rpc"]`) → kill-switch `copilot_config.disabled` → for `MUTATING_ACTIONS`, `_origin_allowed(injector._url, cfg.allowed_origins)` (empty allowlist = allow all) → confirm policy → `bridge.send_dom_action`. `_run(...)` is the single-action wrapper (adds `say`, suppression, `arm_continue`).
213
+ - `execute_plan({steps, say})` — one LLM call runs an ordered batch of steps on the CURRENT view via `_run_step` each (`MAX_PLAN_STEPS=8`, `PLAN_BATCH_BUDGET_S=60`), stops at the first failure/denial, then issues one `read_dom` and folds the fresh page into `[Browser context]` (`injector.absorb(CT_DOM_SUMMARY, …)`), returning ActionResult-v1-shaped `results` + a compact `page` digest. NO suppression sentinel / no `arm_continue` → the follow-up LLM runs and plans the next batch off the real page. `disarm_continue()`s at start (a batch supersedes the passive chain). Confirm-eligible steps use the blocking confirm bar mid-batch.
212
214
  - Confirm policy: `_CONFIRM_EXEMPT_ACTIONS = {click, scroll, select}` ALWAYS auto-run (product decision), even destructive-named, even under `high_risk_confirm`. Only `type` can confirm: `require_confirm=True` when `high_risk_confirm` OR `_is_destructive`.
213
215
  - `_is_destructive`: `_target_name` from `target.text`, else the cp handle looked up in the injector's live summary (actions/content/form fields); matched against the `_DESTRUCTIVE` lexicon (submit/pay/delete/cancel subscription/publish/…); fail-open when no name resolves (destructive intents are named explicitly).
214
216
  - Act-then-wait: successful MUTATING actions return `_suppress_followup_llm: True` (flows manager → `run_llm=False`; single-node adapter translates) — the fresh `[Browser context]` arrives async, so an immediate completion could only act on the STALE page. The optional `say` ack is spoken out-of-band via `_speak` (scrubbed; `state["_speak"]` shim or `task.queue_frame(TTSSpeakFrame)`) so the bot isn't silent. Failures never suppress.
215
- - Single-node adapter: `resolve_copilot_config(call_config)` (env fallbacks `COPILOT_ALLOWED_ORIGINS` / `COPILOT_HIGH_RISK_CONFIRM` / `COPILOT_DISABLED`); `register_single_node_browser_tools(...)` registers the same 8 schemas on bot.py when `copilot_config.enabled`, wrapping each flows handler in a `SimpleNamespace(state=…)` shim — same `_run` guardrails, zero duplication; suppression becomes `FunctionCallResultProperties(run_llm=False)`.
217
+ - Single-node adapter: `resolve_copilot_config(call_config)` (env fallbacks `COPILOT_ALLOWED_ORIGINS` / `COPILOT_HIGH_RISK_CONFIRM` / `COPILOT_DISABLED`); `register_single_node_browser_tools(...)` registers the same 9 schemas on bot.py when `copilot_config.enabled`, wrapping each flows handler in a `SimpleNamespace(state=…)` shim — same `_run` guardrails, zero duplication; suppression becomes `FunctionCallResultProperties(run_llm=False)`.
216
218
 
217
219
  ### bot wiring (copilot-relevant only)
218
220
  - `bot_with_flows.py` — LiveKit channel, `on_connected`: builds `DomContextInjector` (+`set_flow_manager`), resolves `copilot_cfg` from `call_config.copilot_config` with the same env fallbacks, then `LiveKitRPCBridge(room, task, …, copilot_sink=dom_injector.absorb)` + `register_handlers()`; stores `dom_injector` / `copilot_config` / `livekit_rpc` in `flow_manager.state` (read by `_run`). Tools arrive pre-injected in the flow nodes (backend's `enable_browser_copilot_tools_in_flow`) and resolve to the `BROWSER_ACTION_SCHEMAS` handlers. Text mode: `transcript_handler.on_message_added = _relay_chat_reply` → `bridge.send_chat_text` (the widget's chat rendering path).
219
221
  - `bot.py` — single-node: `copilot_state_ref: dict` created before tool build; `register_single_node_browser_tools(call_config=…, state_ref=copilot_state_ref, …)` at tool-registration time (LiveKit channel only); `on_connected` fills the ref with `livekit_rpc` + `dom_injector` (`DomContextInjector.set_context_aggregator(context_aggregator.user())`) — handlers read them lazily through the shim.
220
222
 
221
223
  ### tests
222
- - `tests/test_browser_action_suppression.py` — act-then-wait (mutating success suppresses; reads/highlight/failures never), kill-switch, origin allowlist, click/scroll/select NEVER confirm, destructive `type` confirms by default, `high_risk_confirm` forwarding, `scrub_handles` + say-ack scrubbing, destructive-name lookup via the injector summary, single-node registration (8 tools; skipped when disabled) and adapter `run_llm` translation.
223
- - `tests/test_dom_context_injector.py` — legacy summary renders golden-identical, v2 flags/content/focused/viewport, dropdown options line, hostile `value` key never rendered, content/message caps, activity line.
224
+ - `tests/test_browser_action_suppression.py` — act-then-wait (mutating success suppresses; reads/highlight/failures never), kill-switch, origin allowlist, click/scroll/select NEVER confirm, destructive `type` confirms by default, `high_risk_confirm` forwarding, `scrub_handles` + say-ack scrubbing, destructive-name lookup via the injector summary, single-node registration (9 tools; skipped when disabled) and adapter `run_llm` translation.
225
+ - `tests/test_execute_plan.py` — ordered batch runs then re-perceives, stop-on-failure / denied-confirm / invalid-step, per-step origin & confirm guardrails, kill-switch skips read_dom, `MAX_PLAN_STEPS` truncation, one scrubbed `say`, read_dom-failure `page_error`, single-node adapter runs the LLM (no suppression), `_step_to_action` mapping.
226
+ - `tests/test_dom_context_injector.py` — legacy summary renders golden-identical, v2 flags/content/focused/viewport, dropdown options line + long-list truncation, `[checked]`/`[unchecked]` state, hostile `value` key never rendered, content/message caps, activity line, auto-continue chain + stale-summary watchdog (fires on no-summary, cancelled by summary/disarm, one-shot, respects `MAX_CHAIN`).
224
227
 
225
228
  ## E. Ops
226
229
 
@@ -111,6 +111,17 @@ bot sets `require_confirm` on the `cp_action` when its target is
111
111
  when `copilot_config.high_risk_confirm=true`; the widget shows its confirm bar
112
112
  when that flag is present.
113
113
 
114
+ ### execute_plan — bot-side batching (no new wire action)
115
+
116
+ `execute_plan` is an LLM tool that takes an ordered list of steps and runs them as
117
+ **individual `cp_action`s** in sequence (same wire, same per-step guardrails), then
118
+ issues one `read_dom` and folds the fresh page into `[Browser context]` — the model
119
+ plans the next batch off the real page instead of waiting for the async summary.
120
+ There is NO new `cp_action` verb; the widget is unchanged. Per-step outcomes are
121
+ ActionResult-v1-shaped; confirm-eligible steps use the existing blocking confirm bar
122
+ mid-batch (a `user denied` stops the batch). This revives the retired chat_http
123
+ `one_shot_plan` batching, with active re-perception replacing the `last_results` ack.
124
+
114
125
  ## cp_action ↔ PlannedAction mapping (chat column RETIRED — voice/chat now share cp_action)
115
126
 
116
127
  | voice `cp_action` | chat `PlannedAction.action` (retired) | notes |
@@ -0,0 +1,79 @@
1
+ # Browser Copilot — intelligence backlog
2
+
3
+ Ideas for making the copilot smarter. Guiding principle:
4
+
5
+ > **Prompt** for judgment/planning the LLM is good at (when to reassure, how to
6
+ > decompose a vague ask, which page to open). **Code** for deterministic guarantees
7
+ > the LLM is unreliable at (a filler that MUST fire on slow loads; the multi-step
8
+ > loop that carries a plan across turns).
9
+
10
+ Status legend: **SHIPPED** · **DEFERRED** · **IDEA**.
11
+
12
+ ---
13
+
14
+ ## 1. Optional on-demand screenshots (html2canvas) — DEFERRED
15
+
16
+ An optional per-agent toggle; when on, the agent can call a `read_screen` tool to
17
+ capture a visual of the page (charts, images, exact layout the text DOM view can't
18
+ convey). Chosen approach: **html2canvas, on-demand** — no browser permission prompt,
19
+ page-scoped, captured only when asked.
20
+
21
+ Current state is mostly unbuilt: `ScreenAutoInjector`
22
+ (`ringg_processors/screen_auto_injector.py`) exists but is Daily-only; the LiveKit
23
+ `CT_SCREENSHOT` path is a stub; the widget has no capture and sends data only via
24
+ `performRpc` (~15 KB cap). Build outline:
25
+
26
+ - **Config:** `screenshot: bool` on `CopilotConfig` (`new_calling_agent_backend/calling/schema.py`); rides the existing `copilot_config` threading.
27
+ - **Widget:** add `html2canvas`; implement the `screenshot` cp_action (executor
28
+ renders the page → JPEG data-URL, excluding `[data-copilot-ui]`); `use-copilot.ts`
29
+ sends it over a LiveKit **text stream** (bypasses the RPC size cap).
30
+ - **Bot:** inbound text-stream handler (mirror `utils/participant_handler.py`'s
31
+ `register_text_stream_handler`) → feed a `ScreenAutoInjector` (add an `inject_b64`
32
+ method reusing `_sync_to_context`), wired on the copilot path when the flag is on;
33
+ reinject after node transitions.
34
+ - **Tool:** `read_screen` schema/handler in `browser_actions.py`, registered only
35
+ when the flag is on (single-node + multi-node paths).
36
+
37
+ Caveats: async (the image lands on the agent's next turn); html2canvas can't render
38
+ cross-origin iframes / some CSS / DRM `<video>` / tainted `<canvas>`; keep JPEG
39
+ quality ~0.6 and cap dimensions to bound tokens.
40
+
41
+ ---
42
+
43
+ ## 2. Spoken "waiting / fetching data" filler — PROMPT shipped, code watchdog DEFERRED
44
+
45
+ When the agent triggers something that loads data, the user shouldn't sit in silence.
46
+
47
+ - **Prompt (SHIPPED via the operating-guidance block):** the agent is told to add a
48
+ short spoken `say` ("one moment, pulling that up") when it clicks/submits something
49
+ that loads. `say` is already a param on every action tool.
50
+ - **Code watchdog (DEFERRED):** the LLM is silent during the act-then-wait gap, so
51
+ for genuinely slow loads add a watchdog in
52
+ `ringg_processors/dom_context_injector.py::_maybe_continue` — if the fresh
53
+ perception hasn't arrived within ~N s, queue a `TTSSpeakFrame` filler. The injector
54
+ already holds the task + wait loop. Only worth it if slow loads are common.
55
+
56
+ ---
57
+
58
+ ## 3. Vague ask → navigate → explore → act — PROMPT (SHIPPED)
59
+
60
+ Pure prompt discipline; no new code. The tools (`navigate`/`scroll`,
61
+ `read_dom`/`find_elements`, `click`/`type`/`select`) and the multi-step chaining loop
62
+ already let the LLM do navigate→settle→perceive→act across turns.
63
+
64
+ - **SHIPPED via the operating-guidance block:** "if the target isn't on the current
65
+ screen, navigate first; then read_dom/find_elements to locate it before acting;
66
+ never act on a handle you haven't seen."
67
+ - **IDEA:** give each agent a small site-map (page → what's there) so a vague intent
68
+ resolves to a page. This is knowledge for the LLM, NOT a hard-coded intent→page
69
+ branch (code can't infer "vague ask → which page" without app-specific mapping).
70
+
71
+ ---
72
+
73
+ ## Where shared prompt content lives
74
+
75
+ Behaviors #2 (ack) and #3 (explore-first) ship as a single **operating-guidance
76
+ block injected by code** when copilot is active — the `_COPILOT_GUIDANCE` message in
77
+ `dom_context_injector.py`, kept present once per turn and across RESET* transitions
78
+ via the sticky-context mechanism. Prompt content, delivered by code, once, for every
79
+ copilot agent — no per-agent prompt edits.
@@ -0,0 +1,220 @@
1
+ #!/usr/bin/env python3
2
+ """Azure OpenAI latency probe — run it FROM Azure to isolate the GCP->Azure network hop.
3
+
4
+ We currently call Azure OpenAI from GCP pods, so every LLM request eats a
5
+ cross-cloud round-trip. Run this same script (a) from a GCP pod and (b) from an
6
+ Azure VM in the SAME region as the deployment, then diff the numbers. The
7
+ network-layer stats (DNS/TCP/TLS) show the raw hop cost; the streaming chat
8
+ stats (TTFT/total) show what actually reaches the voice pipeline.
9
+
10
+ Zero repo dependencies — the only third-party import is `openai` (already in our
11
+ images). Config comes entirely from env vars whose names match env_config.py, so
12
+ you can drop this one file on any box and run it.
13
+
14
+ Usage:
15
+ # Point it at whichever deployment you want to test. Defaults to the
16
+ # AZURE_CHATGPT_* (eastus gpt-4.1) deployment.
17
+ export AZURE_CHATGPT_ENDPOINT="https://<resource>.openai.azure.com"
18
+ export AZURE_CHATGPT_API_KEY="..."
19
+ python3 azure_latency_probe.py --model gpt-4.1 --runs 20
20
+
21
+ # Test a different deployment by pointing at its env-var pair:
22
+ python3 azure_latency_probe.py --endpoint-var AZURE_GPT_5 --model gpt-5-mini
23
+
24
+ # Or override endpoint/key directly:
25
+ python3 azure_latency_probe.py --endpoint https://x.openai.azure.com --api-key ... --model gpt-4.1
26
+ """
27
+
28
+ import argparse
29
+ import os
30
+ import socket
31
+ import ssl
32
+ import statistics
33
+ import sys
34
+ import time
35
+ from urllib.parse import urlparse
36
+
37
+ API_VERSION = "2025-04-01-preview"
38
+
39
+ # Small, fixed prompt so inter-environment differences reflect network, not
40
+ # prompt-processing variance. Ask for a bit of output so TTFT is meaningful.
41
+ PROMPT = "Reply with a single sentence describing what a load balancer does."
42
+
43
+
44
+ def _p(name: str, values: list[float]) -> str:
45
+ if not values:
46
+ return f"{name}: (no samples)"
47
+ values = sorted(values)
48
+ mean = statistics.mean(values)
49
+ p50 = statistics.median(values)
50
+ p95 = values[min(len(values) - 1, int(round(0.95 * (len(values) - 1))))]
51
+ return (
52
+ f"{name:<28} n={len(values):<3} "
53
+ f"min={min(values):7.1f} p50={p50:7.1f} mean={mean:7.1f} "
54
+ f"p95={p95:7.1f} max={max(values):7.1f} (ms)"
55
+ )
56
+
57
+
58
+ def measure_network(host: str, port: int, runs: int) -> dict[str, list[float]]:
59
+ """Raw network cost to the endpoint: DNS resolve, TCP connect, TLS handshake.
60
+
61
+ Each run opens a fresh connection (no keep-alive) so we see the full
62
+ cold-path cost every time — this is the part a same-region Azure VM should
63
+ slash relative to a GCP pod.
64
+ """
65
+ stats: dict[str, list[float]] = {"dns": [], "tcp": [], "tls": []}
66
+ ctx = ssl.create_default_context()
67
+ for _ in range(runs):
68
+ try:
69
+ t0 = time.perf_counter()
70
+ infos = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP)
71
+ t_dns = time.perf_counter()
72
+ stats["dns"].append((t_dns - t0) * 1000)
73
+
74
+ family, socktype, proto, _, sockaddr = infos[0]
75
+ sock = socket.socket(family, socktype, proto)
76
+ sock.settimeout(10)
77
+ t1 = time.perf_counter()
78
+ sock.connect(sockaddr)
79
+ t_tcp = time.perf_counter()
80
+ stats["tcp"].append((t_tcp - t1) * 1000)
81
+
82
+ t2 = time.perf_counter()
83
+ ssock = ctx.wrap_socket(sock, server_hostname=host)
84
+ t_tls = time.perf_counter()
85
+ stats["tls"].append((t_tls - t2) * 1000)
86
+ ssock.close()
87
+ except Exception as exc: # noqa: BLE001 — probe should never abort the run
88
+ print(f" network probe error: {exc}", file=sys.stderr)
89
+ return stats
90
+
91
+
92
+ def measure_chat(
93
+ endpoint: str, api_key: str, model: str, runs: int
94
+ ) -> dict[str, list[float]]:
95
+ """Real streaming chat completion: time-to-first-token and total time.
96
+
97
+ Uses the same Azure OpenAI surface the bot uses (dated api_version on the
98
+ /openai/deployments path). Streaming so TTFT is the latency the voice
99
+ pipeline actually feels before it can start speaking.
100
+ """
101
+ try:
102
+ from openai import AzureOpenAI
103
+ except ImportError:
104
+ print(
105
+ "openai package not installed. `pip install openai` and re-run.",
106
+ file=sys.stderr,
107
+ )
108
+ return {"ttft": [], "total": []}
109
+
110
+ client = AzureOpenAI(
111
+ azure_endpoint=endpoint,
112
+ api_key=api_key,
113
+ api_version=API_VERSION,
114
+ max_retries=0,
115
+ )
116
+ stats: dict[str, list[float]] = {"ttft": [], "total": []}
117
+ for i in range(runs):
118
+ try:
119
+ t0 = time.perf_counter()
120
+ stream = client.chat.completions.create(
121
+ model=model,
122
+ messages=[{"role": "user", "content": PROMPT}],
123
+ stream=True,
124
+ max_tokens=64,
125
+ temperature=1,
126
+ )
127
+ first = None
128
+ for chunk in stream:
129
+ # Match pipecat's TTFB semantics exactly (see
130
+ # dv-pipecat base_llm.py::_process_context): TTFB stops on the
131
+ # FIRST chunk that carries a non-empty `choices` list — the
132
+ # opening role delta — NOT the first chunk with `delta.content`.
133
+ # usage-only / empty-choices chunks are skipped, same as pipecat.
134
+ if chunk.choices is None or len(chunk.choices) == 0:
135
+ continue
136
+ if first is None:
137
+ first = time.perf_counter()
138
+ t_end = time.perf_counter()
139
+ if first is not None:
140
+ stats["ttft"].append((first - t0) * 1000)
141
+ stats["total"].append((t_end - t0) * 1000)
142
+ except Exception as exc: # noqa: BLE001
143
+ print(f" chat request {i} error: {exc}", file=sys.stderr)
144
+ return stats
145
+
146
+
147
+ def resolve_config(args) -> tuple[str, str]:
148
+ """Return (endpoint, api_key) from --endpoint/--api-key or the env-var pair."""
149
+ if args.endpoint and args.api_key:
150
+ return args.endpoint.rstrip("/"), args.api_key
151
+ prefix = args.endpoint_var # e.g. "AZURE_CHATGPT"
152
+ endpoint = args.endpoint or os.environ.get(f"{prefix}_ENDPOINT")
153
+ api_key = args.api_key or os.environ.get(f"{prefix}_API_KEY")
154
+ if not endpoint or not api_key:
155
+ print(
156
+ f"Missing config. Set {prefix}_ENDPOINT and {prefix}_API_KEY "
157
+ f"(or pass --endpoint/--api-key).",
158
+ file=sys.stderr,
159
+ )
160
+ sys.exit(2)
161
+ return endpoint.rstrip("/"), api_key
162
+
163
+
164
+ def main() -> None:
165
+ parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
166
+ parser.add_argument(
167
+ "--endpoint-var",
168
+ default="AZURE_CHATGPT",
169
+ help="Env-var prefix to read <PREFIX>_ENDPOINT/<PREFIX>_API_KEY from "
170
+ "(default: AZURE_CHATGPT). e.g. AZURE_GPT_5, AZURE_INDIA, AZURE_SECONDARY.",
171
+ )
172
+ parser.add_argument("--endpoint", help="Override endpoint directly")
173
+ parser.add_argument("--api-key", help="Override API key directly")
174
+ parser.add_argument("--model", default="gpt-4.1", help="Deployment / model name")
175
+ parser.add_argument("--runs", type=int, default=20, help="Iterations per test")
176
+ parser.add_argument(
177
+ "--skip-chat",
178
+ action="store_true",
179
+ help="Only run the network-layer probe (no billed LLM calls)",
180
+ )
181
+ args = parser.parse_args()
182
+
183
+ endpoint, api_key = resolve_config(args)
184
+ host = urlparse(endpoint).hostname or endpoint
185
+ port = urlparse(endpoint).port or 443
186
+
187
+ print(f"Target endpoint : {endpoint}")
188
+ print(f"Host : {host}:{port}")
189
+ print(f"Model : {args.model}")
190
+ print(f"Runs : {args.runs}")
191
+ print(f"api_version : {API_VERSION}")
192
+ print("-" * 78)
193
+
194
+ print("Network layer (fresh connection each run):")
195
+ net = measure_network(host, port, args.runs)
196
+ print(" " + _p("DNS resolve", net["dns"]))
197
+ print(" " + _p("TCP connect", net["tcp"]))
198
+ print(" " + _p("TLS handshake", net["tls"]))
199
+ conn = [d + t + s for d, t, s in zip(net["dns"], net["tcp"], net["tls"])]
200
+ print(" " + _p("connect total (D+T+T)", conn))
201
+ print("-" * 78)
202
+
203
+ if args.skip_chat:
204
+ print("Chat probe skipped (--skip-chat).")
205
+ return
206
+
207
+ print("Streaming chat completion (real LLM request):")
208
+ chat = measure_chat(endpoint, api_key, args.model, args.runs)
209
+ print(" " + _p("TTFT (first token)", chat["ttft"]))
210
+ print(" " + _p("total (full response)", chat["total"]))
211
+ print("-" * 78)
212
+ print(
213
+ "Interpretation: run this from a GCP pod and from a same-region Azure VM.\n"
214
+ "The delta in 'connect total' and 'TTFT' is the cross-cloud network cost\n"
215
+ "you'd eliminate by co-locating the caller with the Azure deployment."
216
+ )
217
+
218
+
219
+ if __name__ == "__main__":
220
+ main()