crowddrop-pubsub-sdk 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- crowddrop_pubsub_sdk-0.2.0/PKG-INFO +181 -0
- crowddrop_pubsub_sdk-0.2.0/README.md +169 -0
- crowddrop_pubsub_sdk-0.2.0/crowddrop_pubsub_sdk.egg-info/PKG-INFO +181 -0
- crowddrop_pubsub_sdk-0.2.0/crowddrop_pubsub_sdk.egg-info/SOURCES.txt +22 -0
- crowddrop_pubsub_sdk-0.2.0/crowddrop_pubsub_sdk.egg-info/dependency_links.txt +1 -0
- crowddrop_pubsub_sdk-0.2.0/crowddrop_pubsub_sdk.egg-info/requires.txt +5 -0
- crowddrop_pubsub_sdk-0.2.0/crowddrop_pubsub_sdk.egg-info/top_level.txt +1 -0
- crowddrop_pubsub_sdk-0.2.0/pubsub_sdk/__init__.py +31 -0
- crowddrop_pubsub_sdk-0.2.0/pubsub_sdk/chat.py +118 -0
- crowddrop_pubsub_sdk-0.2.0/pubsub_sdk/client.py +381 -0
- crowddrop_pubsub_sdk-0.2.0/pubsub_sdk/config.py +43 -0
- crowddrop_pubsub_sdk-0.2.0/pubsub_sdk/events.py +47 -0
- crowddrop_pubsub_sdk-0.2.0/pubsub_sdk/langchain_callback.py +227 -0
- crowddrop_pubsub_sdk-0.2.0/pubsub_sdk/publisher.py +61 -0
- crowddrop_pubsub_sdk-0.2.0/pubsub_sdk/py.typed +0 -0
- crowddrop_pubsub_sdk-0.2.0/pubsub_sdk/subscriber.py +33 -0
- crowddrop_pubsub_sdk-0.2.0/pyproject.toml +20 -0
- crowddrop_pubsub_sdk-0.2.0/setup.cfg +4 -0
- crowddrop_pubsub_sdk-0.2.0/tests/test_chat.py +108 -0
- crowddrop_pubsub_sdk-0.2.0/tests/test_client.py +250 -0
- crowddrop_pubsub_sdk-0.2.0/tests/test_config.py +49 -0
- crowddrop_pubsub_sdk-0.2.0/tests/test_langchain_callback.py +156 -0
- crowddrop_pubsub_sdk-0.2.0/tests/test_publisher.py +163 -0
- crowddrop_pubsub_sdk-0.2.0/tests/test_subscriber.py +159 -0
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: crowddrop-pubsub-sdk
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Framework-agnostic Google Cloud Pub/Sub step-event publisher/subscriber for embodied AI agents.
|
|
5
|
+
Project-URL: Repository, https://github.com/crowddrop-ai/crowddrop_ai_agents
|
|
6
|
+
Requires-Python: >=3.9
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: google-cloud-pubsub<3.0.0,>=2.21.0
|
|
9
|
+
Requires-Dist: google-api-core<3.0.0,>=2.17.1
|
|
10
|
+
Provides-Extra: langchain
|
|
11
|
+
Requires-Dist: langchain-core>=0.3.0; extra == "langchain"
|
|
12
|
+
|
|
13
|
+
# pubsub_sdk
|
|
14
|
+
|
|
15
|
+
A small, framework-agnostic wrapper around Google Cloud Pub/Sub for publishing and
|
|
16
|
+
consuming structured step/progress events, and for direct agent-to-agent
|
|
17
|
+
instruction/reply exchanges (chat). It has no dependency on LangChain, on any
|
|
18
|
+
specific agent framework, or on any particular storage backend — any Python project
|
|
19
|
+
(an agent, a backend service, a future embodied-AI-agent codebase) can import it
|
|
20
|
+
directly.
|
|
21
|
+
|
|
22
|
+
## Install
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install ./pubsub_sdk # core publisher/subscriber only
|
|
26
|
+
pip install ./pubsub_sdk[langchain] # + the optional LangChain callback adapter
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Published to public PyPI as `crowddrop-pubsub-sdk` (the import name stays
|
|
30
|
+
`pubsub_sdk` either way):
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install crowddrop-pubsub-sdk
|
|
34
|
+
pip install crowddrop-pubsub-sdk[langchain]
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Configuration (environment variables)
|
|
38
|
+
|
|
39
|
+
| Variable | Default | Purpose |
|
|
40
|
+
|---|---|---|
|
|
41
|
+
| `GCP_PROJECT_ID` | `test` | GCP project the client talks to. |
|
|
42
|
+
| `PUBSUB_SDK_TOPIC_ID` | `agent-step-events` | Topic events are published to. |
|
|
43
|
+
| `PUBSUB_SDK_SUBSCRIPTION_ID` | `agent-step-events-firestore-persister` | Subscription a consumer pulls from. |
|
|
44
|
+
| `PUBSUB_SDK_CHAT_TOPIC_ID` | `agent-chat-messages` | Fixed topic for agent-to-agent chat (see below). |
|
|
45
|
+
| `PUBSUB_SDK_ENABLED` | `true` | Kill switch — set to `false` to disable publishing/subscribing entirely. |
|
|
46
|
+
| `PUBSUB_EMULATOR_HOST` | unset | Standard Google client env var; set to point at a local Pub/Sub emulator instead of real GCP. |
|
|
47
|
+
|
|
48
|
+
Credentials are picked up automatically via `GOOGLE_APPLICATION_CREDENTIALS`, exactly
|
|
49
|
+
like every other `google-cloud-*` client — no SDK-specific credential handling.
|
|
50
|
+
|
|
51
|
+
## Publishing (any consumer)
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
from pubsub_sdk import StepEventPublisher
|
|
55
|
+
|
|
56
|
+
publisher = StepEventPublisher()
|
|
57
|
+
publisher.publish_step(
|
|
58
|
+
activity_id="task-123",
|
|
59
|
+
agent_id="MissionCommander",
|
|
60
|
+
event_type="tool_start",
|
|
61
|
+
message="Invoking tool: delegate_to_agent",
|
|
62
|
+
status="in_progress",
|
|
63
|
+
metadata={"tool_name": "delegate_to_agent"},
|
|
64
|
+
)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
`publish_step()` never raises — if Pub/Sub is unavailable it logs the failure and
|
|
68
|
+
returns `False`. Check `publisher.is_available()` if you need to know the state
|
|
69
|
+
up front.
|
|
70
|
+
|
|
71
|
+
## Subscribing (any consumer)
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
from pubsub_sdk import StepEventSubscriber, StepEvent
|
|
75
|
+
|
|
76
|
+
def handle(event: StepEvent) -> None:
|
|
77
|
+
... # e.g. persist it somewhere
|
|
78
|
+
|
|
79
|
+
subscriber = StepEventSubscriber()
|
|
80
|
+
subscriber.pull_forever(handle) # blocks; acks on success, nacks (redelivers) on exception
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## Agent-to-agent chat (ChatMessage)
|
|
84
|
+
|
|
85
|
+
A direct instruction/reply exchange between two agent instances (e.g. a
|
|
86
|
+
coordinator persona delegating to a field agent), as an alternative transport to
|
|
87
|
+
a live MCP/SSE connection. Both directions of one exchange ride the same fixed
|
|
88
|
+
topic (`agent-chat-messages`) and are distinguished by `role`
|
|
89
|
+
(`ChatMessageRole.INSTRUCTION` / `.REPLY`), correlated by `message_id`.
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
from pubsub_sdk import ChatMessage, ChatMessageRole, get_default_chat_publisher, ChatMessageSubscriber
|
|
93
|
+
|
|
94
|
+
# Sending side: publish an instruction addressed to another agent instance's
|
|
95
|
+
# routing identity (its session_id - never a shared persona name; two ephemeral
|
|
96
|
+
# instances of the same persona would otherwise collide on the same subscription).
|
|
97
|
+
publisher = get_default_chat_publisher()
|
|
98
|
+
publisher.publish(ChatMessage(
|
|
99
|
+
message_id="m1", from_agent="MissionCommander__task-42", to_agent="PreciseHumanoid",
|
|
100
|
+
role=ChatMessageRole.INSTRUCTION, content="what is your current battery level?",
|
|
101
|
+
activity_id="task-42",
|
|
102
|
+
))
|
|
103
|
+
|
|
104
|
+
# Receiving side: each participating instance self-provisions its own filtered
|
|
105
|
+
# subscription (against real GCP, not just the emulator - a per-instance
|
|
106
|
+
# subscription's identity only exists at runtime and can't be pre-provisioned).
|
|
107
|
+
subscriber = ChatMessageSubscriber(routing_identity="PreciseHumanoid")
|
|
108
|
+
subscriber.pull_forever(lambda msg: ...) # blocks; ack on success, nack (redeliver) on exception
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
`GenericSubscriber.pull_forever` also accepts a `stop()` call from another thread
|
|
112
|
+
to cancel an in-progress streaming pull cleanly.
|
|
113
|
+
|
|
114
|
+
**Security note:** `agent-chat-messages` is one shared topic with no
|
|
115
|
+
per-agent access control beyond GCP IAM on the topic/subscription resources
|
|
116
|
+
themselves - any credential with publish/subscribe rights can spoof
|
|
117
|
+
`from_agent` or read another agent's traffic via an unfiltered subscription.
|
|
118
|
+
Fine within one trust domain (this app's own fleet today); see
|
|
119
|
+
`docs/pubsub_chat_access_control.md` before granting a genuinely independent/
|
|
120
|
+
external party a credential onto this topic.
|
|
121
|
+
|
|
122
|
+
## LangChain integration (optional extra)
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
from pubsub_sdk.langchain_callback import StepEventCallbackHandler
|
|
126
|
+
|
|
127
|
+
callback = StepEventCallbackHandler(activity_id="task-123", agent_id="MissionCommander")
|
|
128
|
+
callback.log_mission_start()
|
|
129
|
+
# pass `callback` into a LangChain AgentExecutor's `config={"callbacks": [callback]}` —
|
|
130
|
+
# every LLM/tool/chain/agent lifecycle event then publishes automatically.
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
`langchain_callback` is the only module in this package that imports `langchain` —
|
|
134
|
+
a consumer who only needs `StepEventPublisher`/`StepEventSubscriber` never pulls
|
|
135
|
+
that dependency in.
|
|
136
|
+
|
|
137
|
+
## Releasing (publishing a new version to PyPI)
|
|
138
|
+
|
|
139
|
+
Releases are tag-triggered via `.github/workflows/publish-pubsub-sdk.yml`,
|
|
140
|
+
using PyPI's **Trusted Publishing** (OIDC) — no API token is stored as a
|
|
141
|
+
GitHub secret.
|
|
142
|
+
|
|
143
|
+
1. Bump `version` in `pubsub_sdk/pyproject.toml`.
|
|
144
|
+
2. Commit that change (on a branch, via the normal PR flow).
|
|
145
|
+
3. Once merged, tag the merge commit and push the tag:
|
|
146
|
+
```bash
|
|
147
|
+
git tag pubsub-sdk-v<version> # e.g. pubsub-sdk-v0.2.1
|
|
148
|
+
git push origin pubsub-sdk-v<version>
|
|
149
|
+
```
|
|
150
|
+
The tag push is what fires the workflow — it builds `pubsub_sdk/` and
|
|
151
|
+
uploads it to [pypi.org/project/crowddrop-pubsub-sdk](https://pypi.org/project/crowddrop-pubsub-sdk/).
|
|
152
|
+
No other trigger publishes this package.
|
|
153
|
+
|
|
154
|
+
**One-time setup, not yet done as of this writing — needed before the first
|
|
155
|
+
tag push, and again only if this ever moves to a different PyPI
|
|
156
|
+
account/org:**
|
|
157
|
+
- Register a **pending publisher** for `crowddrop-pubsub-sdk` at
|
|
158
|
+
https://pypi.org/manage/account/publishing/ — this can be done before the
|
|
159
|
+
PyPI project exists, so it covers the *first-ever* release too, not just
|
|
160
|
+
subsequent ones. Fill in: PyPI project name `crowddrop-pubsub-sdk`, repo
|
|
161
|
+
owner `crowddrop-ai`, repo name `crowddrop_ai_agents`, workflow filename
|
|
162
|
+
`publish-pubsub-sdk.yml`, environment name `pypi`. Requires a PyPI account
|
|
163
|
+
with 2FA enabled — no API token to generate or store.
|
|
164
|
+
- The `pypi` GitHub Environment referenced by the workflow is created
|
|
165
|
+
automatically the first time the workflow runs against it; create it
|
|
166
|
+
manually in this repo's Settings → Environments beforehand only if you
|
|
167
|
+
want a required-reviewer protection rule (so a tag push pauses for human
|
|
168
|
+
approval before it actually publishes).
|
|
169
|
+
- `../scripts/publish_python_packages.sh` (manual `build` + `twine upload`)
|
|
170
|
+
is kept as a fallback/local-dry-run tool only — with a pending publisher
|
|
171
|
+
registered, it's no longer needed even for the first release.
|
|
172
|
+
|
|
173
|
+
Versioning is manual — nothing cross-checks the tag against
|
|
174
|
+
`pyproject.toml`'s `version`. Bump the file first, commit, *then* tag that
|
|
175
|
+
exact commit; tagging a commit whose `pyproject.toml` still has an
|
|
176
|
+
already-published version will fail the upload (PyPI rejects re-uploading an
|
|
177
|
+
existing version).
|
|
178
|
+
|
|
179
|
+
Note: `crowddrop-sdk`'s `cloud-brain` extra depends on `crowddrop-pubsub-sdk`
|
|
180
|
+
(see `../crowddrop_sdk/pyproject.toml`) — release this package first if both
|
|
181
|
+
need a coordinated update.
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
# pubsub_sdk
|
|
2
|
+
|
|
3
|
+
A small, framework-agnostic wrapper around Google Cloud Pub/Sub for publishing and
|
|
4
|
+
consuming structured step/progress events, and for direct agent-to-agent
|
|
5
|
+
instruction/reply exchanges (chat). It has no dependency on LangChain, on any
|
|
6
|
+
specific agent framework, or on any particular storage backend — any Python project
|
|
7
|
+
(an agent, a backend service, a future embodied-AI-agent codebase) can import it
|
|
8
|
+
directly.
|
|
9
|
+
|
|
10
|
+
## Install
|
|
11
|
+
|
|
12
|
+
```bash
|
|
13
|
+
pip install ./pubsub_sdk # core publisher/subscriber only
|
|
14
|
+
pip install ./pubsub_sdk[langchain] # + the optional LangChain callback adapter
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Published to public PyPI as `crowddrop-pubsub-sdk` (the import name stays
|
|
18
|
+
`pubsub_sdk` either way):
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
pip install crowddrop-pubsub-sdk
|
|
22
|
+
pip install crowddrop-pubsub-sdk[langchain]
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## Configuration (environment variables)
|
|
26
|
+
|
|
27
|
+
| Variable | Default | Purpose |
|
|
28
|
+
|---|---|---|
|
|
29
|
+
| `GCP_PROJECT_ID` | `test` | GCP project the client talks to. |
|
|
30
|
+
| `PUBSUB_SDK_TOPIC_ID` | `agent-step-events` | Topic events are published to. |
|
|
31
|
+
| `PUBSUB_SDK_SUBSCRIPTION_ID` | `agent-step-events-firestore-persister` | Subscription a consumer pulls from. |
|
|
32
|
+
| `PUBSUB_SDK_CHAT_TOPIC_ID` | `agent-chat-messages` | Fixed topic for agent-to-agent chat (see below). |
|
|
33
|
+
| `PUBSUB_SDK_ENABLED` | `true` | Kill switch — set to `false` to disable publishing/subscribing entirely. |
|
|
34
|
+
| `PUBSUB_EMULATOR_HOST` | unset | Standard Google client env var; set to point at a local Pub/Sub emulator instead of real GCP. |
|
|
35
|
+
|
|
36
|
+
Credentials are picked up automatically via `GOOGLE_APPLICATION_CREDENTIALS`, exactly
|
|
37
|
+
like every other `google-cloud-*` client — no SDK-specific credential handling.
|
|
38
|
+
|
|
39
|
+
## Publishing (any consumer)
|
|
40
|
+
|
|
41
|
+
```python
|
|
42
|
+
from pubsub_sdk import StepEventPublisher
|
|
43
|
+
|
|
44
|
+
publisher = StepEventPublisher()
|
|
45
|
+
publisher.publish_step(
|
|
46
|
+
activity_id="task-123",
|
|
47
|
+
agent_id="MissionCommander",
|
|
48
|
+
event_type="tool_start",
|
|
49
|
+
message="Invoking tool: delegate_to_agent",
|
|
50
|
+
status="in_progress",
|
|
51
|
+
metadata={"tool_name": "delegate_to_agent"},
|
|
52
|
+
)
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
`publish_step()` never raises — if Pub/Sub is unavailable it logs the failure and
|
|
56
|
+
returns `False`. Check `publisher.is_available()` if you need to know the state
|
|
57
|
+
up front.
|
|
58
|
+
|
|
59
|
+
## Subscribing (any consumer)
|
|
60
|
+
|
|
61
|
+
```python
|
|
62
|
+
from pubsub_sdk import StepEventSubscriber, StepEvent
|
|
63
|
+
|
|
64
|
+
def handle(event: StepEvent) -> None:
|
|
65
|
+
... # e.g. persist it somewhere
|
|
66
|
+
|
|
67
|
+
subscriber = StepEventSubscriber()
|
|
68
|
+
subscriber.pull_forever(handle) # blocks; acks on success, nacks (redelivers) on exception
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Agent-to-agent chat (ChatMessage)
|
|
72
|
+
|
|
73
|
+
A direct instruction/reply exchange between two agent instances (e.g. a
|
|
74
|
+
coordinator persona delegating to a field agent), as an alternative transport to
|
|
75
|
+
a live MCP/SSE connection. Both directions of one exchange ride the same fixed
|
|
76
|
+
topic (`agent-chat-messages`) and are distinguished by `role`
|
|
77
|
+
(`ChatMessageRole.INSTRUCTION` / `.REPLY`), correlated by `message_id`.
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
from pubsub_sdk import ChatMessage, ChatMessageRole, get_default_chat_publisher, ChatMessageSubscriber
|
|
81
|
+
|
|
82
|
+
# Sending side: publish an instruction addressed to another agent instance's
|
|
83
|
+
# routing identity (its session_id - never a shared persona name; two ephemeral
|
|
84
|
+
# instances of the same persona would otherwise collide on the same subscription).
|
|
85
|
+
publisher = get_default_chat_publisher()
|
|
86
|
+
publisher.publish(ChatMessage(
|
|
87
|
+
message_id="m1", from_agent="MissionCommander__task-42", to_agent="PreciseHumanoid",
|
|
88
|
+
role=ChatMessageRole.INSTRUCTION, content="what is your current battery level?",
|
|
89
|
+
activity_id="task-42",
|
|
90
|
+
))
|
|
91
|
+
|
|
92
|
+
# Receiving side: each participating instance self-provisions its own filtered
|
|
93
|
+
# subscription (against real GCP, not just the emulator - a per-instance
|
|
94
|
+
# subscription's identity only exists at runtime and can't be pre-provisioned).
|
|
95
|
+
subscriber = ChatMessageSubscriber(routing_identity="PreciseHumanoid")
|
|
96
|
+
subscriber.pull_forever(lambda msg: ...) # blocks; ack on success, nack (redeliver) on exception
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
`GenericSubscriber.pull_forever` also accepts a `stop()` call from another thread
|
|
100
|
+
to cancel an in-progress streaming pull cleanly.
|
|
101
|
+
|
|
102
|
+
**Security note:** `agent-chat-messages` is one shared topic with no
|
|
103
|
+
per-agent access control beyond GCP IAM on the topic/subscription resources
|
|
104
|
+
themselves - any credential with publish/subscribe rights can spoof
|
|
105
|
+
`from_agent` or read another agent's traffic via an unfiltered subscription.
|
|
106
|
+
Fine within one trust domain (this app's own fleet today); see
|
|
107
|
+
`docs/pubsub_chat_access_control.md` before granting a genuinely independent/
|
|
108
|
+
external party a credential onto this topic.
|
|
109
|
+
|
|
110
|
+
## LangChain integration (optional extra)
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
from pubsub_sdk.langchain_callback import StepEventCallbackHandler
|
|
114
|
+
|
|
115
|
+
callback = StepEventCallbackHandler(activity_id="task-123", agent_id="MissionCommander")
|
|
116
|
+
callback.log_mission_start()
|
|
117
|
+
# pass `callback` into a LangChain AgentExecutor's `config={"callbacks": [callback]}` —
|
|
118
|
+
# every LLM/tool/chain/agent lifecycle event then publishes automatically.
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
`langchain_callback` is the only module in this package that imports `langchain` —
|
|
122
|
+
a consumer who only needs `StepEventPublisher`/`StepEventSubscriber` never pulls
|
|
123
|
+
that dependency in.
|
|
124
|
+
|
|
125
|
+
## Releasing (publishing a new version to PyPI)
|
|
126
|
+
|
|
127
|
+
Releases are tag-triggered via `.github/workflows/publish-pubsub-sdk.yml`,
|
|
128
|
+
using PyPI's **Trusted Publishing** (OIDC) — no API token is stored as a
|
|
129
|
+
GitHub secret.
|
|
130
|
+
|
|
131
|
+
1. Bump `version` in `pubsub_sdk/pyproject.toml`.
|
|
132
|
+
2. Commit that change (on a branch, via the normal PR flow).
|
|
133
|
+
3. Once merged, tag the merge commit and push the tag:
|
|
134
|
+
```bash
|
|
135
|
+
git tag pubsub-sdk-v<version> # e.g. pubsub-sdk-v0.2.1
|
|
136
|
+
git push origin pubsub-sdk-v<version>
|
|
137
|
+
```
|
|
138
|
+
The tag push is what fires the workflow — it builds `pubsub_sdk/` and
|
|
139
|
+
uploads it to [pypi.org/project/crowddrop-pubsub-sdk](https://pypi.org/project/crowddrop-pubsub-sdk/).
|
|
140
|
+
No other trigger publishes this package.
|
|
141
|
+
|
|
142
|
+
**One-time setup, not yet done as of this writing — needed before the first
|
|
143
|
+
tag push, and again only if this ever moves to a different PyPI
|
|
144
|
+
account/org:**
|
|
145
|
+
- Register a **pending publisher** for `crowddrop-pubsub-sdk` at
|
|
146
|
+
https://pypi.org/manage/account/publishing/ — this can be done before the
|
|
147
|
+
PyPI project exists, so it covers the *first-ever* release too, not just
|
|
148
|
+
subsequent ones. Fill in: PyPI project name `crowddrop-pubsub-sdk`, repo
|
|
149
|
+
owner `crowddrop-ai`, repo name `crowddrop_ai_agents`, workflow filename
|
|
150
|
+
`publish-pubsub-sdk.yml`, environment name `pypi`. Requires a PyPI account
|
|
151
|
+
with 2FA enabled — no API token to generate or store.
|
|
152
|
+
- The `pypi` GitHub Environment referenced by the workflow is created
|
|
153
|
+
automatically the first time the workflow runs against it; create it
|
|
154
|
+
manually in this repo's Settings → Environments beforehand only if you
|
|
155
|
+
want a required-reviewer protection rule (so a tag push pauses for human
|
|
156
|
+
approval before it actually publishes).
|
|
157
|
+
- `../scripts/publish_python_packages.sh` (manual `build` + `twine upload`)
|
|
158
|
+
is kept as a fallback/local-dry-run tool only — with a pending publisher
|
|
159
|
+
registered, it's no longer needed even for the first release.
|
|
160
|
+
|
|
161
|
+
Versioning is manual — nothing cross-checks the tag against
|
|
162
|
+
`pyproject.toml`'s `version`. Bump the file first, commit, *then* tag that
|
|
163
|
+
exact commit; tagging a commit whose `pyproject.toml` still has an
|
|
164
|
+
already-published version will fail the upload (PyPI rejects re-uploading an
|
|
165
|
+
existing version).
|
|
166
|
+
|
|
167
|
+
Note: `crowddrop-sdk`'s `cloud-brain` extra depends on `crowddrop-pubsub-sdk`
|
|
168
|
+
(see `../crowddrop_sdk/pyproject.toml`) — release this package first if both
|
|
169
|
+
need a coordinated update.
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: crowddrop-pubsub-sdk
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Framework-agnostic Google Cloud Pub/Sub step-event publisher/subscriber for embodied AI agents.
|
|
5
|
+
Project-URL: Repository, https://github.com/crowddrop-ai/crowddrop_ai_agents
|
|
6
|
+
Requires-Python: >=3.9
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: google-cloud-pubsub<3.0.0,>=2.21.0
|
|
9
|
+
Requires-Dist: google-api-core<3.0.0,>=2.17.1
|
|
10
|
+
Provides-Extra: langchain
|
|
11
|
+
Requires-Dist: langchain-core>=0.3.0; extra == "langchain"
|
|
12
|
+
|
|
13
|
+
# pubsub_sdk
|
|
14
|
+
|
|
15
|
+
A small, framework-agnostic wrapper around Google Cloud Pub/Sub for publishing and
|
|
16
|
+
consuming structured step/progress events, and for direct agent-to-agent
|
|
17
|
+
instruction/reply exchanges (chat). It has no dependency on LangChain, on any
|
|
18
|
+
specific agent framework, or on any particular storage backend — any Python project
|
|
19
|
+
(an agent, a backend service, a future embodied-AI-agent codebase) can import it
|
|
20
|
+
directly.
|
|
21
|
+
|
|
22
|
+
## Install
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install ./pubsub_sdk # core publisher/subscriber only
|
|
26
|
+
pip install ./pubsub_sdk[langchain] # + the optional LangChain callback adapter
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Published to public PyPI as `crowddrop-pubsub-sdk` (the import name stays
|
|
30
|
+
`pubsub_sdk` either way):
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install crowddrop-pubsub-sdk
|
|
34
|
+
pip install crowddrop-pubsub-sdk[langchain]
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Configuration (environment variables)
|
|
38
|
+
|
|
39
|
+
| Variable | Default | Purpose |
|
|
40
|
+
|---|---|---|
|
|
41
|
+
| `GCP_PROJECT_ID` | `test` | GCP project the client talks to. |
|
|
42
|
+
| `PUBSUB_SDK_TOPIC_ID` | `agent-step-events` | Topic events are published to. |
|
|
43
|
+
| `PUBSUB_SDK_SUBSCRIPTION_ID` | `agent-step-events-firestore-persister` | Subscription a consumer pulls from. |
|
|
44
|
+
| `PUBSUB_SDK_CHAT_TOPIC_ID` | `agent-chat-messages` | Fixed topic for agent-to-agent chat (see below). |
|
|
45
|
+
| `PUBSUB_SDK_ENABLED` | `true` | Kill switch — set to `false` to disable publishing/subscribing entirely. |
|
|
46
|
+
| `PUBSUB_EMULATOR_HOST` | unset | Standard Google client env var; set to point at a local Pub/Sub emulator instead of real GCP. |
|
|
47
|
+
|
|
48
|
+
Credentials are picked up automatically via `GOOGLE_APPLICATION_CREDENTIALS`, exactly
|
|
49
|
+
like every other `google-cloud-*` client — no SDK-specific credential handling.
|
|
50
|
+
|
|
51
|
+
## Publishing (any consumer)
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
from pubsub_sdk import StepEventPublisher
|
|
55
|
+
|
|
56
|
+
publisher = StepEventPublisher()
|
|
57
|
+
publisher.publish_step(
|
|
58
|
+
activity_id="task-123",
|
|
59
|
+
agent_id="MissionCommander",
|
|
60
|
+
event_type="tool_start",
|
|
61
|
+
message="Invoking tool: delegate_to_agent",
|
|
62
|
+
status="in_progress",
|
|
63
|
+
metadata={"tool_name": "delegate_to_agent"},
|
|
64
|
+
)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
`publish_step()` never raises — if Pub/Sub is unavailable it logs the failure and
|
|
68
|
+
returns `False`. Check `publisher.is_available()` if you need to know the state
|
|
69
|
+
up front.
|
|
70
|
+
|
|
71
|
+
## Subscribing (any consumer)
|
|
72
|
+
|
|
73
|
+
```python
|
|
74
|
+
from pubsub_sdk import StepEventSubscriber, StepEvent
|
|
75
|
+
|
|
76
|
+
def handle(event: StepEvent) -> None:
|
|
77
|
+
... # e.g. persist it somewhere
|
|
78
|
+
|
|
79
|
+
subscriber = StepEventSubscriber()
|
|
80
|
+
subscriber.pull_forever(handle) # blocks; acks on success, nacks (redelivers) on exception
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
## Agent-to-agent chat (ChatMessage)
|
|
84
|
+
|
|
85
|
+
A direct instruction/reply exchange between two agent instances (e.g. a
|
|
86
|
+
coordinator persona delegating to a field agent), as an alternative transport to
|
|
87
|
+
a live MCP/SSE connection. Both directions of one exchange ride the same fixed
|
|
88
|
+
topic (`agent-chat-messages`) and are distinguished by `role`
|
|
89
|
+
(`ChatMessageRole.INSTRUCTION` / `.REPLY`), correlated by `message_id`.
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
from pubsub_sdk import ChatMessage, ChatMessageRole, get_default_chat_publisher, ChatMessageSubscriber
|
|
93
|
+
|
|
94
|
+
# Sending side: publish an instruction addressed to another agent instance's
|
|
95
|
+
# routing identity (its session_id - never a shared persona name; two ephemeral
|
|
96
|
+
# instances of the same persona would otherwise collide on the same subscription).
|
|
97
|
+
publisher = get_default_chat_publisher()
|
|
98
|
+
publisher.publish(ChatMessage(
|
|
99
|
+
message_id="m1", from_agent="MissionCommander__task-42", to_agent="PreciseHumanoid",
|
|
100
|
+
role=ChatMessageRole.INSTRUCTION, content="what is your current battery level?",
|
|
101
|
+
activity_id="task-42",
|
|
102
|
+
))
|
|
103
|
+
|
|
104
|
+
# Receiving side: each participating instance self-provisions its own filtered
|
|
105
|
+
# subscription (against real GCP, not just the emulator - a per-instance
|
|
106
|
+
# subscription's identity only exists at runtime and can't be pre-provisioned).
|
|
107
|
+
subscriber = ChatMessageSubscriber(routing_identity="PreciseHumanoid")
|
|
108
|
+
subscriber.pull_forever(lambda msg: ...) # blocks; ack on success, nack (redeliver) on exception
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
`GenericSubscriber.pull_forever` also accepts a `stop()` call from another thread
|
|
112
|
+
to cancel an in-progress streaming pull cleanly.
|
|
113
|
+
|
|
114
|
+
**Security note:** `agent-chat-messages` is one shared topic with no
|
|
115
|
+
per-agent access control beyond GCP IAM on the topic/subscription resources
|
|
116
|
+
themselves - any credential with publish/subscribe rights can spoof
|
|
117
|
+
`from_agent` or read another agent's traffic via an unfiltered subscription.
|
|
118
|
+
Fine within one trust domain (this app's own fleet today); see
|
|
119
|
+
`docs/pubsub_chat_access_control.md` before granting a genuinely independent/
|
|
120
|
+
external party a credential onto this topic.
|
|
121
|
+
|
|
122
|
+
## LangChain integration (optional extra)
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
from pubsub_sdk.langchain_callback import StepEventCallbackHandler
|
|
126
|
+
|
|
127
|
+
callback = StepEventCallbackHandler(activity_id="task-123", agent_id="MissionCommander")
|
|
128
|
+
callback.log_mission_start()
|
|
129
|
+
# pass `callback` into a LangChain AgentExecutor's `config={"callbacks": [callback]}` —
|
|
130
|
+
# every LLM/tool/chain/agent lifecycle event then publishes automatically.
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
`langchain_callback` is the only module in this package that imports `langchain` —
|
|
134
|
+
a consumer who only needs `StepEventPublisher`/`StepEventSubscriber` never pulls
|
|
135
|
+
that dependency in.
|
|
136
|
+
|
|
137
|
+
## Releasing (publishing a new version to PyPI)
|
|
138
|
+
|
|
139
|
+
Releases are tag-triggered via `.github/workflows/publish-pubsub-sdk.yml`,
|
|
140
|
+
using PyPI's **Trusted Publishing** (OIDC) — no API token is stored as a
|
|
141
|
+
GitHub secret.
|
|
142
|
+
|
|
143
|
+
1. Bump `version` in `pubsub_sdk/pyproject.toml`.
|
|
144
|
+
2. Commit that change (on a branch, via the normal PR flow).
|
|
145
|
+
3. Once merged, tag the merge commit and push the tag:
|
|
146
|
+
```bash
|
|
147
|
+
git tag pubsub-sdk-v<version> # e.g. pubsub-sdk-v0.2.1
|
|
148
|
+
git push origin pubsub-sdk-v<version>
|
|
149
|
+
```
|
|
150
|
+
The tag push is what fires the workflow — it builds `pubsub_sdk/` and
|
|
151
|
+
uploads it to [pypi.org/project/crowddrop-pubsub-sdk](https://pypi.org/project/crowddrop-pubsub-sdk/).
|
|
152
|
+
No other trigger publishes this package.
|
|
153
|
+
|
|
154
|
+
**One-time setup, not yet done as of this writing — needed before the first
|
|
155
|
+
tag push, and again only if this ever moves to a different PyPI
|
|
156
|
+
account/org:**
|
|
157
|
+
- Register a **pending publisher** for `crowddrop-pubsub-sdk` at
|
|
158
|
+
https://pypi.org/manage/account/publishing/ — this can be done before the
|
|
159
|
+
PyPI project exists, so it covers the *first-ever* release too, not just
|
|
160
|
+
subsequent ones. Fill in: PyPI project name `crowddrop-pubsub-sdk`, repo
|
|
161
|
+
owner `crowddrop-ai`, repo name `crowddrop_ai_agents`, workflow filename
|
|
162
|
+
`publish-pubsub-sdk.yml`, environment name `pypi`. Requires a PyPI account
|
|
163
|
+
with 2FA enabled — no API token to generate or store.
|
|
164
|
+
- The `pypi` GitHub Environment referenced by the workflow is created
|
|
165
|
+
automatically the first time the workflow runs against it; create it
|
|
166
|
+
manually in this repo's Settings → Environments beforehand only if you
|
|
167
|
+
want a required-reviewer protection rule (so a tag push pauses for human
|
|
168
|
+
approval before it actually publishes).
|
|
169
|
+
- `../scripts/publish_python_packages.sh` (manual `build` + `twine upload`)
|
|
170
|
+
is kept as a fallback/local-dry-run tool only — with a pending publisher
|
|
171
|
+
registered, it's no longer needed even for the first release.
|
|
172
|
+
|
|
173
|
+
Versioning is manual — nothing cross-checks the tag against
|
|
174
|
+
`pyproject.toml`'s `version`. Bump the file first, commit, *then* tag that
|
|
175
|
+
exact commit; tagging a commit whose `pyproject.toml` still has an
|
|
176
|
+
already-published version will fail the upload (PyPI rejects re-uploading an
|
|
177
|
+
existing version).
|
|
178
|
+
|
|
179
|
+
Note: `crowddrop-sdk`'s `cloud-brain` extra depends on `crowddrop-pubsub-sdk`
|
|
180
|
+
(see `../crowddrop_sdk/pyproject.toml`) — release this package first if both
|
|
181
|
+
need a coordinated update.
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
crowddrop_pubsub_sdk.egg-info/PKG-INFO
|
|
4
|
+
crowddrop_pubsub_sdk.egg-info/SOURCES.txt
|
|
5
|
+
crowddrop_pubsub_sdk.egg-info/dependency_links.txt
|
|
6
|
+
crowddrop_pubsub_sdk.egg-info/requires.txt
|
|
7
|
+
crowddrop_pubsub_sdk.egg-info/top_level.txt
|
|
8
|
+
pubsub_sdk/__init__.py
|
|
9
|
+
pubsub_sdk/chat.py
|
|
10
|
+
pubsub_sdk/client.py
|
|
11
|
+
pubsub_sdk/config.py
|
|
12
|
+
pubsub_sdk/events.py
|
|
13
|
+
pubsub_sdk/langchain_callback.py
|
|
14
|
+
pubsub_sdk/publisher.py
|
|
15
|
+
pubsub_sdk/py.typed
|
|
16
|
+
pubsub_sdk/subscriber.py
|
|
17
|
+
tests/test_chat.py
|
|
18
|
+
tests/test_client.py
|
|
19
|
+
tests/test_config.py
|
|
20
|
+
tests/test_langchain_callback.py
|
|
21
|
+
tests/test_publisher.py
|
|
22
|
+
tests/test_subscriber.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
pubsub_sdk
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Framework-agnostic Google Cloud Pub/Sub step-event publisher/subscriber.
|
|
2
|
+
|
|
3
|
+
The LangChain adapter (StepEventCallbackHandler) is intentionally NOT re-exported
|
|
4
|
+
here — importing it requires the optional `langchain` extra, and this top-level
|
|
5
|
+
package must stay importable without that dependency. Import it explicitly from
|
|
6
|
+
`pubsub_sdk.langchain_callback` when you need it.
|
|
7
|
+
"""
|
|
8
|
+
from .chat import (
|
|
9
|
+
ChatMessage,
|
|
10
|
+
ChatMessagePublisher,
|
|
11
|
+
ChatMessageRole,
|
|
12
|
+
ChatMessageSubscriber,
|
|
13
|
+
get_default_chat_publisher,
|
|
14
|
+
)
|
|
15
|
+
from .config import PubSubConfig
|
|
16
|
+
from .events import StepEvent
|
|
17
|
+
from .publisher import StepEventPublisher, get_default_publisher
|
|
18
|
+
from .subscriber import StepEventSubscriber
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"PubSubConfig",
|
|
22
|
+
"StepEvent",
|
|
23
|
+
"StepEventPublisher",
|
|
24
|
+
"StepEventSubscriber",
|
|
25
|
+
"get_default_publisher",
|
|
26
|
+
"ChatMessage",
|
|
27
|
+
"ChatMessageRole",
|
|
28
|
+
"ChatMessagePublisher",
|
|
29
|
+
"ChatMessageSubscriber",
|
|
30
|
+
"get_default_chat_publisher",
|
|
31
|
+
]
|