stemtrace 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. stemtrace-0.1.0/.bumpversion.toml +33 -0
  2. stemtrace-0.1.0/.context/active-context.md +141 -0
  3. stemtrace-0.1.0/.context/product-context.md +108 -0
  4. stemtrace-0.1.0/.context/progress.md +383 -0
  5. stemtrace-0.1.0/.context/project-brief.md +70 -0
  6. stemtrace-0.1.0/.context/project-structure.md +124 -0
  7. stemtrace-0.1.0/.context/strategic-analysis.md +174 -0
  8. stemtrace-0.1.0/.context/system-patterns.md +265 -0
  9. stemtrace-0.1.0/.context/tech-context.md +223 -0
  10. stemtrace-0.1.0/.cursor/commands/audit-context.md +110 -0
  11. stemtrace-0.1.0/.cursor/commands/commit.md +89 -0
  12. stemtrace-0.1.0/.cursor/commands/pr-review.md +233 -0
  13. stemtrace-0.1.0/.cursor/commands/pr.md +149 -0
  14. stemtrace-0.1.0/.cursor/rules/architecture.mdc +111 -0
  15. stemtrace-0.1.0/.cursor/rules/open-source.mdc +261 -0
  16. stemtrace-0.1.0/.cursor/rules/project.mdc +79 -0
  17. stemtrace-0.1.0/.cursor/rules/python.mdc +136 -0
  18. stemtrace-0.1.0/.cursor/rules/testing.mdc +99 -0
  19. stemtrace-0.1.0/.cursor/rules/ui-websocket.mdc +278 -0
  20. stemtrace-0.1.0/.cursor/rules/workflow.mdc +208 -0
  21. stemtrace-0.1.0/.dockerignore +64 -0
  22. stemtrace-0.1.0/.github/dependabot.yml +22 -0
  23. stemtrace-0.1.0/.github/workflows/ci.yml +150 -0
  24. stemtrace-0.1.0/.github/workflows/e2e.yml +154 -0
  25. stemtrace-0.1.0/.github/workflows/release.yml +163 -0
  26. stemtrace-0.1.0/.gitignore +35 -0
  27. stemtrace-0.1.0/.pre-commit-config.yaml +30 -0
  28. stemtrace-0.1.0/.python-version +1 -0
  29. stemtrace-0.1.0/CHANGELOG.md +33 -0
  30. stemtrace-0.1.0/CONTRIBUTING.md +144 -0
  31. stemtrace-0.1.0/Dockerfile +58 -0
  32. stemtrace-0.1.0/Dockerfile.e2e +27 -0
  33. stemtrace-0.1.0/LICENSE +21 -0
  34. stemtrace-0.1.0/Makefile +189 -0
  35. stemtrace-0.1.0/PKG-INFO +428 -0
  36. stemtrace-0.1.0/README.md +373 -0
  37. stemtrace-0.1.0/build_ui.py +54 -0
  38. stemtrace-0.1.0/docker-compose.e2e.yml +60 -0
  39. stemtrace-0.1.0/docker-compose.yml +50 -0
  40. stemtrace-0.1.0/examples/celery_app.py +293 -0
  41. stemtrace-0.1.0/examples/fastapi_integration.py +57 -0
  42. stemtrace-0.1.0/examples/with_auth.py +45 -0
  43. stemtrace-0.1.0/pyproject.toml +198 -0
  44. stemtrace-0.1.0/src/stemtrace/__init__.py +173 -0
  45. stemtrace-0.1.0/src/stemtrace/core/__init__.py +15 -0
  46. stemtrace-0.1.0/src/stemtrace/core/events.py +67 -0
  47. stemtrace-0.1.0/src/stemtrace/core/exceptions.py +25 -0
  48. stemtrace-0.1.0/src/stemtrace/core/graph.py +348 -0
  49. stemtrace-0.1.0/src/stemtrace/core/ports.py +42 -0
  50. stemtrace-0.1.0/src/stemtrace/library/__init__.py +1 -0
  51. stemtrace-0.1.0/src/stemtrace/library/bootsteps.py +169 -0
  52. stemtrace-0.1.0/src/stemtrace/library/config.py +55 -0
  53. stemtrace-0.1.0/src/stemtrace/library/scrubbing.py +195 -0
  54. stemtrace-0.1.0/src/stemtrace/library/signals.py +434 -0
  55. stemtrace-0.1.0/src/stemtrace/library/transports/__init__.py +36 -0
  56. stemtrace-0.1.0/src/stemtrace/library/transports/memory.py +33 -0
  57. stemtrace-0.1.0/src/stemtrace/library/transports/redis.py +92 -0
  58. stemtrace-0.1.0/src/stemtrace/py.typed +0 -0
  59. stemtrace-0.1.0/src/stemtrace/server/__init__.py +24 -0
  60. stemtrace-0.1.0/src/stemtrace/server/__main__.py +136 -0
  61. stemtrace-0.1.0/src/stemtrace/server/api/__init__.py +29 -0
  62. stemtrace-0.1.0/src/stemtrace/server/api/routes.py +270 -0
  63. stemtrace-0.1.0/src/stemtrace/server/api/schemas.py +136 -0
  64. stemtrace-0.1.0/src/stemtrace/server/api/websocket.py +28 -0
  65. stemtrace-0.1.0/src/stemtrace/server/consumer.py +133 -0
  66. stemtrace-0.1.0/src/stemtrace/server/fastapi/__init__.py +13 -0
  67. stemtrace-0.1.0/src/stemtrace/server/fastapi/auth.py +77 -0
  68. stemtrace-0.1.0/src/stemtrace/server/fastapi/extension.py +121 -0
  69. stemtrace-0.1.0/src/stemtrace/server/fastapi/router.py +42 -0
  70. stemtrace-0.1.0/src/stemtrace/server/store.py +260 -0
  71. stemtrace-0.1.0/src/stemtrace/server/ui/__init__.py +5 -0
  72. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/biome.json +58 -0
  73. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/index.html +16 -0
  74. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/package-lock.json +4049 -0
  75. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/package.json +46 -0
  76. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/playwright.config.ts +40 -0
  77. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/postcss.config.js +5 -0
  78. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/src/api/client.ts +173 -0
  79. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/src/api/queries.ts +151 -0
  80. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/src/components/DateRangePicker.tsx +244 -0
  81. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/src/components/Filters.tsx +116 -0
  82. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/src/components/TaskGraph.tsx +453 -0
  83. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/src/components/TaskList.tsx +172 -0
  84. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/src/components/TaskStateBadge.tsx +27 -0
  85. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/src/components/TaskTimeline.tsx +144 -0
  86. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/src/components/TimelineBar.tsx +224 -0
  87. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/src/components/Tooltip.tsx +54 -0
  88. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/src/hooks/WebSocketContext.tsx +143 -0
  89. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/src/hooks/useInfiniteScroll.ts +56 -0
  90. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/src/index.css +104 -0
  91. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/src/main.tsx +44 -0
  92. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/src/routes/__root.tsx +103 -0
  93. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/src/routes/graph.$rootId.tsx +51 -0
  94. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/src/routes/graphs.tsx +186 -0
  95. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/src/routes/index.tsx +119 -0
  96. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/src/routes/registry.tsx +211 -0
  97. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/src/routes/tasks.$taskId.tsx +313 -0
  98. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/src/utils/format.ts +38 -0
  99. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/tests/graphs.spec.ts +485 -0
  100. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/tests/registry.spec.ts +117 -0
  101. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/tests/task-detail.spec.ts +147 -0
  102. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/tests/tasks.spec.ts +92 -0
  103. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/tsconfig.json +32 -0
  104. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/tsconfig.tsbuildinfo +1 -0
  105. stemtrace-0.1.0/src/stemtrace/server/ui/frontend/vite.config.ts +30 -0
  106. stemtrace-0.1.0/src/stemtrace/server/ui/static.py +56 -0
  107. stemtrace-0.1.0/src/stemtrace/server/websocket.py +99 -0
  108. stemtrace-0.1.0/tests/__init__.py +1 -0
  109. stemtrace-0.1.0/tests/conftest.py +18 -0
  110. stemtrace-0.1.0/tests/e2e/__init__.py +10 -0
  111. stemtrace-0.1.0/tests/e2e/conftest.py +208 -0
  112. stemtrace-0.1.0/tests/e2e/tasks.py +118 -0
  113. stemtrace-0.1.0/tests/e2e/test_api_e2e.py +332 -0
  114. stemtrace-0.1.0/tests/integration/__init__.py +1 -0
  115. stemtrace-0.1.0/tests/integration/test_fastapi_integration.py +580 -0
  116. stemtrace-0.1.0/tests/integration/test_redis_transport.py +211 -0
  117. stemtrace-0.1.0/tests/integration/test_static_ui.py +114 -0
  118. stemtrace-0.1.0/tests/unit/__init__.py +1 -0
  119. stemtrace-0.1.0/tests/unit/test_auth.py +120 -0
  120. stemtrace-0.1.0/tests/unit/test_bootsteps.py +179 -0
  121. stemtrace-0.1.0/tests/unit/test_config.py +98 -0
  122. stemtrace-0.1.0/tests/unit/test_consumer.py +167 -0
  123. stemtrace-0.1.0/tests/unit/test_events.py +244 -0
  124. stemtrace-0.1.0/tests/unit/test_examples.py +192 -0
  125. stemtrace-0.1.0/tests/unit/test_exceptions.py +32 -0
  126. stemtrace-0.1.0/tests/unit/test_graph.py +1168 -0
  127. stemtrace-0.1.0/tests/unit/test_init.py +154 -0
  128. stemtrace-0.1.0/tests/unit/test_routes.py +371 -0
  129. stemtrace-0.1.0/tests/unit/test_scrubbing.py +249 -0
  130. stemtrace-0.1.0/tests/unit/test_signals.py +692 -0
  131. stemtrace-0.1.0/tests/unit/test_store.py +824 -0
  132. stemtrace-0.1.0/tests/unit/test_transports.py +431 -0
  133. stemtrace-0.1.0/tests/unit/test_websocket.py +187 -0
  134. stemtrace-0.1.0/uv.lock +2150 -0
@@ -0,0 +1,33 @@
1
+ [tool.bumpversion]
2
+ current_version = "0.1.0"
3
+ parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
4
+ serialize = ["{major}.{minor}.{patch}"]
5
+ commit = true
6
+ tag = false # Tag created via GitHub Release UI
7
+ commit_message = "chore: release v{new_version}"
8
+
9
+ # Python package version in pyproject.toml
10
+ [[tool.bumpversion.files]]
11
+ filename = "pyproject.toml"
12
+ search = 'version = "{current_version}"'
13
+ replace = 'version = "{new_version}"'
14
+
15
+ # Python __version__ in package __init__.py
16
+ [[tool.bumpversion.files]]
17
+ filename = "src/stemtrace/__init__.py"
18
+ search = '__version__ = "{current_version}"'
19
+ replace = '__version__ = "{new_version}"'
20
+
21
+ # Frontend package.json version
22
+ [[tool.bumpversion.files]]
23
+ filename = "src/stemtrace/server/ui/frontend/package.json"
24
+ search = '"version": "{current_version}"'
25
+ replace = '"version": "{new_version}"'
26
+
27
+ # CHANGELOG.md - insert new version header
28
+ [[tool.bumpversion.files]]
29
+ filename = "CHANGELOG.md"
30
+ search = "## [Unreleased]"
31
+ replace = """## [Unreleased]
32
+
33
+ ## [{new_version}] - {now:%Y-%m-%d}"""
@@ -0,0 +1,141 @@
1
+ # Active Context: stemtrace
2
+
3
+ ## Current Focus
4
+
5
+ **Phase 7 Complete - Ready for v0.1.0 Release**
6
+
7
+ Canvas Graph Reconstruction (group_id capture, synthetic GROUP nodes) implemented and tested. All checks passing. Ready to tag and release.
8
+
9
+ ## Phase 7: Canvas Graph Reconstruction ✅
10
+
11
+ 1. [x] Add `group_id` field to TaskEvent model
12
+ 2. [x] Capture `group_id` from `task.request.group` in signal handlers
13
+ 3. [x] Capture `group_id` from message headers in bootsteps
14
+ 4. [x] Add `NodeType` enum (TASK/GROUP/CHORD) to core/graph.py
15
+ 5. [x] Add `node_type` and `group_id` fields to TaskNode model
16
+ 6. [x] Implement synthetic GROUP node generation in TaskGraph.add_event()
17
+ 7. [x] Add `node_type` and `group_id` to API response schemas
18
+ 8. [x] Render GROUP nodes with distinct styling in TaskGraph.tsx
19
+ 9. [x] Show start time and duration in graph node labels
20
+ 10. [x] Unit tests for group detection and synthetic node creation (41 graph tests)
21
+ 11. [x] Integration tests for synthetic group nodes (15 FastAPI tests)
22
+
23
+ ## Release Checklist
24
+
25
+ 1. [ ] Tag v0.1.0: `git tag v0.1.0 && git push origin v0.1.0`
26
+ 2. [ ] Configure PyPI trusted publishing
27
+ 3. [ ] Verify GitHub release automation
28
+
29
+ ## Strategic Direction
30
+
31
+ > **stemtrace models Celery as a graph of executions derived from events.**
32
+
33
+ See `progress.md` for full strategic principles.
34
+ See `strategic-analysis.md` for competitive positioning and roadmap priorities.
35
+
36
+ ## Roadmap Overview
37
+
38
+ | Phase | Focus | Status |
39
+ |-------|-------|--------|
40
+ | 6 | UX/DX Enhancements | ✅ Complete |
41
+ | Pre-Release | E2E Testing (Docker + Playwright) | ✅ Complete |
42
+ | 7 | Canvas Graph Reconstruction (group_id, synthetic nodes) | ✅ Complete |
43
+ | v0.1.0 | Release | **Current** |
44
+ | 8 | Execution & Worker Lifecycle (worker events, registry capture) | Planned |
45
+ | 8.5 | Anomaly Detection (unregistered, stuck, interrupted tasks) | Planned 🆕 |
46
+ | 9 | UI reorganization (Dashboard + Executions + Registry) | Planned |
47
+ | 10 | Export & Integration (OpenTelemetry, webhooks) | Planned |
48
+ | 11 | RabbitMQ + documentation polish | Planned |
49
+
50
+ See `progress.md` for detailed task lists.
51
+
52
+ ## Project State
53
+
54
+ | Metric | Value |
55
+ |--------|-------|
56
+ | Tests | All passing (350+ Python tests + Playwright E2E) |
57
+ | Coverage | 90%+ |
58
+ | Python | 3.10+ |
59
+ | Status | Ready for v0.1.0 release |
60
+
61
+ ## Competitive Position
62
+
63
+ vs. Kanchi (see `strategic-analysis.md` for full breakdown):
64
+
65
+ **Our Strengths:**
66
+ - **Zero infrastructure** — No PostgreSQL requirement, uses existing Redis
67
+ - **Flow-first** — DAG visualization is the killer feature Kanchi lacks
68
+ - **FastAPI embedding** — Mount in existing app
69
+ - **Read-only** — Safe for production
70
+ - **Canvas awareness** — Synthetic GROUP/CHORD nodes
71
+
72
+ **Gaps to Close:**
73
+ - Anomaly detection (Phase 8.5) — #1 priority for production adoption
74
+ - Worker lifecycle (Phase 8) — Prerequisite for anomaly detection
75
+ - Dashboard (Phase 9) — First-impression UX
76
+ - RabbitMQ (Phase 11) — Unlocks half the Celery user base
77
+
78
+ ## What's New in Phase 7
79
+
80
+ ### group_id Capture
81
+ Events now capture `group_id` from Celery task requests, enabling tracking of tasks spawned via `group()` and `chord()`.
82
+
83
+ ### Synthetic GROUP & CHORD Nodes
84
+ When 2+ tasks share the same `group_id`, a synthetic GROUP node is automatically created:
85
+ - GROUP nodes are roots in the graph
86
+ - Member tasks become children of the GROUP node
87
+ - GROUP state is computed from member states (all SUCCESS = SUCCESS, any FAILURE = FAILURE, etc.)
88
+ - When a group has a callback (chord), GROUP is upgraded to CHORD with callback linking
89
+
90
+ ### UI Enhancements
91
+ - GROUP nodes rendered with dashed indigo borders
92
+ - CHORD nodes rendered with dashed purple borders
93
+ - Task nodes show start time and duration directly in the graph
94
+ - Clear visual distinction between TASK, GROUP, and CHORD node types
95
+
96
+ ## Known Issues
97
+
98
+ ### Task Visibility Gaps
99
+
100
+ Three scenarios where tasks get "stuck" with no clear indication to users:
101
+
102
+ | Scenario | What User Sees | Root Cause |
103
+ |----------|----------------|------------|
104
+ | Unregistered task | PENDING forever | Worker can't find task, KeyError before signals |
105
+ | Worker crash | STARTED forever | No shutdown signal, no completion event |
106
+ | Worker graceful stop | STARTED (maybe) | Depends on acks_late config |
107
+
108
+ **Planned fix:** Phase 8 (worker lifecycle) + Phase 8.5 (anomaly detection)
109
+
110
+ ### Solution Architecture
111
+
112
+ ```
113
+ ┌─────────────────────────────────────────────────────────────┐
114
+ │ Phase 8: Worker Lifecycle Events │
115
+ │ - Capture worker_ready (hostname, pid, registered_tasks) │
116
+ │ - Capture worker_shutdown │
117
+ │ - Build global registry from all workers │
118
+ └────────────────────┬────────────────────────────────────────┘
119
+ │
120
+ ▼
121
+ ┌─────────────────────────────────────────────────────────────┐
122
+ │ Phase 8.5: Anomaly Detection │
123
+ │ - Unregistered: PENDING task not in any registry │
124
+ │ - Stuck: PENDING/STARTED > timeout │
125
+ │ - Interrupted: STARTED when worker SHUTDOWN │
126
+ └────────────────────┬────────────────────────────────────────┘
127
+ │
128
+ ▼
129
+ ┌─────────────────────────────────────────────────────────────┐
130
+ │ UI: Visibility │
131
+ │ - Warning badges on anomalous tasks │
132
+ │ - Worker session timeline markers │
133
+ │ - "2 new tasks registered" on worker restart │
134
+ └─────────────────────────────────────────────────────────────┘
135
+ ```
136
+
137
+ See `progress.md` → Phase 8 and 8.5 for detailed task lists.
138
+
139
+ ## Known Blockers
140
+
141
+ None.
@@ -0,0 +1,108 @@
1
+ # Product Context: stemtrace
2
+
3
+ ## Why This Project Exists
4
+
5
+ Celery is the de-facto standard for Python task queues, but observability into task flows is surprisingly poor. When a complex workflow fails, developers spend hours tracing what happened:
6
+
7
+ - Which task spawned this one?
8
+ - What tasks ran in parallel?
9
+ - Where did the chain break?
10
+ - Which retry actually succeeded?
11
+
12
+ **stemtrace exists to answer these questions instantly.**
13
+
14
+ ## Problems It Solves
15
+
16
+ ### 1. "Why is this task slow/stuck?"
17
+ Show the timeline: when it was queued, when picked up, how long in each state.
18
+
19
+ ### 2. "What spawned this task?"
20
+ Display parent-child relationships, trace back to the originating request.
21
+
22
+ ### 3. "What tasks fan-out / fan-in?"
23
+ Visualize groups and chords as a DAG, not just a flat list.
24
+
25
+ ### 4. "Which retries actually happened?"
26
+ Show retry count, timing, and failure reasons for each attempt.
27
+
28
+ ### 5. "What happened for this one request end-to-end?"
29
+ Correlate all tasks triggered by a single API request via trace_id.
30
+
31
+ ## How It Should Work
32
+
33
+ ### For the Developer (Library User)
34
+
35
+ ```python
36
+ from stemtrace import init
37
+ init(app) # Done. That's it.
38
+ ```
39
+
40
+ - Zero configuration required
41
+ - No code changes to existing tasks
42
+ - No performance impact (fire-and-forget)
43
+
44
+ ### For the FastAPI User (Pluggable Integration)
45
+
46
+ ```python
47
+ from fastapi import FastAPI
48
+ from stemtrace.server import StemtraceExtension
49
+
50
+ flow = StemtraceExtension(broker_url="redis://localhost:6379/0")
51
+ app = FastAPI(lifespan=flow.lifespan)
52
+ app.include_router(flow.router, prefix="/stemtrace")
53
+ ```
54
+
55
+ - Mount directly into existing FastAPI app
56
+ - No separate server to deploy
57
+ - Leverage existing auth middleware
58
+ - Optional embedded consumer (background task)
59
+
60
+ ### For the Observer (UI User)
61
+
62
+ 1. Open the web UI (standalone or mounted at `/stemtrace`)
63
+ 2. See a list of recent task executions
64
+ 3. Click on any task to see:
65
+ - Full execution timeline
66
+ - Parent/child relationships
67
+ - Retry history with full exception details
68
+ - Task arguments and return values
69
+ - Sensitive data automatically scrubbed (passwords, API keys, etc.)
70
+ 4. View flow graphs for complex workflows
71
+ 5. Browse the Task Registry to see all discovered task definitions
72
+
73
+ ## User Experience Goals
74
+
75
+ ### Simplicity
76
+ - Install in < 5 minutes
77
+ - No infrastructure changes (reuse existing broker)
78
+ - Sensible defaults, optional configuration
79
+ - **Plug into existing FastAPI** — no new containers
80
+
81
+ ### Clarity
82
+ - Clear visual hierarchy
83
+ - Obvious state indicators (pending, running, success, failure)
84
+ - No information overload
85
+ - **React-based UI** — responsive, interactive graphs
86
+
87
+ ### Safety
88
+ - Read-only by design
89
+ - Never interferes with task execution
90
+ - Can be deployed/removed without affecting production
91
+ - **Flexible auth** — use your existing FastAPI auth or built-in defaults
92
+
93
+ ### Flexibility
94
+ - **Standalone server** — run separately via CLI
95
+ - **FastAPI plugin** — mount as a router in your app
96
+ - **Hybrid consumer** — embedded background task or separate process
97
+
98
+ ## Positioning
99
+
100
+ | Tool | Focus | stemtrace's Relationship |
101
+ |------|-------|---------------------------|
102
+ | Flower | Worker status, task list | Complements (we show flows) |
103
+ | Prometheus | Metrics, alerting | Complements (we show executions) |
104
+ | Sentry | Error tracking | Complements (we show context) |
105
+ | Jaeger/Zipkin | Distributed tracing | Alternative (simpler, Celery-specific) |
106
+
107
+ **We don't replace monitoring. We add understanding.**
108
+
@@ -0,0 +1,383 @@
1
+ # Progress: stemtrace
2
+
3
+ ## Current Status: Ready for v0.1.0 Release
4
+
5
+ **Phase:** Phase 7 Complete (Canvas Graph Reconstruction)
6
+ **Sprint:** v0.1.0 Release
7
+ **Last Updated:** 2025-12-29
8
+
9
+ ---
10
+
11
+ ## What Works
12
+
13
+ - [x] Project concept defined
14
+ - [x] Architecture decisions made
15
+ - [x] Project name confirmed (stemtrace, PyPI available)
16
+ - [x] README.md created
17
+ - [x] Project context files created (.context/)
18
+ - [x] Cursor rules created (.cursor/rules/)
19
+ - [x] pyproject.toml with all dependencies, mypy, ruff, pytest config
20
+ - [x] **Broker-agnostic design** - Redis, RabbitMQ, extensible registry
21
+ - [x] **FastAPI pluggability design** - Mount as router in user's FastAPI app
22
+ - [x] **UI technology decided** - React SPA, pre-built with Vite
23
+ - [x] **Library component complete** - init(), signals, transports
24
+ - [x] **Server component complete** - GraphStore, EventConsumer, REST API, WebSocket, React UI
25
+
26
+ ## What's Left to Build
27
+
28
+ ### Phase 1: Project Setup ✅
29
+ - [x] pyproject.toml with all dependencies
30
+ - [x] Source directory structure (src/stemtrace/)
31
+ - [x] py.typed marker
32
+ - [x] pre-commit configuration
33
+ - [x] GitHub Actions CI
34
+ - [x] LICENSE file
35
+ - [x] CONTRIBUTING.md
36
+
37
+ ### Phase 2: Core Domain ✅
38
+ - [x] TaskEvent dataclass *(frozen, with full fields)*
39
+ - [x] TaskState enum *(all states, str inheritance)*
40
+ - [x] TaskNode and TaskGraph models *(full implementation)*
41
+ - [x] Port interfaces (Protocols) *(EventTransport, TaskRepository)*
42
+ - [x] Unit tests for edge cases *(45 tests, 96% coverage)*
43
+ - [x] Documentation strings review *(Google style, examples included)*
44
+
45
+ ### Phase 3: Library Component ✅
46
+ - [x] Celery signal handlers *(task_prerun, task_postrun, task_failure, task_retry, task_revoked)*
47
+ - [x] Redis publisher *(RedisTransport with Redis Streams)*
48
+ - [x] Memory transport *(MemoryTransport for testing)*
49
+ - [x] Transport registry *(get_transport() factory with scheme detection)*
50
+ - [x] init() public API *(wires config, transport, signals)*
51
+ - [x] Configuration handling *(StemtraceConfig frozen model)*
52
+
53
+ ### Phase 4: Server Component ✅
54
+ - [x] Broker-agnostic event consumer *(EventConsumer, AsyncEventConsumer)*
55
+ - [x] In-memory graph store *(GraphStore with thread-safe access)*
56
+ - [x] FastAPI REST API *(tasks, graphs, health endpoints)*
57
+ - [x] FastAPI pluggable router *(`create_router()` factory)*
58
+ - [x] `StemtraceExtension` with lifespan/embedded consumer
59
+ - [x] Auth helpers *(require_basic_auth, require_api_key, no_auth)*
60
+ - [x] WebSocket support *(WebSocketManager, real-time event broadcasting)*
61
+ - [x] React SPA UI *(TanStack Router + Query, react-flow for graphs)*
62
+ - [x] Static file serving from bundled assets
63
+ - [x] Build pipeline *(hatchling hook for npm build)*
64
+
65
+ ### Phase 5: Polish ✅
66
+
67
+ #### Testing
68
+ - [x] Write unit tests for server components (GraphStore, Consumer, WebSocket, Auth, Routes)
69
+ - [x] Integration tests for low-coverage modules (StemtraceExtension, create_router, WebSocket endpoint)
70
+ - [x] Integration tests with real Redis transport (7 tests)
71
+
72
+ #### CLI Implementation
73
+ - [x] Implement `stemtrace server` command *(uvicorn + StemtraceExtension)*
74
+ - [x] Implement `stemtrace consume` command *(standalone consumer with signal handling)*
75
+
76
+ #### Infrastructure
77
+ - [x] Docker setup (Dockerfile, docker-compose.yml, .dockerignore)
78
+
79
+ #### Release
80
+ - [x] PyPI release preparation (build verification, twine check, version bump)
81
+ - [x] CI pipeline enhancements (concurrency, caching, frontend checks, build verification)
82
+ - [x] Release automation (release.yml for PyPI + Docker + GitHub releases)
83
+ - [x] Dependabot for automated dependency updates
84
+ - [x] CI and coverage badges in README
85
+
86
+ ### Phase 6: UX/DX Enhancements ✅
87
+
88
+ #### Task Data Capture
89
+ - [x] Capture PENDING state via task_sent signal (fires on .delay())
90
+ - [x] Capture RECEIVED state via worker bootstep (fires when worker picks up task)
91
+ - [x] Capture task args/kwargs in PENDING and STARTED events
92
+ - [x] Capture task result in SUCCESS event
93
+ - [x] Capture exception + traceback in FAILURE/RETRY events
94
+ - [x] Robust event ordering (retry_count, state_priority, timestamp)
95
+ - [x] Frontend deduplication of PENDING/RECEIVED from retry re-queues
96
+
97
+ #### Sensitive Data Scrubbing
98
+ - [x] `library/scrubbing.py` with Sentry-style key matching
99
+ - [x] Default sensitive keys (password, api_key, token, credit_card, etc.)
100
+ - [x] Configurable `additional_sensitive_keys` and `safe_keys`
101
+
102
+ #### Task Registry
103
+ - [x] `/api/tasks/registry` endpoint
104
+ - [x] Registry UI page with search
105
+
106
+ #### UI Enhancements
107
+ - [x] Parameters section in task detail view
108
+ - [x] Result section in task detail view
109
+ - [x] Expandable error details (exception + traceback)
110
+
111
+ #### Examples
112
+ - [x] `fetch_api_data` — retry demo with real exceptions
113
+ - [x] `process_user_data` — sensitive data scrubbing demo
114
+ - [x] `always_fails` — failure with traceback demo
115
+ - [x] CLI demo runner (`python examples/celery_app.py <demo>`)
116
+
117
+ ### Pre-Release: E2E Testing ✅
118
+
119
+ Complete E2E test coverage before v0.1.0 release.
120
+
121
+ #### Docker-based API E2E
122
+ - [x] Docker Compose test environment (Redis + Worker + Server)
123
+ - [x] E2E test: task execution → event capture → API visibility
124
+ - [x] E2E test: workflow (chain/group) → graph construction
125
+ - [x] E2E test: WebSocket real-time updates
126
+
127
+ #### Playwright Browser E2E
128
+ - [x] Playwright setup and configuration
129
+ - [x] E2E test: Tasks page navigation and filtering
130
+ - [x] E2E test: Task detail view (timeline, parameters, results)
131
+ - [x] E2E test: Graph visualization renders correctly
132
+ - [x] E2E test: Registry page search
133
+
134
+ #### CI Integration
135
+ - [x] E2E tests in GitHub Actions (separate workflow)
136
+ - [x] Playwright report artifacts
137
+
138
+ ### Phase 7: Canvas Graph Reconstruction ✅
139
+
140
+ **Implemented:** group_id capture and synthetic GROUP node generation.
141
+
142
+ #### What's Implemented
143
+ - [x] Add `group_id` field to `TaskEvent` model
144
+ - [x] Capture `group_id` from `task.request.group` in signal handlers
145
+ - [x] Capture `group_id` from message headers in bootstep
146
+ - [x] `NodeType` enum (TASK/GROUP/CHORD)
147
+ - [x] Detect group: multiple tasks with same `group_id`
148
+ - [x] Create synthetic GROUP node when 2+ tasks share group_id
149
+ - [x] Upgrade GROUP to CHORD when callback detected
150
+ - [x] Link CHORD node to callback task
151
+ - [x] `/api/graphs/{id}` returns synthetic nodes with `node_type`
152
+ - [x] Render GROUP/CHORD nodes with distinct styling
153
+ - [x] Show start time + duration in graph node labels
154
+ - [x] Unit tests (41 graph tests, 23 signal tests)
155
+ - [x] Integration tests (15 FastAPI tests including synthetic group nodes)
156
+
157
+ #### What's Deferred
158
+ - [ ] Nested canvas detection (chain inside group)
159
+ - [ ] RabbitMQ group_id capture
160
+
161
+ #### What Works Now
162
+ | Celery Pattern | Status |
163
+ |---------------|--------|
164
+ | `chain(a, b, c)` | ✅ a→b→c edges via parent_id |
165
+ | `group(a, b, c)` | ✅ Grouped via synthetic GROUP node |
166
+ | `chord(group, callback)` | ✅ CHORD node with callback linking |
167
+ | Dynamic `task.delay()` inside task | ✅ Works (parent_id) |
168
+
169
+ ### Phase 8: Execution & Worker Lifecycle Enrichment ⚡ HIGH PRIORITY
170
+
171
+ **Strategic importance:** Prerequisite for anomaly detection (Phase 8.5). Without worker lifecycle events, we can't detect orphaned tasks or worker crashes. See `strategic-analysis.md`.
172
+
173
+ Enrich events with worker/queue context, capture worker lifecycle, and detect anomalies.
174
+
175
+ #### Task Enrichment
176
+ - [ ] Add `worker` field to TaskEvent (hostname of executing worker)
177
+ - [ ] Add `queue` field to TaskEvent (routing key / queue name)
178
+ - [ ] Update signal handlers to capture worker/queue from task request
179
+
180
+ #### Worker Lifecycle Events 🆕
181
+
182
+ Celery provides these signals for worker lifecycle:
183
+ - `worker_ready` — Worker online (hostname, pid, registered tasks)
184
+ - `worker_shutdown` — Worker going offline (graceful)
185
+ - `worker_process_init` / `worker_process_shutdown` — Pool process lifecycle
186
+
187
+ Implementation:
188
+ - [ ] Add `WorkerEvent` model (worker_id, hostname, pid, state, timestamp, registered_tasks)
189
+ - [ ] Hook `worker_ready` signal → emit WORKER_READY event
190
+ - [ ] Hook `worker_shutdown` signal → emit WORKER_SHUTDOWN event
191
+ - [ ] Store worker sessions in GraphStore (worker_id → WorkerSession)
192
+ - [ ] Track registered tasks per worker for registry cross-check
193
+
194
+ #### Interrupted Task Detection 🆕
195
+
196
+ When worker shuts down, correlate with in-flight tasks:
197
+ - [ ] On WORKER_SHUTDOWN, find tasks in STARTED state for that worker
198
+ - [ ] Mark as INTERRUPTED (virtual state) or add `interrupted: true` flag
199
+ - [ ] Handle `acks_late` case: duplicate STARTED events after restart
200
+
201
+ #### Monitoring APIs
202
+ - [ ] `GET /api/workers` — Active workers with current task counts
203
+ - [ ] `GET /api/workers/{id}/history` — Worker sessions with start/stop times
204
+ - [ ] `GET /api/stats` — Duration percentiles, failure rates by task name
205
+ - [ ] `GET /api/orphans` — Tasks stuck in STARTED > threshold OR interrupted
206
+
207
+ ### Phase 8.5: Anomaly Detection & Visibility 🆕 ⚡ CRITICAL
208
+
209
+ **Strategic importance:** #1 blocker for production adoption. Users need to know when things break. See `strategic-analysis.md`.
210
+
211
+ Detect task anomalies and clearly communicate problems to users. Builds on Phase 8's worker lifecycle capture.
212
+
213
+ #### Unregistered Task Detection
214
+
215
+ **Problem:** Tasks submitted to unregistered names show as PENDING forever. Workers silently fail with `KeyError` before any stemtrace signals fire.
216
+
217
+ Detection (depends on Phase 8 WorkerEvent with registered_tasks):
218
+ - [ ] Build global registry from all active workers' registered_tasks
219
+ - [ ] On new PENDING event, check if task name exists in global registry
220
+ - [ ] Add `registered: bool` field to task/node responses
221
+ - [ ] Fallback: timeout-based detection for PENDING > N minutes
222
+
223
+ #### Stuck/Orphan Task Detection
224
+
225
+ | Scenario | Detection |
226
+ |----------|-----------|
227
+ | PENDING + unregistered | No worker has task in registry |
228
+ | PENDING + timeout | PENDING > 5 min (configurable) |
229
+ | STARTED + worker crash | Worker SHUTDOWN event with no task completion |
230
+ | STARTED + timeout | STARTED > 30 min (configurable) |
231
+
232
+ - [ ] Compute `anomaly` field on tasks: `null | 'unregistered' | 'stuck' | 'interrupted'`
233
+ - [ ] `GET /api/tasks/anomalies` — All problematic tasks
234
+
235
+ #### UI Indicators
236
+ - [ ] Warning badge on tasks with anomalies
237
+ - [ ] Tooltip explaining the issue ("Task not in any worker registry", etc.)
238
+ - [ ] Filter: "Problematic tasks" in task list
239
+ - [ ] Dashboard section: Anomaly summary
240
+
241
+ #### Worker Session Timeline 🆕
242
+
243
+ Visualize task executions segmented by worker sessions:
244
+ - [ ] In task list: Insert "Worker @host started" / "Worker @host stopped" markers
245
+ - [ ] In graphs page: Show which worker session each graph belongs to
246
+ - [ ] On worker restart: Show "2 new tasks registered" diff indicator
247
+
248
+ ### Phase 9: UI Reorganization
249
+
250
+ Restructure UI to Dashboard-first architecture.
251
+
252
+ #### Navigation
253
+ - [ ] New navigation: Dashboard | Executions | Registry
254
+ - [ ] Merge Tasks + Graphs into unified Executions page
255
+ - [ ] Enhance Registry with runtime stats (runs, duration, failure rate)
256
+
257
+ #### Dashboard Page (`/`)
258
+ - [ ] Stats cards: Total tasks (24h), success rate, avg duration
259
+ - [ ] Recent failures: Last 5 failed tasks
260
+ - [ ] Orphan alerts: Stale STARTED tasks
261
+ - [ ] Live activity: Mini timeline of recent events
262
+
263
+ #### Executions Page (`/executions`)
264
+ - [ ] Keep List/Timeline toggle
265
+ - [ ] Add "Flows only" filter (root tasks)
266
+ - [ ] Click task → detail with inline flow graph
267
+
268
+ #### Execution Detail (`/executions/$taskId`)
269
+ - [ ] Add flow graph section showing execution in context
270
+ - [ ] Highlight current task in the flow graph
271
+
272
+ #### Registry Page (`/registry`)
273
+ - [ ] Show runs (24h), last run, avg duration, failure rate per task
274
+ - [ ] Click → filtered Executions view
275
+
276
+ ### Phase 10: Export and Integration ⚡ HIGH PRIORITY
277
+
278
+ **Strategic importance:** Enables long-term retention without adding database complexity. OTEL export is the answer for users needing 30-day analytics. See `strategic-analysis.md`.
279
+
280
+ Enable long-term analytics via external systems. stemtrace stays lean; users bring their own storage.
281
+
282
+ #### OpenTelemetry Export
283
+ - [ ] Optional dependency: `opentelemetry-sdk`
284
+ - [ ] Config: `otel_endpoint` in init() or env var
285
+ - [ ] Map ExecutionGraph → OTEL Trace with spans
286
+ - [ ] Push to Jaeger, Tempo, Honeycomb, Datadog
287
+
288
+ #### Webhook Export
289
+ - [ ] `init(app, webhook_url="...")` for fire-and-forget event push
290
+ - [ ] Non-blocking POST to user endpoint
291
+
292
+ #### JSON Export API
293
+ - [ ] `GET /api/export/executions?from=...&to=...`
294
+ - [ ] Download execution data as JSON
295
+
296
+ ### Phase 11: Ecosystem Polish
297
+
298
+ #### RabbitMQ Transport
299
+ - [ ] `RabbitMQTransport` using fanout exchange
300
+ - [ ] Auto-detect from `amqp://` URL scheme
301
+
302
+ #### Documentation
303
+ - [ ] Update README roadmap
304
+ - [ ] Add integration examples (Jaeger, Grafana)
305
+ - [ ] Architecture diagrams
306
+
307
+ ### Not Planned (Strategic Decisions)
308
+
309
+ See `strategic-analysis.md` for full competitive rationale.
310
+
311
+ | Feature | Reason |
312
+ |---------|--------|
313
+ | PostgreSQL persistence | Zero-infra advantage — use OTEL export instead |
314
+ | SQLite persistence | Low priority; ship OTEL first, reassess if demand emerges |
315
+ | Task control (retry/cancel) | Stay read-only for production safety |
316
+ | Ops alerting (PagerDuty, etc.) | Users have existing monitoring stacks |
317
+ | Workflow automation | Not an ops platform, stay focused on visualization |
318
+
319
+ ---
320
+
321
+ ## Current Sprint Tasks
322
+
323
+ **v0.1.0 Release**
324
+
325
+ Phase 7 (Canvas Graph Reconstruction) complete. Ready to release:
326
+ 1. Tag v0.1.0: `git tag v0.1.0 && git push origin v0.1.0`
327
+ 2. Configure PyPI trusted publishing
328
+ 3. GitHub release will trigger PyPI + Docker publish
329
+
330
+ ---
331
+
332
+ ## Known Issues
333
+
334
+ - **Coverage at 90%+** — All checks passing
335
+ - **All tests passing** — 350+ Python tests (unit + integration), plus Playwright E2E tests
336
+
337
+ ### Unregistered Tasks Appear Stuck in PENDING
338
+
339
+ **Priority: High (UX issue)**
340
+
341
+ When a task is submitted but the worker doesn't have it registered:
342
+ 1. `task_sent` fires → PENDING event captured ✓
343
+ 2. Worker receives message → `KeyError: 'task.name'` in Celery internals
344
+ 3. Task stays PENDING forever in stemtrace (never progresses)
345
+
346
+ **Root cause:** Celery's task routing fails before any worker signals fire. The error:
347
+ ```
348
+ celery/worker/consumer/consumer.py, line 670, in on_task_received
349
+ strategy = strategies[type_]
350
+ KeyError: 'examples.celery_app.parallel_chord'
351
+ ```
352
+
353
+ **Planned fix:** See Phase 8.5 below — Detect unregistered/orphan tasks.
354
+
355
+ ### Remaining Low Coverage (edge cases only)
356
+
357
+ | Module | Coverage | Missing |
358
+ |--------|----------|---------|
359
+ | `consumer.py` | 84% | Error handling paths |
360
+ | `extension.py` | 83% | UI serving when dist exists |
361
+ | `static.py` | 90% | 404 fallback path |
362
+
363
+ ---
364
+
365
+ ## Strategic Principles
366
+
367
+ - **Lead with flow** — Model Celery as a graph of executions, not just task statuses
368
+ - **Match monitoring** — Orphans, workers, stats emerge naturally from flow data
369
+ - **Stay read-only** — No task control, safe for production
370
+ - **Zero new infrastructure** — Redis only, no PostgreSQL requirement
371
+ - **Export for long-term** — Integrate with existing observability stacks (OTEL, webhooks)
372
+
373
+ ## Notes
374
+
375
+ - Clean architecture: `core/` → `library/` / `server/` (no cross-imports)
376
+ - Broker-agnostic: Redis Streams (implemented), RabbitMQ (planned)
377
+ - FastAPI: `StemtraceExtension` for full integration
378
+
379
+ ## References
380
+
381
+ - **Frontend build pipeline**: See `tech-context.md`
382
+ - **Design decisions**: See `system-patterns.md`
383
+ - **Competitive positioning**: See `strategic-analysis.md`