glasscube 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. glasscube-0.1.0/LICENSE +21 -0
  2. glasscube-0.1.0/PKG-INFO +203 -0
  3. glasscube-0.1.0/README.md +153 -0
  4. glasscube-0.1.0/glasscube/__init__.py +7 -0
  5. glasscube-0.1.0/glasscube/__main__.py +3 -0
  6. glasscube-0.1.0/glasscube/app/__init__.py +4 -0
  7. glasscube-0.1.0/glasscube/app/frontend_dev.py +68 -0
  8. glasscube-0.1.0/glasscube/app/main.py +120 -0
  9. glasscube-0.1.0/glasscube/app/pages.py +185 -0
  10. glasscube-0.1.0/glasscube/app/settings.py +93 -0
  11. glasscube-0.1.0/glasscube/assistant/__init__.py +3 -0
  12. glasscube-0.1.0/glasscube/assistant/assistant.py +370 -0
  13. glasscube-0.1.0/glasscube/assistant/llm/__init__.py +5 -0
  14. glasscube-0.1.0/glasscube/assistant/llm/backends/__init__.py +4 -0
  15. glasscube-0.1.0/glasscube/assistant/llm/backends/anthropic.py +45 -0
  16. glasscube-0.1.0/glasscube/assistant/llm/backends/openai_compat.py +27 -0
  17. glasscube-0.1.0/glasscube/assistant/llm/base.py +21 -0
  18. glasscube-0.1.0/glasscube/assistant/llm/client.py +20 -0
  19. glasscube-0.1.0/glasscube/assistant/rag/__init__.py +11 -0
  20. glasscube-0.1.0/glasscube/assistant/rag/chunker.py +68 -0
  21. glasscube-0.1.0/glasscube/assistant/rag/embedder.py +37 -0
  22. glasscube-0.1.0/glasscube/assistant/rag/retriever.py +28 -0
  23. glasscube-0.1.0/glasscube/assistant/rag/seed_corpus/about_ieee_insat.txt +5 -0
  24. glasscube-0.1.0/glasscube/assistant/rag/seed_corpus/ieee_insat_events.md +5 -0
  25. glasscube-0.1.0/glasscube/assistant/rag/seed_corpus/smart_mirror_project.txt +4 -0
  26. glasscube-0.1.0/glasscube/assistant/rag/store.py +107 -0
  27. glasscube-0.1.0/glasscube/assistant/stt/__init__.py +5 -0
  28. glasscube-0.1.0/glasscube/assistant/stt/vad.py +148 -0
  29. glasscube-0.1.0/glasscube/assistant/stt/wake_word.py +186 -0
  30. glasscube-0.1.0/glasscube/assistant/stt/whisper.py +97 -0
  31. glasscube-0.1.0/glasscube/assistant/tools/__init__.py +3 -0
  32. glasscube-0.1.0/glasscube/assistant/tools/manager.py +25 -0
  33. glasscube-0.1.0/glasscube/assistant/tts/__init__.py +3 -0
  34. glasscube-0.1.0/glasscube/assistant/tts/piper.py +63 -0
  35. glasscube-0.1.0/glasscube/config/__init__.py +13 -0
  36. glasscube-0.1.0/glasscube/config/env.py +16 -0
  37. glasscube-0.1.0/glasscube/config/json_config/default_gesture_config.json +8 -0
  38. glasscube-0.1.0/glasscube/config/json_config/default_widget_event_bindings.json +58 -0
  39. glasscube-0.1.0/glasscube/config/loader.py +21 -0
  40. glasscube-0.1.0/glasscube/config/paths.py +32 -0
  41. glasscube-0.1.0/glasscube/content/__init__.py +23 -0
  42. glasscube-0.1.0/glasscube/content/cache.py +97 -0
  43. glasscube-0.1.0/glasscube/content/config.py +68 -0
  44. glasscube-0.1.0/glasscube/content/indexer.py +54 -0
  45. glasscube-0.1.0/glasscube/content/routing.py +180 -0
  46. glasscube-0.1.0/glasscube/content/sync.py +420 -0
  47. glasscube-0.1.0/glasscube/content/templates.py +122 -0
  48. glasscube-0.1.0/glasscube/core/__init__.py +3 -0
  49. glasscube-0.1.0/glasscube/core/actions.py +48 -0
  50. glasscube-0.1.0/glasscube/core/callables.py +23 -0
  51. glasscube-0.1.0/glasscube/core/navigation.py +66 -0
  52. glasscube-0.1.0/glasscube/core/state_manager.py +82 -0
  53. glasscube-0.1.0/glasscube/core/widget_events.py +120 -0
  54. glasscube-0.1.0/glasscube/hal/__init__.py +3 -0
  55. glasscube-0.1.0/glasscube/hal/hal.py +131 -0
  56. glasscube-0.1.0/glasscube/server/__init__.py +3 -0
  57. glasscube-0.1.0/glasscube/server/app.py +241 -0
  58. glasscube-0.1.0/glasscube/server/camera.py +147 -0
  59. glasscube-0.1.0/glasscube/server/camera_preview.py +54 -0
  60. glasscube-0.1.0/glasscube/server/hand_pointer.py +42 -0
  61. glasscube-0.1.0/glasscube/server/static_ui.py +47 -0
  62. glasscube-0.1.0/glasscube/server/websocket.py +54 -0
  63. glasscube-0.1.0/glasscube/server/widget_data.py +33 -0
  64. glasscube-0.1.0/glasscube/ui/assets/5 1-CpBR8EWH.png +0 -0
  65. glasscube-0.1.0/glasscube/ui/assets/index-BB9bg-We.js +9 -0
  66. glasscube-0.1.0/glasscube/ui/assets/index-CQRKDs3v.css +1 -0
  67. glasscube-0.1.0/glasscube/ui/assets/press-start-2p-cyrillic-400-normal-DBn7fddJ.woff2 +0 -0
  68. glasscube-0.1.0/glasscube/ui/assets/press-start-2p-cyrillic-ext-400-normal-CIYZrvmP.woff2 +0 -0
  69. glasscube-0.1.0/glasscube/ui/assets/press-start-2p-greek-400-normal-DQfalijU.woff2 +0 -0
  70. glasscube-0.1.0/glasscube/ui/assets/press-start-2p-latin-400-normal-KLytu4cr.woff +0 -0
  71. glasscube-0.1.0/glasscube/ui/assets/press-start-2p-latin-400-normal-_wFEWmAB.woff2 +0 -0
  72. glasscube-0.1.0/glasscube/ui/assets/press-start-2p-latin-ext-400-normal-DBgyq7ky.woff +0 -0
  73. glasscube-0.1.0/glasscube/ui/assets/press-start-2p-latin-ext-400-normal-DpfYvR5A.woff2 +0 -0
  74. glasscube-0.1.0/glasscube/ui/assets/sleeping-abdo-J4WKKI3q.gif +0 -0
  75. glasscube-0.1.0/glasscube/ui/assets/transition-abdo-DsBJJ6fU.gif +0 -0
  76. glasscube-0.1.0/glasscube/ui/assets/vt323-latin-400-normal-Dqn9ZSPj.woff +0 -0
  77. glasscube-0.1.0/glasscube/ui/assets/vt323-latin-400-normal-wDgJuOC9.woff2 +0 -0
  78. glasscube-0.1.0/glasscube/ui/assets/vt323-latin-ext-400-normal-DqOq58ri.woff +0 -0
  79. glasscube-0.1.0/glasscube/ui/assets/vt323-latin-ext-400-normal-iUdYUaX4.woff2 +0 -0
  80. glasscube-0.1.0/glasscube/ui/assets/vt323-vietnamese-400-normal-D0VnvwZ9.woff2 +0 -0
  81. glasscube-0.1.0/glasscube/ui/assets/vt323-vietnamese-400-normal-D_5Q7Vcv.woff +0 -0
  82. glasscube-0.1.0/glasscube/ui/index.html +13 -0
  83. glasscube-0.1.0/glasscube/vision/__init__.py +43 -0
  84. glasscube-0.1.0/glasscube/vision/debug_overlay.py +141 -0
  85. glasscube-0.1.0/glasscube/vision/gesture_detector.py +178 -0
  86. glasscube-0.1.0/glasscube/vision/gesture_types.py +54 -0
  87. glasscube-0.1.0/glasscube/vision/gestures/__init__.py +22 -0
  88. glasscube-0.1.0/glasscube/vision/gestures/base.py +198 -0
  89. glasscube-0.1.0/glasscube/vision/gestures/click.py +38 -0
  90. glasscube-0.1.0/glasscube/vision/gestures/default_set.py +21 -0
  91. glasscube-0.1.0/glasscube/vision/gestures/pinch.py +44 -0
  92. glasscube-0.1.0/glasscube/vision/gestures/scroll.py +37 -0
  93. glasscube-0.1.0/glasscube/vision/gestures/swipe.py +23 -0
  94. glasscube-0.1.0/glasscube/vision/gestures/two_fingers_swipe.py +39 -0
  95. glasscube-0.1.0/glasscube/vision/gestures/zoom.py +52 -0
  96. glasscube-0.1.0/glasscube/vision/hand_data.py +43 -0
  97. glasscube-0.1.0/glasscube/vision/landmark_detector.py +104 -0
  98. glasscube-0.1.0/glasscube/widgets/__init__.py +21 -0
  99. glasscube-0.1.0/glasscube/widgets/base.py +75 -0
  100. glasscube-0.1.0/glasscube/widgets/calendar.py +13 -0
  101. glasscube-0.1.0/glasscube/widgets/drawing.py +57 -0
  102. glasscube-0.1.0/glasscube/widgets/events.py +16 -0
  103. glasscube-0.1.0/glasscube/widgets/games.py +15 -0
  104. glasscube-0.1.0/glasscube/widgets/news.py +7 -0
  105. glasscube-0.1.0/glasscube/widgets/providers/__init__.py +12 -0
  106. glasscube-0.1.0/glasscube/widgets/providers/mongo_calendar.py +44 -0
  107. glasscube-0.1.0/glasscube/widgets/providers/open_meteo.py +40 -0
  108. glasscube-0.1.0/glasscube/widgets/providers/qr_catalog.py +6 -0
  109. glasscube-0.1.0/glasscube/widgets/qr_gallery.py +7 -0
  110. glasscube-0.1.0/glasscube/widgets/text.py +85 -0
  111. glasscube-0.1.0/glasscube/widgets/weather.py +11 -0
  112. glasscube-0.1.0/glasscube.egg-info/PKG-INFO +203 -0
  113. glasscube-0.1.0/glasscube.egg-info/SOURCES.txt +118 -0
  114. glasscube-0.1.0/glasscube.egg-info/dependency_links.txt +1 -0
  115. glasscube-0.1.0/glasscube.egg-info/entry_points.txt +2 -0
  116. glasscube-0.1.0/glasscube.egg-info/requires.txt +27 -0
  117. glasscube-0.1.0/glasscube.egg-info/top_level.txt +1 -0
  118. glasscube-0.1.0/pyproject.toml +102 -0
  119. glasscube-0.1.0/setup.cfg +4 -0
  120. glasscube-0.1.0/tests/test_legacy_env.py +13 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Smart Glass Cube
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,203 @@
1
+ Metadata-Version: 2.4
2
+ Name: glasscube
3
+ Version: 0.1.0
4
+ Summary: Build gesture-controlled smart mirrors: webcam hand gestures drive pages and widgets of a React UI, plus a voice assistant
5
+ Author-email: Moetez Fradi <moetezfradi7@gmail.com>
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/SmartGlassCube/Smart-Mirror-SDK
8
+ Project-URL: Documentation, https://github.com/SmartGlassCube/Smart-Mirror-SDK/tree/main/docs
9
+ Project-URL: Repository, https://github.com/SmartGlassCube/Smart-Mirror-SDK
10
+ Project-URL: Issues, https://github.com/SmartGlassCube/Smart-Mirror-SDK/issues
11
+ Keywords: smart mirror,gesture recognition,hand tracking,mediapipe,computer vision,voice assistant,websocket
12
+ Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Intended Audience :: Education
15
+ Classifier: Operating System :: OS Independent
16
+ Classifier: Programming Language :: Python :: 3
17
+ Classifier: Programming Language :: Python :: 3.9
18
+ Classifier: Programming Language :: Python :: 3.10
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Topic :: Multimedia :: Video :: Capture
22
+ Classifier: Topic :: Scientific/Engineering :: Image Recognition
23
+ Classifier: Topic :: Home Automation
24
+ Requires-Python: >=3.9
25
+ Description-Content-Type: text/markdown
26
+ License-File: LICENSE
27
+ Requires-Dist: mediapipe>=0.10.0
28
+ Requires-Dist: opencv-python>=4.8.0
29
+ Requires-Dist: websockets>=14.0
30
+ Requires-Dist: numpy>=1.24.0
31
+ Provides-Extra: dev
32
+ Requires-Dist: pytest; extra == "dev"
33
+ Requires-Dist: black; extra == "dev"
34
+ Requires-Dist: mypy; extra == "dev"
35
+ Requires-Dist: ruff; extra == "dev"
36
+ Requires-Dist: pytest-asyncio>=1.2.0; extra == "dev"
37
+ Provides-Extra: data
38
+ Requires-Dist: pymongo>=4.7.0; extra == "data"
39
+ Requires-Dist: python-dotenv>=1.0; extra == "data"
40
+ Requires-Dist: tzdata; sys_platform == "win32" and extra == "data"
41
+ Provides-Extra: assistant
42
+ Requires-Dist: openai>=1.0.0; extra == "assistant"
43
+ Requires-Dist: anthropic>=0.30.0; extra == "assistant"
44
+ Requires-Dist: qdrant-client>=1.9.0; extra == "assistant"
45
+ Requires-Dist: sentence-transformers>=3.0.0; extra == "assistant"
46
+ Requires-Dist: sounddevice>=0.4.6; extra == "assistant"
47
+ Requires-Dist: pywhispercpp>=1.2.0; extra == "assistant"
48
+ Requires-Dist: piper-tts>=1.2.0; extra == "assistant"
49
+ Dynamic: license-file
50
+
51
+ # glasscube
52
+
53
+ Build gesture-controlled smart mirrors in Python. A webcam watches your hands,
54
+ [MediaPipe](https://developers.google.com/mediapipe) hand tracking turns their movement
55
+ into gestures (pinch, click, scroll, zoom, swipes), and those gestures move between
56
+ pages and drive widgets in a React UI that the package serves itself. An optional voice
57
+ assistant (wake word, Whisper, any OpenAI-compatible LLM, Piper TTS) answers out loud.
58
+
59
+ Everything runs locally: no account, no cloud service required.
60
+
61
+ ## Install
62
+
63
+ ```bash
64
+ pip install glasscube # gestures, pages, widgets, the server and its UI
65
+ pip install "glasscube[assistant]" # + the voice assistant
66
+ pip install "glasscube[data]" # + MongoDB providers and .env support
67
+ ```
68
+
69
+ Python 3.9 or newer (3.10 to 3.12 for the `assistant` extra, whose speech models lag behind), and a webcam.
70
+
71
+ ## Run the demo mirror
72
+
73
+ ```bash
74
+ glasscube
75
+ ```
76
+
77
+ Then open <http://localhost:8080> (full screen it on the mirror's display). Raise a hand
78
+ in front of the camera: the cursor follows your index finger, a two-finger swipe changes
79
+ page, a pinch grabs and moves a widget. Without a camera it still starts, with no gestures.
80
+
81
+ The voice assistant runs when the `assistant` extra is installed. It uses
82
+ [Ollama](https://ollama.com) by default (`ollama pull qwen2.5:0.5b`, then `ollama serve`);
83
+ point it at any OpenAI-compatible endpoint with `GLASSCUBE_LLM_BASE_URL`,
84
+ `GLASSCUBE_LLM_MODEL` and `GLASSCUBE_LLM_API_KEY`, or turn it off with
85
+ `GLASSCUBE_ASSISTANT=0`.
86
+
87
+ Settings are environment variables, also read from a `.env` file in the working directory
88
+ (with the `data` extra). The full list is in
89
+ [.env.example](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/backend/.env.example).
90
+ Runtime files (the downloaded hand model, the assistant's vector store) go in your user
91
+ data folder (`~/.local/share/glasscube`, `%LOCALAPPDATA%\glasscube` or
92
+ `~/Library/Application Support/glasscube`); move them with `GLASSCUBE_DATA_DIR`.
93
+
94
+ ## Build your own mirror
95
+
96
+ A mirror is a graph: pages (nodes) hold widgets, and gestures (edges) move between them.
97
+
98
+ ```python
99
+ import asyncio
100
+
101
+ from glasscube.server import SmartMirrorServer
102
+ from glasscube.widgets import CalendarWidget, TextWidget, WeatherWidget
103
+ from glasscube.widgets.providers import OpenMeteoWeatherProvider
104
+
105
+ weather = WeatherWidget(
106
+ position={"x": 0.1, "y": 0.1}, # fractions of the screen
107
+ data_provider=OpenMeteoWeatherProvider(city="Paris", latitude=48.86, longitude=2.35),
108
+ refresh_interval=600, # seconds
109
+ )
110
+ agenda = CalendarWidget(
111
+ position={"x": 0.6, "y": 0.1},
112
+ # Any callable, sync or async: its result becomes the widget's data.
113
+ data_provider=lambda: {"events": [{"title": "Standup", "start": "2026-11-18T09:00:00"}]},
114
+ refresh_interval=60,
115
+ )
116
+ tips = TextWidget(
117
+ position={"x": 0.5, "y": 0.5},
118
+ content={"tips": ["Swipe to cycle", "Scroll for sections"], "about": ["Made with glasscube"]},
119
+ )
120
+
121
+ graph = {
122
+ "nodes": [
123
+ {"id": "home", "widgets": [weather, agenda]},
124
+ {"id": "tips", "widgets": [tips]},
125
+ ],
126
+ "edges": [
127
+ {"from": "home", "to": "tips", "gesture": "two_fingers_swipe_left"},
128
+ {"from": "tips", "to": "home", "gesture": "two_fingers_swipe_right"},
129
+ ],
130
+ }
131
+
132
+ asyncio.run(SmartMirrorServer(graph=graph).run()) # UI on http://localhost:8080
133
+ ```
134
+
135
+ Built-in widgets: `TextWidget`, `WeatherWidget`, `CalendarWidget`, `NewsWidget`,
136
+ `EventsWidget`, `QRGalleryWidget`, `GamesWidget`, `DrawingWidget`. Built-in gestures:
137
+ `pinch_start/move/end`, `left_click`, `right_click`, `scroll_*`, `zoom_*`,
138
+ `swipe_left/right/up/down` and `two_fingers_swipe_left/right/up/down`.
139
+
140
+ ### Your own gestures, widgets and reactions
141
+
142
+ ```python
143
+ from glasscube.core import StateManager
144
+ from glasscube.vision import Gesture, GestureDetector, GestureEvent, SIDES, default_gesture_set
145
+
146
+ class FistGesture(Gesture):
147
+ name = "fist"
148
+
149
+ def __init__(self):
150
+ self._was_fist = {side: False for side in SIDES}
151
+
152
+ def detect(self, context):
153
+ events = []
154
+ for side in SIDES:
155
+ hand = context.hands[side]
156
+ fingers = ("index_up", "middle_up", "ring_up", "pinky_up")
157
+ is_fist = hand.detected and not any(context.signals[f][side] for f in fingers)
158
+ if is_fist and not self._was_fist[side]:
159
+ events.append(GestureEvent(type="fist", hand=side, location=hand.palm_center))
160
+ self._was_fist[side] = is_fist
161
+ return events
162
+
163
+ state = StateManager(graph=graph, event_bindings={
164
+ "fist": {"method": "set_current_node", "kwargs": {"node_id": "home"}}, # a fist goes home
165
+ })
166
+ detector = GestureDetector(gestures=default_gesture_set() + [FistGesture()])
167
+ server = SmartMirrorServer(state_manager=state, gesture_detector=detector)
168
+ ```
169
+
170
+ A new widget type needs a React component on the UI side as well: see
171
+ [extending.md](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/docs/extending.md).
172
+
173
+ ## Documentation
174
+
175
+ - [Architecture](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/docs/architecture.md): the camera → gestures → state → UI loop, module by module
176
+ - [Extending](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/docs/extending.md): gestures, widgets, pages, bindings
177
+ - [Vision](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/docs/vision.md): the hand signals and gesture tuning
178
+ - [Protocol](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/docs/protocol.md): the WebSocket messages between backend and UI
179
+
180
+ ## Developing the SDK
181
+
182
+ From a checkout of [the repository](https://github.com/SmartGlassCube/Smart-Mirror-SDK),
183
+ in `backend/`, with [uv](https://docs.astral.sh/uv/):
184
+
185
+ ```bash
186
+ uv sync --extra data --extra assistant # backend/.venv; plain `uv sync` = gestures + server only
187
+ uv run glasscube # also starts ../frontend's Vite dev server (needs npm)
188
+ uv run pytest # no camera needed
189
+ uv run python scripts/build_frontend.py # bundle the UI into glasscube/ui/ before `uv build`
190
+ ```
191
+
192
+ | Path | |
193
+ |---|---|
194
+ | `glasscube/app/` | The demo mirror app: `settings.py` (env vars), `pages.py` (widgets + page graph), `main.py` |
195
+ | `glasscube/server/` | `SmartMirrorServer`: WebSocket, camera loop, widget refresh, assistant hooks, the bundled UI |
196
+ | `glasscube/{vision,core,widgets,assistant,content,hal,config}/` | The SDK pieces, usable on their own |
197
+ | `tests/` | One folder per package |
198
+ | `scripts/` | `lan_media_server.py` (share a camera/mic over the LAN), `content_sync_daemon.py`, `build_frontend.py`, checks |
199
+ | `examples/` | `gesture_detector_debug.py`: annotated camera window + gesture feed |
200
+
201
+ ## License
202
+
203
+ MIT
@@ -0,0 +1,153 @@
1
+ # glasscube
2
+
3
+ Build gesture-controlled smart mirrors in Python. A webcam watches your hands,
4
+ [MediaPipe](https://developers.google.com/mediapipe) hand tracking turns their movement
5
+ into gestures (pinch, click, scroll, zoom, swipes), and those gestures move between
6
+ pages and drive widgets in a React UI that the package serves itself. An optional voice
7
+ assistant (wake word, Whisper, any OpenAI-compatible LLM, Piper TTS) answers out loud.
8
+
9
+ Everything runs locally: no account, no cloud service required.
10
+
11
+ ## Install
12
+
13
+ ```bash
14
+ pip install glasscube # gestures, pages, widgets, the server and its UI
15
+ pip install "glasscube[assistant]" # + the voice assistant
16
+ pip install "glasscube[data]" # + MongoDB providers and .env support
17
+ ```
18
+
19
+ Python 3.9 or newer (3.10 to 3.12 for the `assistant` extra, whose speech models lag behind), and a webcam.
20
+
21
+ ## Run the demo mirror
22
+
23
+ ```bash
24
+ glasscube
25
+ ```
26
+
27
+ Then open <http://localhost:8080> (full screen it on the mirror's display). Raise a hand
28
+ in front of the camera: the cursor follows your index finger, a two-finger swipe changes
29
+ page, a pinch grabs and moves a widget. Without a camera it still starts, with no gestures.
30
+
31
+ The voice assistant runs when the `assistant` extra is installed. It uses
32
+ [Ollama](https://ollama.com) by default (`ollama pull qwen2.5:0.5b`, then `ollama serve`);
33
+ point it at any OpenAI-compatible endpoint with `GLASSCUBE_LLM_BASE_URL`,
34
+ `GLASSCUBE_LLM_MODEL` and `GLASSCUBE_LLM_API_KEY`, or turn it off with
35
+ `GLASSCUBE_ASSISTANT=0`.
36
+
37
+ Settings are environment variables, also read from a `.env` file in the working directory
38
+ (with the `data` extra). The full list is in
39
+ [.env.example](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/backend/.env.example).
40
+ Runtime files (the downloaded hand model, the assistant's vector store) go in your user
41
+ data folder (`~/.local/share/glasscube`, `%LOCALAPPDATA%\glasscube` or
42
+ `~/Library/Application Support/glasscube`); move them with `GLASSCUBE_DATA_DIR`.
43
+
44
+ ## Build your own mirror
45
+
46
+ A mirror is a graph: pages (nodes) hold widgets, and gestures (edges) move between them.
47
+
48
+ ```python
49
+ import asyncio
50
+
51
+ from glasscube.server import SmartMirrorServer
52
+ from glasscube.widgets import CalendarWidget, TextWidget, WeatherWidget
53
+ from glasscube.widgets.providers import OpenMeteoWeatherProvider
54
+
55
+ weather = WeatherWidget(
56
+ position={"x": 0.1, "y": 0.1}, # fractions of the screen
57
+ data_provider=OpenMeteoWeatherProvider(city="Paris", latitude=48.86, longitude=2.35),
58
+ refresh_interval=600, # seconds
59
+ )
60
+ agenda = CalendarWidget(
61
+ position={"x": 0.6, "y": 0.1},
62
+ # Any callable, sync or async: its result becomes the widget's data.
63
+ data_provider=lambda: {"events": [{"title": "Standup", "start": "2026-11-18T09:00:00"}]},
64
+ refresh_interval=60,
65
+ )
66
+ tips = TextWidget(
67
+ position={"x": 0.5, "y": 0.5},
68
+ content={"tips": ["Swipe to cycle", "Scroll for sections"], "about": ["Made with glasscube"]},
69
+ )
70
+
71
+ graph = {
72
+ "nodes": [
73
+ {"id": "home", "widgets": [weather, agenda]},
74
+ {"id": "tips", "widgets": [tips]},
75
+ ],
76
+ "edges": [
77
+ {"from": "home", "to": "tips", "gesture": "two_fingers_swipe_left"},
78
+ {"from": "tips", "to": "home", "gesture": "two_fingers_swipe_right"},
79
+ ],
80
+ }
81
+
82
+ asyncio.run(SmartMirrorServer(graph=graph).run()) # UI on http://localhost:8080
83
+ ```
84
+
85
+ Built-in widgets: `TextWidget`, `WeatherWidget`, `CalendarWidget`, `NewsWidget`,
86
+ `EventsWidget`, `QRGalleryWidget`, `GamesWidget`, `DrawingWidget`. Built-in gestures:
87
+ `pinch_start/move/end`, `left_click`, `right_click`, `scroll_*`, `zoom_*`,
88
+ `swipe_left/right/up/down` and `two_fingers_swipe_left/right/up/down`.
89
+
90
+ ### Your own gestures, widgets and reactions
91
+
92
+ ```python
93
+ from glasscube.core import StateManager
94
+ from glasscube.vision import Gesture, GestureDetector, GestureEvent, SIDES, default_gesture_set
95
+
96
+ class FistGesture(Gesture):
97
+ name = "fist"
98
+
99
+ def __init__(self):
100
+ self._was_fist = {side: False for side in SIDES}
101
+
102
+ def detect(self, context):
103
+ events = []
104
+ for side in SIDES:
105
+ hand = context.hands[side]
106
+ fingers = ("index_up", "middle_up", "ring_up", "pinky_up")
107
+ is_fist = hand.detected and not any(context.signals[f][side] for f in fingers)
108
+ if is_fist and not self._was_fist[side]:
109
+ events.append(GestureEvent(type="fist", hand=side, location=hand.palm_center))
110
+ self._was_fist[side] = is_fist
111
+ return events
112
+
113
+ state = StateManager(graph=graph, event_bindings={
114
+ "fist": {"method": "set_current_node", "kwargs": {"node_id": "home"}}, # a fist goes home
115
+ })
116
+ detector = GestureDetector(gestures=default_gesture_set() + [FistGesture()])
117
+ server = SmartMirrorServer(state_manager=state, gesture_detector=detector)
118
+ ```
119
+
120
+ A new widget type needs a React component on the UI side as well: see
121
+ [extending.md](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/docs/extending.md).
122
+
123
+ ## Documentation
124
+
125
+ - [Architecture](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/docs/architecture.md): the camera → gestures → state → UI loop, module by module
126
+ - [Extending](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/docs/extending.md): gestures, widgets, pages, bindings
127
+ - [Vision](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/docs/vision.md): the hand signals and gesture tuning
128
+ - [Protocol](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/docs/protocol.md): the WebSocket messages between backend and UI
129
+
130
+ ## Developing the SDK
131
+
132
+ From a checkout of [the repository](https://github.com/SmartGlassCube/Smart-Mirror-SDK),
133
+ in `backend/`, with [uv](https://docs.astral.sh/uv/):
134
+
135
+ ```bash
136
+ uv sync --extra data --extra assistant # backend/.venv; plain `uv sync` = gestures + server only
137
+ uv run glasscube # also starts ../frontend's Vite dev server (needs npm)
138
+ uv run pytest # no camera needed
139
+ uv run python scripts/build_frontend.py # bundle the UI into glasscube/ui/ before `uv build`
140
+ ```
141
+
142
+ | Path | |
143
+ |---|---|
144
+ | `glasscube/app/` | The demo mirror app: `settings.py` (env vars), `pages.py` (widgets + page graph), `main.py` |
145
+ | `glasscube/server/` | `SmartMirrorServer`: WebSocket, camera loop, widget refresh, assistant hooks, the bundled UI |
146
+ | `glasscube/{vision,core,widgets,assistant,content,hal,config}/` | The SDK pieces, usable on their own |
147
+ | `tests/` | One folder per package |
148
+ | `scripts/` | `lan_media_server.py` (share a camera/mic over the LAN), `content_sync_daemon.py`, `build_frontend.py`, checks |
149
+ | `examples/` | `gesture_detector_debug.py`: annotated camera window + gesture feed |
150
+
151
+ ## License
152
+
153
+ MIT
@@ -0,0 +1,7 @@
1
+ """GlassCube: the Smart Mirror SDK. Webcam hand gestures -> pages and widgets -> a React UI over WebSocket, plus a voice assistant."""
2
+
3
+ from .config.env import adopt_legacy_env
4
+
5
+ __version__ = "0.1.0"
6
+
7
+ adopt_legacy_env()
@@ -0,0 +1,3 @@
1
+ from .app.main import main
2
+
3
+ main()
@@ -0,0 +1,4 @@
1
+ """The mirror app: the SDK pieces (server, widgets, assistant, content sync) put together.
2
+
3
+ Run it with `python -m glasscube` from backend/.
4
+ """
@@ -0,0 +1,68 @@
1
+ import asyncio
2
+ import logging
3
+ import os
4
+ import shutil
5
+ from pathlib import Path
6
+
7
+ # <repo>/frontend, next to backend/. Only exists in a checkout of the repo, not in a pip install.
8
+ DEFAULT_FRONTEND_DIR = Path(__file__).resolve().parents[3] / "frontend"
9
+
10
+ logger = logging.getLogger(__name__)
11
+
12
+
13
+ class FrontendLauncher:
14
+ """Dev convenience: runs the frontend's Vite dev server (npm install first if needed).
15
+
16
+ The frontend is a separate app; this only saves starting it by hand. It is
17
+ told the backend's WebSocket port through VITE_GLASSCUBE_WS_PORT.
18
+ """
19
+
20
+ def __init__(self, frontend_dir=None, host="localhost", port=5173, ws_port=8080):
21
+ self.frontend_dir = Path(frontend_dir) if frontend_dir else DEFAULT_FRONTEND_DIR
22
+ self.host = host
23
+ self.port = port
24
+ self.ws_port = ws_port
25
+ self.process = None
26
+
27
+ def available(self):
28
+ """True when there's a frontend/ checkout (with its package.json) and npm to run it."""
29
+ return (self.frontend_dir / "package.json").is_file() and shutil.which("npm") is not None
30
+
31
+ async def start(self):
32
+ npm = shutil.which("npm")
33
+ if not self.frontend_dir.exists():
34
+ logger.warning("Skipping frontend launch: directory not found at %s", self.frontend_dir)
35
+ return
36
+ if npm is None:
37
+ logger.warning("Skipping frontend launch: npm is not installed or not in PATH")
38
+ return
39
+
40
+ if not (self.frontend_dir / "node_modules").exists():
41
+ logger.info("Installing frontend dependencies (npm install)...")
42
+ install = await asyncio.create_subprocess_exec(npm, "install", cwd=str(self.frontend_dir))
43
+ if await install.wait() != 0:
44
+ logger.error("npm install failed with exit code %s", install.returncode)
45
+ return
46
+
47
+ logger.info("Starting frontend on http://%s:%s", self.host, self.port)
48
+ self.process = await asyncio.create_subprocess_exec(
49
+ npm, "run", "dev", "--", "--host", self.host, "--port", str(self.port),
50
+ cwd=str(self.frontend_dir),
51
+ # Read by frontend/src/services/useWebSocket.ts
52
+ env={**os.environ, "VITE_GLASSCUBE_WS_PORT": str(self.ws_port)},
53
+ )
54
+
55
+ async def stop(self):
56
+ if self.process is None:
57
+ return
58
+ if os.name == "nt":
59
+ # terminate() would only end npm's cmd wrapper and leave Vite holding the port.
60
+ killer = await asyncio.create_subprocess_exec(
61
+ "taskkill", "/T", "/F", "/PID", str(self.process.pid),
62
+ stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL,
63
+ )
64
+ await killer.wait()
65
+ else:
66
+ self.process.terminate()
67
+ await self.process.wait()
68
+ self.process = None
@@ -0,0 +1,120 @@
1
+ """Entry point: the `glasscube` command, or `python -m glasscube`."""
2
+
3
+ import asyncio
4
+ import logging
5
+
6
+ from ..core import StateManager
7
+ from ..hal import HAL
8
+ from ..server import SmartMirrorServer
9
+ from ..server.static_ui import bundled_ui_dir
10
+ from .frontend_dev import FrontendLauncher
11
+ from .pages import GESTURE_CONFIG, HOME, build_graph
12
+ from .settings import Settings
13
+
14
+ logger = logging.getLogger(__name__)
15
+
16
+
17
+ def build_assistant(settings, content_sync=None):
18
+ """The voice chatbot. Ollama-backed by default, so it runs without a paid API key --
19
+ pull a model first (e.g. `ollama pull qwen2.5:0.5b`) and start `ollama serve`.
20
+
21
+ With `content_sync`, it retrieves from the Mongo-derived Qdrant collection and
22
+ gets the live events/announcements prompt block instead of the seed corpus.
23
+
24
+ None when disabled (GLASSCUBE_ASSISTANT=0) or when the `assistant` extra isn't installed.
25
+ """
26
+ if not settings.assistant:
27
+ return None
28
+ try:
29
+ return _assistant(settings, content_sync)
30
+ except ImportError as error: # the `assistant` extra isn't installed
31
+ logger.warning(
32
+ "Voice assistant off: %s. Install it with `pip install \"glasscube[assistant]\"`, "
33
+ "or set GLASSCUBE_ASSISTANT=0 to hide this.", error,
34
+ )
35
+ return None
36
+
37
+
38
+ def _assistant(settings, content_sync):
39
+ from ..assistant import Assistant
40
+ from ..assistant.llm import OpenAICompatBackend
41
+ from ..assistant.rag import SEED_CORPUS_DIR
42
+
43
+ hal = HAL(input_device=settings.mic)
44
+ backend = OpenAICompatBackend(
45
+ api_key=settings.llm_api_key,
46
+ base_url=settings.llm_base_url,
47
+ model=settings.llm_model,
48
+ )
49
+ if content_sync is not None:
50
+ return Assistant(
51
+ llm=backend, stt=True, tts=True, hal=hal,
52
+ store=content_sync.indexer.store,
53
+ extra_context=content_sync.prompt_block,
54
+ )
55
+ return Assistant(llm=backend, documents=str(SEED_CORPUS_DIR), stt=True, tts=True, hal=hal)
56
+
57
+
58
+ def build_content_sync(settings):
59
+ """Club content from the Dashboard repo's Mongo (glasscube.content); None without MONGO_URI."""
60
+ if not settings.content_mongo_uri:
61
+ return None
62
+ from ..content import build_content_sync as build # needs the `data` extra
63
+
64
+ return build()
65
+
66
+
67
+ def build_server(settings):
68
+ content_sync = build_content_sync(settings)
69
+ state_manager = StateManager(
70
+ graph=build_graph(settings, content_sync),
71
+ initial_node_id=HOME,
72
+ gesture_config=GESTURE_CONFIG,
73
+ )
74
+ return SmartMirrorServer(
75
+ state_manager=state_manager,
76
+ camera_id=settings.camera,
77
+ ws_port=settings.ws_port,
78
+ assistant=build_assistant(settings, content_sync),
79
+ content_sync=content_sync,
80
+ )
81
+
82
+
83
+ async def run(server, frontend=None):
84
+ if frontend is not None:
85
+ await frontend.start()
86
+ try:
87
+ await server.run()
88
+ finally:
89
+ if frontend is not None:
90
+ await frontend.stop()
91
+
92
+
93
+ def build_frontend(settings):
94
+ """The Vite dev server in a checkout with npm; None when the bundled UI is served instead (or turned off)."""
95
+ if not settings.frontend:
96
+ return None
97
+ launcher = FrontendLauncher(port=settings.frontend_port, ws_port=settings.ws_port)
98
+ if launcher.available():
99
+ return launcher
100
+ if bundled_ui_dir() is not None:
101
+ logger.info("Mirror UI on http://localhost:%s", settings.ws_port)
102
+ else:
103
+ logger.warning("No mirror UI: this install has no bundled UI and there's no frontend/ checkout to run.")
104
+ return None
105
+
106
+
107
+ def main():
108
+ # INFO on the glasscube.* loggers surfaces the voice pipeline's play-by-play
109
+ # (VAD, raw Whisper transcripts, wake-word matches; see docs/voice-pipeline-testing.md).
110
+ # Root stays at WARNING so third-party libraries don't drown it out.
111
+ logging.basicConfig(level=logging.WARNING, format="%(asctime)s %(name)s %(levelname)s %(message)s")
112
+ logging.getLogger("glasscube").setLevel(logging.INFO)
113
+
114
+ settings = Settings.from_env()
115
+ server = build_server(settings)
116
+ frontend = build_frontend(settings)
117
+ try:
118
+ asyncio.run(run(server, frontend))
119
+ except KeyboardInterrupt:
120
+ pass