glasscube 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- glasscube-0.1.0/LICENSE +21 -0
- glasscube-0.1.0/PKG-INFO +203 -0
- glasscube-0.1.0/README.md +153 -0
- glasscube-0.1.0/glasscube/__init__.py +7 -0
- glasscube-0.1.0/glasscube/__main__.py +3 -0
- glasscube-0.1.0/glasscube/app/__init__.py +4 -0
- glasscube-0.1.0/glasscube/app/frontend_dev.py +68 -0
- glasscube-0.1.0/glasscube/app/main.py +120 -0
- glasscube-0.1.0/glasscube/app/pages.py +185 -0
- glasscube-0.1.0/glasscube/app/settings.py +93 -0
- glasscube-0.1.0/glasscube/assistant/__init__.py +3 -0
- glasscube-0.1.0/glasscube/assistant/assistant.py +370 -0
- glasscube-0.1.0/glasscube/assistant/llm/__init__.py +5 -0
- glasscube-0.1.0/glasscube/assistant/llm/backends/__init__.py +4 -0
- glasscube-0.1.0/glasscube/assistant/llm/backends/anthropic.py +45 -0
- glasscube-0.1.0/glasscube/assistant/llm/backends/openai_compat.py +27 -0
- glasscube-0.1.0/glasscube/assistant/llm/base.py +21 -0
- glasscube-0.1.0/glasscube/assistant/llm/client.py +20 -0
- glasscube-0.1.0/glasscube/assistant/rag/__init__.py +11 -0
- glasscube-0.1.0/glasscube/assistant/rag/chunker.py +68 -0
- glasscube-0.1.0/glasscube/assistant/rag/embedder.py +37 -0
- glasscube-0.1.0/glasscube/assistant/rag/retriever.py +28 -0
- glasscube-0.1.0/glasscube/assistant/rag/seed_corpus/about_ieee_insat.txt +5 -0
- glasscube-0.1.0/glasscube/assistant/rag/seed_corpus/ieee_insat_events.md +5 -0
- glasscube-0.1.0/glasscube/assistant/rag/seed_corpus/smart_mirror_project.txt +4 -0
- glasscube-0.1.0/glasscube/assistant/rag/store.py +107 -0
- glasscube-0.1.0/glasscube/assistant/stt/__init__.py +5 -0
- glasscube-0.1.0/glasscube/assistant/stt/vad.py +148 -0
- glasscube-0.1.0/glasscube/assistant/stt/wake_word.py +186 -0
- glasscube-0.1.0/glasscube/assistant/stt/whisper.py +97 -0
- glasscube-0.1.0/glasscube/assistant/tools/__init__.py +3 -0
- glasscube-0.1.0/glasscube/assistant/tools/manager.py +25 -0
- glasscube-0.1.0/glasscube/assistant/tts/__init__.py +3 -0
- glasscube-0.1.0/glasscube/assistant/tts/piper.py +63 -0
- glasscube-0.1.0/glasscube/config/__init__.py +13 -0
- glasscube-0.1.0/glasscube/config/env.py +16 -0
- glasscube-0.1.0/glasscube/config/json_config/default_gesture_config.json +8 -0
- glasscube-0.1.0/glasscube/config/json_config/default_widget_event_bindings.json +58 -0
- glasscube-0.1.0/glasscube/config/loader.py +21 -0
- glasscube-0.1.0/glasscube/config/paths.py +32 -0
- glasscube-0.1.0/glasscube/content/__init__.py +23 -0
- glasscube-0.1.0/glasscube/content/cache.py +97 -0
- glasscube-0.1.0/glasscube/content/config.py +68 -0
- glasscube-0.1.0/glasscube/content/indexer.py +54 -0
- glasscube-0.1.0/glasscube/content/routing.py +180 -0
- glasscube-0.1.0/glasscube/content/sync.py +420 -0
- glasscube-0.1.0/glasscube/content/templates.py +122 -0
- glasscube-0.1.0/glasscube/core/__init__.py +3 -0
- glasscube-0.1.0/glasscube/core/actions.py +48 -0
- glasscube-0.1.0/glasscube/core/callables.py +23 -0
- glasscube-0.1.0/glasscube/core/navigation.py +66 -0
- glasscube-0.1.0/glasscube/core/state_manager.py +82 -0
- glasscube-0.1.0/glasscube/core/widget_events.py +120 -0
- glasscube-0.1.0/glasscube/hal/__init__.py +3 -0
- glasscube-0.1.0/glasscube/hal/hal.py +131 -0
- glasscube-0.1.0/glasscube/server/__init__.py +3 -0
- glasscube-0.1.0/glasscube/server/app.py +241 -0
- glasscube-0.1.0/glasscube/server/camera.py +147 -0
- glasscube-0.1.0/glasscube/server/camera_preview.py +54 -0
- glasscube-0.1.0/glasscube/server/hand_pointer.py +42 -0
- glasscube-0.1.0/glasscube/server/static_ui.py +47 -0
- glasscube-0.1.0/glasscube/server/websocket.py +54 -0
- glasscube-0.1.0/glasscube/server/widget_data.py +33 -0
- glasscube-0.1.0/glasscube/ui/assets/5 1-CpBR8EWH.png +0 -0
- glasscube-0.1.0/glasscube/ui/assets/index-BB9bg-We.js +9 -0
- glasscube-0.1.0/glasscube/ui/assets/index-CQRKDs3v.css +1 -0
- glasscube-0.1.0/glasscube/ui/assets/press-start-2p-cyrillic-400-normal-DBn7fddJ.woff2 +0 -0
- glasscube-0.1.0/glasscube/ui/assets/press-start-2p-cyrillic-ext-400-normal-CIYZrvmP.woff2 +0 -0
- glasscube-0.1.0/glasscube/ui/assets/press-start-2p-greek-400-normal-DQfalijU.woff2 +0 -0
- glasscube-0.1.0/glasscube/ui/assets/press-start-2p-latin-400-normal-KLytu4cr.woff +0 -0
- glasscube-0.1.0/glasscube/ui/assets/press-start-2p-latin-400-normal-_wFEWmAB.woff2 +0 -0
- glasscube-0.1.0/glasscube/ui/assets/press-start-2p-latin-ext-400-normal-DBgyq7ky.woff +0 -0
- glasscube-0.1.0/glasscube/ui/assets/press-start-2p-latin-ext-400-normal-DpfYvR5A.woff2 +0 -0
- glasscube-0.1.0/glasscube/ui/assets/sleeping-abdo-J4WKKI3q.gif +0 -0
- glasscube-0.1.0/glasscube/ui/assets/transition-abdo-DsBJJ6fU.gif +0 -0
- glasscube-0.1.0/glasscube/ui/assets/vt323-latin-400-normal-Dqn9ZSPj.woff +0 -0
- glasscube-0.1.0/glasscube/ui/assets/vt323-latin-400-normal-wDgJuOC9.woff2 +0 -0
- glasscube-0.1.0/glasscube/ui/assets/vt323-latin-ext-400-normal-DqOq58ri.woff +0 -0
- glasscube-0.1.0/glasscube/ui/assets/vt323-latin-ext-400-normal-iUdYUaX4.woff2 +0 -0
- glasscube-0.1.0/glasscube/ui/assets/vt323-vietnamese-400-normal-D0VnvwZ9.woff2 +0 -0
- glasscube-0.1.0/glasscube/ui/assets/vt323-vietnamese-400-normal-D_5Q7Vcv.woff +0 -0
- glasscube-0.1.0/glasscube/ui/index.html +13 -0
- glasscube-0.1.0/glasscube/vision/__init__.py +43 -0
- glasscube-0.1.0/glasscube/vision/debug_overlay.py +141 -0
- glasscube-0.1.0/glasscube/vision/gesture_detector.py +178 -0
- glasscube-0.1.0/glasscube/vision/gesture_types.py +54 -0
- glasscube-0.1.0/glasscube/vision/gestures/__init__.py +22 -0
- glasscube-0.1.0/glasscube/vision/gestures/base.py +198 -0
- glasscube-0.1.0/glasscube/vision/gestures/click.py +38 -0
- glasscube-0.1.0/glasscube/vision/gestures/default_set.py +21 -0
- glasscube-0.1.0/glasscube/vision/gestures/pinch.py +44 -0
- glasscube-0.1.0/glasscube/vision/gestures/scroll.py +37 -0
- glasscube-0.1.0/glasscube/vision/gestures/swipe.py +23 -0
- glasscube-0.1.0/glasscube/vision/gestures/two_fingers_swipe.py +39 -0
- glasscube-0.1.0/glasscube/vision/gestures/zoom.py +52 -0
- glasscube-0.1.0/glasscube/vision/hand_data.py +43 -0
- glasscube-0.1.0/glasscube/vision/landmark_detector.py +104 -0
- glasscube-0.1.0/glasscube/widgets/__init__.py +21 -0
- glasscube-0.1.0/glasscube/widgets/base.py +75 -0
- glasscube-0.1.0/glasscube/widgets/calendar.py +13 -0
- glasscube-0.1.0/glasscube/widgets/drawing.py +57 -0
- glasscube-0.1.0/glasscube/widgets/events.py +16 -0
- glasscube-0.1.0/glasscube/widgets/games.py +15 -0
- glasscube-0.1.0/glasscube/widgets/news.py +7 -0
- glasscube-0.1.0/glasscube/widgets/providers/__init__.py +12 -0
- glasscube-0.1.0/glasscube/widgets/providers/mongo_calendar.py +44 -0
- glasscube-0.1.0/glasscube/widgets/providers/open_meteo.py +40 -0
- glasscube-0.1.0/glasscube/widgets/providers/qr_catalog.py +6 -0
- glasscube-0.1.0/glasscube/widgets/qr_gallery.py +7 -0
- glasscube-0.1.0/glasscube/widgets/text.py +85 -0
- glasscube-0.1.0/glasscube/widgets/weather.py +11 -0
- glasscube-0.1.0/glasscube.egg-info/PKG-INFO +203 -0
- glasscube-0.1.0/glasscube.egg-info/SOURCES.txt +118 -0
- glasscube-0.1.0/glasscube.egg-info/dependency_links.txt +1 -0
- glasscube-0.1.0/glasscube.egg-info/entry_points.txt +2 -0
- glasscube-0.1.0/glasscube.egg-info/requires.txt +27 -0
- glasscube-0.1.0/glasscube.egg-info/top_level.txt +1 -0
- glasscube-0.1.0/pyproject.toml +102 -0
- glasscube-0.1.0/setup.cfg +4 -0
- glasscube-0.1.0/tests/test_legacy_env.py +13 -0
glasscube-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Smart Glass Cube
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
glasscube-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: glasscube
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Build gesture-controlled smart mirrors: webcam hand gestures drive pages and widgets of a React UI, plus a voice assistant
|
|
5
|
+
Author-email: Moetez Fradi <moetezfradi7@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/SmartGlassCube/Smart-Mirror-SDK
|
|
8
|
+
Project-URL: Documentation, https://github.com/SmartGlassCube/Smart-Mirror-SDK/tree/main/docs
|
|
9
|
+
Project-URL: Repository, https://github.com/SmartGlassCube/Smart-Mirror-SDK
|
|
10
|
+
Project-URL: Issues, https://github.com/SmartGlassCube/Smart-Mirror-SDK/issues
|
|
11
|
+
Keywords: smart mirror,gesture recognition,hand tracking,mediapipe,computer vision,voice assistant,websocket
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Intended Audience :: Education
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Topic :: Multimedia :: Video :: Capture
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering :: Image Recognition
|
|
23
|
+
Classifier: Topic :: Home Automation
|
|
24
|
+
Requires-Python: >=3.9
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
License-File: LICENSE
|
|
27
|
+
Requires-Dist: mediapipe>=0.10.0
|
|
28
|
+
Requires-Dist: opencv-python>=4.8.0
|
|
29
|
+
Requires-Dist: websockets>=14.0
|
|
30
|
+
Requires-Dist: numpy>=1.24.0
|
|
31
|
+
Provides-Extra: dev
|
|
32
|
+
Requires-Dist: pytest; extra == "dev"
|
|
33
|
+
Requires-Dist: black; extra == "dev"
|
|
34
|
+
Requires-Dist: mypy; extra == "dev"
|
|
35
|
+
Requires-Dist: ruff; extra == "dev"
|
|
36
|
+
Requires-Dist: pytest-asyncio>=1.2.0; extra == "dev"
|
|
37
|
+
Provides-Extra: data
|
|
38
|
+
Requires-Dist: pymongo>=4.7.0; extra == "data"
|
|
39
|
+
Requires-Dist: python-dotenv>=1.0; extra == "data"
|
|
40
|
+
Requires-Dist: tzdata; sys_platform == "win32" and extra == "data"
|
|
41
|
+
Provides-Extra: assistant
|
|
42
|
+
Requires-Dist: openai>=1.0.0; extra == "assistant"
|
|
43
|
+
Requires-Dist: anthropic>=0.30.0; extra == "assistant"
|
|
44
|
+
Requires-Dist: qdrant-client>=1.9.0; extra == "assistant"
|
|
45
|
+
Requires-Dist: sentence-transformers>=3.0.0; extra == "assistant"
|
|
46
|
+
Requires-Dist: sounddevice>=0.4.6; extra == "assistant"
|
|
47
|
+
Requires-Dist: pywhispercpp>=1.2.0; extra == "assistant"
|
|
48
|
+
Requires-Dist: piper-tts>=1.2.0; extra == "assistant"
|
|
49
|
+
Dynamic: license-file
|
|
50
|
+
|
|
51
|
+
# glasscube
|
|
52
|
+
|
|
53
|
+
Build gesture-controlled smart mirrors in Python. A webcam watches your hands,
|
|
54
|
+
[MediaPipe](https://developers.google.com/mediapipe) hand tracking turns their movement
|
|
55
|
+
into gestures (pinch, click, scroll, zoom, swipes), and those gestures move between
|
|
56
|
+
pages and drive widgets in a React UI that the package serves itself. An optional voice
|
|
57
|
+
assistant (wake word, Whisper, any OpenAI-compatible LLM, Piper TTS) answers out loud.
|
|
58
|
+
|
|
59
|
+
Everything runs locally: no account, no cloud service required.
|
|
60
|
+
|
|
61
|
+
## Install
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
pip install glasscube # gestures, pages, widgets, the server and its UI
|
|
65
|
+
pip install "glasscube[assistant]" # + the voice assistant
|
|
66
|
+
pip install "glasscube[data]" # + MongoDB providers and .env support
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Python 3.9 or newer (3.10 to 3.12 for the `assistant` extra, whose speech models lag behind), and a webcam.
|
|
70
|
+
|
|
71
|
+
## Run the demo mirror
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
glasscube
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Then open <http://localhost:8080> (full screen it on the mirror's display). Raise a hand
|
|
78
|
+
in front of the camera: the cursor follows your index finger, a two-finger swipe changes
|
|
79
|
+
page, a pinch grabs and moves a widget. Without a camera it still starts, with no gestures.
|
|
80
|
+
|
|
81
|
+
The voice assistant runs when the `assistant` extra is installed. It uses
|
|
82
|
+
[Ollama](https://ollama.com) by default (`ollama pull qwen2.5:0.5b`, then `ollama serve`);
|
|
83
|
+
point it at any OpenAI-compatible endpoint with `GLASSCUBE_LLM_BASE_URL`,
|
|
84
|
+
`GLASSCUBE_LLM_MODEL` and `GLASSCUBE_LLM_API_KEY`, or turn it off with
|
|
85
|
+
`GLASSCUBE_ASSISTANT=0`.
|
|
86
|
+
|
|
87
|
+
Settings are environment variables, also read from a `.env` file in the working directory
|
|
88
|
+
(with the `data` extra). The full list is in
|
|
89
|
+
[.env.example](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/backend/.env.example).
|
|
90
|
+
Runtime files (the downloaded hand model, the assistant's vector store) go in your user
|
|
91
|
+
data folder (`~/.local/share/glasscube`, `%LOCALAPPDATA%\glasscube` or
|
|
92
|
+
`~/Library/Application Support/glasscube`); move them with `GLASSCUBE_DATA_DIR`.
|
|
93
|
+
|
|
94
|
+
## Build your own mirror
|
|
95
|
+
|
|
96
|
+
A mirror is a graph: pages (nodes) hold widgets, and gestures (edges) move between them.
|
|
97
|
+
|
|
98
|
+
```python
|
|
99
|
+
import asyncio
|
|
100
|
+
|
|
101
|
+
from glasscube.server import SmartMirrorServer
|
|
102
|
+
from glasscube.widgets import CalendarWidget, TextWidget, WeatherWidget
|
|
103
|
+
from glasscube.widgets.providers import OpenMeteoWeatherProvider
|
|
104
|
+
|
|
105
|
+
weather = WeatherWidget(
|
|
106
|
+
position={"x": 0.1, "y": 0.1}, # fractions of the screen
|
|
107
|
+
data_provider=OpenMeteoWeatherProvider(city="Paris", latitude=48.86, longitude=2.35),
|
|
108
|
+
refresh_interval=600, # seconds
|
|
109
|
+
)
|
|
110
|
+
agenda = CalendarWidget(
|
|
111
|
+
position={"x": 0.6, "y": 0.1},
|
|
112
|
+
# Any callable, sync or async: its result becomes the widget's data.
|
|
113
|
+
data_provider=lambda: {"events": [{"title": "Standup", "start": "2026-11-18T09:00:00"}]},
|
|
114
|
+
refresh_interval=60,
|
|
115
|
+
)
|
|
116
|
+
tips = TextWidget(
|
|
117
|
+
position={"x": 0.5, "y": 0.5},
|
|
118
|
+
content={"tips": ["Swipe to cycle", "Scroll for sections"], "about": ["Made with glasscube"]},
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
graph = {
|
|
122
|
+
"nodes": [
|
|
123
|
+
{"id": "home", "widgets": [weather, agenda]},
|
|
124
|
+
{"id": "tips", "widgets": [tips]},
|
|
125
|
+
],
|
|
126
|
+
"edges": [
|
|
127
|
+
{"from": "home", "to": "tips", "gesture": "two_fingers_swipe_left"},
|
|
128
|
+
{"from": "tips", "to": "home", "gesture": "two_fingers_swipe_right"},
|
|
129
|
+
],
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
asyncio.run(SmartMirrorServer(graph=graph).run()) # UI on http://localhost:8080
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
Built-in widgets: `TextWidget`, `WeatherWidget`, `CalendarWidget`, `NewsWidget`,
|
|
136
|
+
`EventsWidget`, `QRGalleryWidget`, `GamesWidget`, `DrawingWidget`. Built-in gestures:
|
|
137
|
+
`pinch_start/move/end`, `left_click`, `right_click`, `scroll_*`, `zoom_*`,
|
|
138
|
+
`swipe_left/right/up/down` and `two_fingers_swipe_left/right/up/down`.
|
|
139
|
+
|
|
140
|
+
### Your own gestures, widgets and reactions
|
|
141
|
+
|
|
142
|
+
```python
|
|
143
|
+
from glasscube.core import StateManager
|
|
144
|
+
from glasscube.vision import Gesture, GestureDetector, GestureEvent, SIDES, default_gesture_set
|
|
145
|
+
|
|
146
|
+
class FistGesture(Gesture):
|
|
147
|
+
name = "fist"
|
|
148
|
+
|
|
149
|
+
def __init__(self):
|
|
150
|
+
self._was_fist = {side: False for side in SIDES}
|
|
151
|
+
|
|
152
|
+
def detect(self, context):
|
|
153
|
+
events = []
|
|
154
|
+
for side in SIDES:
|
|
155
|
+
hand = context.hands[side]
|
|
156
|
+
fingers = ("index_up", "middle_up", "ring_up", "pinky_up")
|
|
157
|
+
is_fist = hand.detected and not any(context.signals[f][side] for f in fingers)
|
|
158
|
+
if is_fist and not self._was_fist[side]:
|
|
159
|
+
events.append(GestureEvent(type="fist", hand=side, location=hand.palm_center))
|
|
160
|
+
self._was_fist[side] = is_fist
|
|
161
|
+
return events
|
|
162
|
+
|
|
163
|
+
state = StateManager(graph=graph, event_bindings={
|
|
164
|
+
"fist": {"method": "set_current_node", "kwargs": {"node_id": "home"}}, # a fist goes home
|
|
165
|
+
})
|
|
166
|
+
detector = GestureDetector(gestures=default_gesture_set() + [FistGesture()])
|
|
167
|
+
server = SmartMirrorServer(state_manager=state, gesture_detector=detector)
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
A new widget type needs a React component on the UI side as well: see
|
|
171
|
+
[extending.md](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/docs/extending.md).
|
|
172
|
+
|
|
173
|
+
## Documentation
|
|
174
|
+
|
|
175
|
+
- [Architecture](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/docs/architecture.md): the camera → gestures → state → UI loop, module by module
|
|
176
|
+
- [Extending](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/docs/extending.md): gestures, widgets, pages, bindings
|
|
177
|
+
- [Vision](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/docs/vision.md): the hand signals and gesture tuning
|
|
178
|
+
- [Protocol](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/docs/protocol.md): the WebSocket messages between backend and UI
|
|
179
|
+
|
|
180
|
+
## Developing the SDK
|
|
181
|
+
|
|
182
|
+
From a checkout of [the repository](https://github.com/SmartGlassCube/Smart-Mirror-SDK),
|
|
183
|
+
in `backend/`, with [uv](https://docs.astral.sh/uv/):
|
|
184
|
+
|
|
185
|
+
```bash
|
|
186
|
+
uv sync --extra data --extra assistant # backend/.venv; plain `uv sync` = gestures + server only
|
|
187
|
+
uv run glasscube # also starts ../frontend's Vite dev server (needs npm)
|
|
188
|
+
uv run pytest # no camera needed
|
|
189
|
+
uv run python scripts/build_frontend.py # bundle the UI into glasscube/ui/ before `uv build`
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
| Path | |
|
|
193
|
+
|---|---|
|
|
194
|
+
| `glasscube/app/` | The demo mirror app: `settings.py` (env vars), `pages.py` (widgets + page graph), `main.py` |
|
|
195
|
+
| `glasscube/server/` | `SmartMirrorServer`: WebSocket, camera loop, widget refresh, assistant hooks, the bundled UI |
|
|
196
|
+
| `glasscube/{vision,core,widgets,assistant,content,hal,config}/` | The SDK pieces, usable on their own |
|
|
197
|
+
| `tests/` | One folder per package |
|
|
198
|
+
| `scripts/` | `lan_media_server.py` (share a camera/mic over the LAN), `content_sync_daemon.py`, `build_frontend.py`, checks |
|
|
199
|
+
| `examples/` | `gesture_detector_debug.py`: annotated camera window + gesture feed |
|
|
200
|
+
|
|
201
|
+
## License
|
|
202
|
+
|
|
203
|
+
MIT
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
# glasscube
|
|
2
|
+
|
|
3
|
+
Build gesture-controlled smart mirrors in Python. A webcam watches your hands,
|
|
4
|
+
[MediaPipe](https://developers.google.com/mediapipe) hand tracking turns their movement
|
|
5
|
+
into gestures (pinch, click, scroll, zoom, swipes), and those gestures move between
|
|
6
|
+
pages and drive widgets in a React UI that the package serves itself. An optional voice
|
|
7
|
+
assistant (wake word, Whisper, any OpenAI-compatible LLM, Piper TTS) answers out loud.
|
|
8
|
+
|
|
9
|
+
Everything runs locally: no account, no cloud service required.
|
|
10
|
+
|
|
11
|
+
## Install
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
pip install glasscube # gestures, pages, widgets, the server and its UI
|
|
15
|
+
pip install "glasscube[assistant]" # + the voice assistant
|
|
16
|
+
pip install "glasscube[data]" # + MongoDB providers and .env support
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Python 3.9 or newer (3.10 to 3.12 for the `assistant` extra, whose speech models lag behind), and a webcam.
|
|
20
|
+
|
|
21
|
+
## Run the demo mirror
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
glasscube
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Then open <http://localhost:8080> (full screen it on the mirror's display). Raise a hand
|
|
28
|
+
in front of the camera: the cursor follows your index finger, a two-finger swipe changes
|
|
29
|
+
page, a pinch grabs and moves a widget. Without a camera it still starts, with no gestures.
|
|
30
|
+
|
|
31
|
+
The voice assistant runs when the `assistant` extra is installed. It uses
|
|
32
|
+
[Ollama](https://ollama.com) by default (`ollama pull qwen2.5:0.5b`, then `ollama serve`);
|
|
33
|
+
point it at any OpenAI-compatible endpoint with `GLASSCUBE_LLM_BASE_URL`,
|
|
34
|
+
`GLASSCUBE_LLM_MODEL` and `GLASSCUBE_LLM_API_KEY`, or turn it off with
|
|
35
|
+
`GLASSCUBE_ASSISTANT=0`.
|
|
36
|
+
|
|
37
|
+
Settings are environment variables, also read from a `.env` file in the working directory
|
|
38
|
+
(with the `data` extra). The full list is in
|
|
39
|
+
[.env.example](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/backend/.env.example).
|
|
40
|
+
Runtime files (the downloaded hand model, the assistant's vector store) go in your user
|
|
41
|
+
data folder (`~/.local/share/glasscube`, `%LOCALAPPDATA%\glasscube` or
|
|
42
|
+
`~/Library/Application Support/glasscube`); move them with `GLASSCUBE_DATA_DIR`.
|
|
43
|
+
|
|
44
|
+
## Build your own mirror
|
|
45
|
+
|
|
46
|
+
A mirror is a graph: pages (nodes) hold widgets, and gestures (edges) move between them.
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
import asyncio
|
|
50
|
+
|
|
51
|
+
from glasscube.server import SmartMirrorServer
|
|
52
|
+
from glasscube.widgets import CalendarWidget, TextWidget, WeatherWidget
|
|
53
|
+
from glasscube.widgets.providers import OpenMeteoWeatherProvider
|
|
54
|
+
|
|
55
|
+
weather = WeatherWidget(
|
|
56
|
+
position={"x": 0.1, "y": 0.1}, # fractions of the screen
|
|
57
|
+
data_provider=OpenMeteoWeatherProvider(city="Paris", latitude=48.86, longitude=2.35),
|
|
58
|
+
refresh_interval=600, # seconds
|
|
59
|
+
)
|
|
60
|
+
agenda = CalendarWidget(
|
|
61
|
+
position={"x": 0.6, "y": 0.1},
|
|
62
|
+
# Any callable, sync or async: its result becomes the widget's data.
|
|
63
|
+
data_provider=lambda: {"events": [{"title": "Standup", "start": "2026-11-18T09:00:00"}]},
|
|
64
|
+
refresh_interval=60,
|
|
65
|
+
)
|
|
66
|
+
tips = TextWidget(
|
|
67
|
+
position={"x": 0.5, "y": 0.5},
|
|
68
|
+
content={"tips": ["Swipe to cycle", "Scroll for sections"], "about": ["Made with glasscube"]},
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
graph = {
|
|
72
|
+
"nodes": [
|
|
73
|
+
{"id": "home", "widgets": [weather, agenda]},
|
|
74
|
+
{"id": "tips", "widgets": [tips]},
|
|
75
|
+
],
|
|
76
|
+
"edges": [
|
|
77
|
+
{"from": "home", "to": "tips", "gesture": "two_fingers_swipe_left"},
|
|
78
|
+
{"from": "tips", "to": "home", "gesture": "two_fingers_swipe_right"},
|
|
79
|
+
],
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
asyncio.run(SmartMirrorServer(graph=graph).run()) # UI on http://localhost:8080
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Built-in widgets: `TextWidget`, `WeatherWidget`, `CalendarWidget`, `NewsWidget`,
|
|
86
|
+
`EventsWidget`, `QRGalleryWidget`, `GamesWidget`, `DrawingWidget`. Built-in gestures:
|
|
87
|
+
`pinch_start/move/end`, `left_click`, `right_click`, `scroll_*`, `zoom_*`,
|
|
88
|
+
`swipe_left/right/up/down` and `two_fingers_swipe_left/right/up/down`.
|
|
89
|
+
|
|
90
|
+
### Your own gestures, widgets and reactions
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
from glasscube.core import StateManager
|
|
94
|
+
from glasscube.vision import Gesture, GestureDetector, GestureEvent, SIDES, default_gesture_set
|
|
95
|
+
|
|
96
|
+
class FistGesture(Gesture):
|
|
97
|
+
name = "fist"
|
|
98
|
+
|
|
99
|
+
def __init__(self):
|
|
100
|
+
self._was_fist = {side: False for side in SIDES}
|
|
101
|
+
|
|
102
|
+
def detect(self, context):
|
|
103
|
+
events = []
|
|
104
|
+
for side in SIDES:
|
|
105
|
+
hand = context.hands[side]
|
|
106
|
+
fingers = ("index_up", "middle_up", "ring_up", "pinky_up")
|
|
107
|
+
is_fist = hand.detected and not any(context.signals[f][side] for f in fingers)
|
|
108
|
+
if is_fist and not self._was_fist[side]:
|
|
109
|
+
events.append(GestureEvent(type="fist", hand=side, location=hand.palm_center))
|
|
110
|
+
self._was_fist[side] = is_fist
|
|
111
|
+
return events
|
|
112
|
+
|
|
113
|
+
state = StateManager(graph=graph, event_bindings={
|
|
114
|
+
"fist": {"method": "set_current_node", "kwargs": {"node_id": "home"}}, # a fist goes home
|
|
115
|
+
})
|
|
116
|
+
detector = GestureDetector(gestures=default_gesture_set() + [FistGesture()])
|
|
117
|
+
server = SmartMirrorServer(state_manager=state, gesture_detector=detector)
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
A new widget type needs a React component on the UI side as well: see
|
|
121
|
+
[extending.md](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/docs/extending.md).
|
|
122
|
+
|
|
123
|
+
## Documentation
|
|
124
|
+
|
|
125
|
+
- [Architecture](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/docs/architecture.md): the camera → gestures → state → UI loop, module by module
|
|
126
|
+
- [Extending](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/docs/extending.md): gestures, widgets, pages, bindings
|
|
127
|
+
- [Vision](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/docs/vision.md): the hand signals and gesture tuning
|
|
128
|
+
- [Protocol](https://github.com/SmartGlassCube/Smart-Mirror-SDK/blob/main/docs/protocol.md): the WebSocket messages between backend and UI
|
|
129
|
+
|
|
130
|
+
## Developing the SDK
|
|
131
|
+
|
|
132
|
+
From a checkout of [the repository](https://github.com/SmartGlassCube/Smart-Mirror-SDK),
|
|
133
|
+
in `backend/`, with [uv](https://docs.astral.sh/uv/):
|
|
134
|
+
|
|
135
|
+
```bash
|
|
136
|
+
uv sync --extra data --extra assistant # backend/.venv; plain `uv sync` = gestures + server only
|
|
137
|
+
uv run glasscube # also starts ../frontend's Vite dev server (needs npm)
|
|
138
|
+
uv run pytest # no camera needed
|
|
139
|
+
uv run python scripts/build_frontend.py # bundle the UI into glasscube/ui/ before `uv build`
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
| Path | |
|
|
143
|
+
|---|---|
|
|
144
|
+
| `glasscube/app/` | The demo mirror app: `settings.py` (env vars), `pages.py` (widgets + page graph), `main.py` |
|
|
145
|
+
| `glasscube/server/` | `SmartMirrorServer`: WebSocket, camera loop, widget refresh, assistant hooks, the bundled UI |
|
|
146
|
+
| `glasscube/{vision,core,widgets,assistant,content,hal,config}/` | The SDK pieces, usable on their own |
|
|
147
|
+
| `tests/` | One folder per package |
|
|
148
|
+
| `scripts/` | `lan_media_server.py` (share a camera/mic over the LAN), `content_sync_daemon.py`, `build_frontend.py`, checks |
|
|
149
|
+
| `examples/` | `gesture_detector_debug.py`: annotated camera window + gesture feed |
|
|
150
|
+
|
|
151
|
+
## License
|
|
152
|
+
|
|
153
|
+
MIT
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import logging
|
|
3
|
+
import os
|
|
4
|
+
import shutil
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
# <repo>/frontend, next to backend/. Only exists in a checkout of the repo, not in a pip install.
|
|
8
|
+
DEFAULT_FRONTEND_DIR = Path(__file__).resolve().parents[3] / "frontend"
|
|
9
|
+
|
|
10
|
+
logger = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class FrontendLauncher:
|
|
14
|
+
"""Dev convenience: runs the frontend's Vite dev server (npm install first if needed).
|
|
15
|
+
|
|
16
|
+
The frontend is a separate app; this only saves starting it by hand. It is
|
|
17
|
+
told the backend's WebSocket port through VITE_GLASSCUBE_WS_PORT.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
def __init__(self, frontend_dir=None, host="localhost", port=5173, ws_port=8080):
|
|
21
|
+
self.frontend_dir = Path(frontend_dir) if frontend_dir else DEFAULT_FRONTEND_DIR
|
|
22
|
+
self.host = host
|
|
23
|
+
self.port = port
|
|
24
|
+
self.ws_port = ws_port
|
|
25
|
+
self.process = None
|
|
26
|
+
|
|
27
|
+
def available(self):
|
|
28
|
+
"""True when there's a frontend/ checkout (with its package.json) and npm to run it."""
|
|
29
|
+
return (self.frontend_dir / "package.json").is_file() and shutil.which("npm") is not None
|
|
30
|
+
|
|
31
|
+
async def start(self):
|
|
32
|
+
npm = shutil.which("npm")
|
|
33
|
+
if not self.frontend_dir.exists():
|
|
34
|
+
logger.warning("Skipping frontend launch: directory not found at %s", self.frontend_dir)
|
|
35
|
+
return
|
|
36
|
+
if npm is None:
|
|
37
|
+
logger.warning("Skipping frontend launch: npm is not installed or not in PATH")
|
|
38
|
+
return
|
|
39
|
+
|
|
40
|
+
if not (self.frontend_dir / "node_modules").exists():
|
|
41
|
+
logger.info("Installing frontend dependencies (npm install)...")
|
|
42
|
+
install = await asyncio.create_subprocess_exec(npm, "install", cwd=str(self.frontend_dir))
|
|
43
|
+
if await install.wait() != 0:
|
|
44
|
+
logger.error("npm install failed with exit code %s", install.returncode)
|
|
45
|
+
return
|
|
46
|
+
|
|
47
|
+
logger.info("Starting frontend on http://%s:%s", self.host, self.port)
|
|
48
|
+
self.process = await asyncio.create_subprocess_exec(
|
|
49
|
+
npm, "run", "dev", "--", "--host", self.host, "--port", str(self.port),
|
|
50
|
+
cwd=str(self.frontend_dir),
|
|
51
|
+
# Read by frontend/src/services/useWebSocket.ts
|
|
52
|
+
env={**os.environ, "VITE_GLASSCUBE_WS_PORT": str(self.ws_port)},
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
async def stop(self):
|
|
56
|
+
if self.process is None:
|
|
57
|
+
return
|
|
58
|
+
if os.name == "nt":
|
|
59
|
+
# terminate() would only end npm's cmd wrapper and leave Vite holding the port.
|
|
60
|
+
killer = await asyncio.create_subprocess_exec(
|
|
61
|
+
"taskkill", "/T", "/F", "/PID", str(self.process.pid),
|
|
62
|
+
stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL,
|
|
63
|
+
)
|
|
64
|
+
await killer.wait()
|
|
65
|
+
else:
|
|
66
|
+
self.process.terminate()
|
|
67
|
+
await self.process.wait()
|
|
68
|
+
self.process = None
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""Entry point: the `glasscube` command, or `python -m glasscube`."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import logging
|
|
5
|
+
|
|
6
|
+
from ..core import StateManager
|
|
7
|
+
from ..hal import HAL
|
|
8
|
+
from ..server import SmartMirrorServer
|
|
9
|
+
from ..server.static_ui import bundled_ui_dir
|
|
10
|
+
from .frontend_dev import FrontendLauncher
|
|
11
|
+
from .pages import GESTURE_CONFIG, HOME, build_graph
|
|
12
|
+
from .settings import Settings
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def build_assistant(settings, content_sync=None):
|
|
18
|
+
"""The voice chatbot. Ollama-backed by default, so it runs without a paid API key --
|
|
19
|
+
pull a model first (e.g. `ollama pull qwen2.5:0.5b`) and start `ollama serve`.
|
|
20
|
+
|
|
21
|
+
With `content_sync`, it retrieves from the Mongo-derived Qdrant collection and
|
|
22
|
+
gets the live events/announcements prompt block instead of the seed corpus.
|
|
23
|
+
|
|
24
|
+
None when disabled (GLASSCUBE_ASSISTANT=0) or when the `assistant` extra isn't installed.
|
|
25
|
+
"""
|
|
26
|
+
if not settings.assistant:
|
|
27
|
+
return None
|
|
28
|
+
try:
|
|
29
|
+
return _assistant(settings, content_sync)
|
|
30
|
+
except ImportError as error: # the `assistant` extra isn't installed
|
|
31
|
+
logger.warning(
|
|
32
|
+
"Voice assistant off: %s. Install it with `pip install \"glasscube[assistant]\"`, "
|
|
33
|
+
"or set GLASSCUBE_ASSISTANT=0 to hide this.", error,
|
|
34
|
+
)
|
|
35
|
+
return None
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _assistant(settings, content_sync):
|
|
39
|
+
from ..assistant import Assistant
|
|
40
|
+
from ..assistant.llm import OpenAICompatBackend
|
|
41
|
+
from ..assistant.rag import SEED_CORPUS_DIR
|
|
42
|
+
|
|
43
|
+
hal = HAL(input_device=settings.mic)
|
|
44
|
+
backend = OpenAICompatBackend(
|
|
45
|
+
api_key=settings.llm_api_key,
|
|
46
|
+
base_url=settings.llm_base_url,
|
|
47
|
+
model=settings.llm_model,
|
|
48
|
+
)
|
|
49
|
+
if content_sync is not None:
|
|
50
|
+
return Assistant(
|
|
51
|
+
llm=backend, stt=True, tts=True, hal=hal,
|
|
52
|
+
store=content_sync.indexer.store,
|
|
53
|
+
extra_context=content_sync.prompt_block,
|
|
54
|
+
)
|
|
55
|
+
return Assistant(llm=backend, documents=str(SEED_CORPUS_DIR), stt=True, tts=True, hal=hal)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def build_content_sync(settings):
|
|
59
|
+
"""Club content from the Dashboard repo's Mongo (glasscube.content); None without MONGO_URI."""
|
|
60
|
+
if not settings.content_mongo_uri:
|
|
61
|
+
return None
|
|
62
|
+
from ..content import build_content_sync as build # needs the `data` extra
|
|
63
|
+
|
|
64
|
+
return build()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def build_server(settings):
|
|
68
|
+
content_sync = build_content_sync(settings)
|
|
69
|
+
state_manager = StateManager(
|
|
70
|
+
graph=build_graph(settings, content_sync),
|
|
71
|
+
initial_node_id=HOME,
|
|
72
|
+
gesture_config=GESTURE_CONFIG,
|
|
73
|
+
)
|
|
74
|
+
return SmartMirrorServer(
|
|
75
|
+
state_manager=state_manager,
|
|
76
|
+
camera_id=settings.camera,
|
|
77
|
+
ws_port=settings.ws_port,
|
|
78
|
+
assistant=build_assistant(settings, content_sync),
|
|
79
|
+
content_sync=content_sync,
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
async def run(server, frontend=None):
|
|
84
|
+
if frontend is not None:
|
|
85
|
+
await frontend.start()
|
|
86
|
+
try:
|
|
87
|
+
await server.run()
|
|
88
|
+
finally:
|
|
89
|
+
if frontend is not None:
|
|
90
|
+
await frontend.stop()
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def build_frontend(settings):
|
|
94
|
+
"""The Vite dev server in a checkout with npm; None when the bundled UI is served instead (or turned off)."""
|
|
95
|
+
if not settings.frontend:
|
|
96
|
+
return None
|
|
97
|
+
launcher = FrontendLauncher(port=settings.frontend_port, ws_port=settings.ws_port)
|
|
98
|
+
if launcher.available():
|
|
99
|
+
return launcher
|
|
100
|
+
if bundled_ui_dir() is not None:
|
|
101
|
+
logger.info("Mirror UI on http://localhost:%s", settings.ws_port)
|
|
102
|
+
else:
|
|
103
|
+
logger.warning("No mirror UI: this install has no bundled UI and there's no frontend/ checkout to run.")
|
|
104
|
+
return None
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def main():
|
|
108
|
+
# INFO on the glasscube.* loggers surfaces the voice pipeline's play-by-play
|
|
109
|
+
# (VAD, raw Whisper transcripts, wake-word matches; see docs/voice-pipeline-testing.md).
|
|
110
|
+
# Root stays at WARNING so third-party libraries don't drown it out.
|
|
111
|
+
logging.basicConfig(level=logging.WARNING, format="%(asctime)s %(name)s %(levelname)s %(message)s")
|
|
112
|
+
logging.getLogger("glasscube").setLevel(logging.INFO)
|
|
113
|
+
|
|
114
|
+
settings = Settings.from_env()
|
|
115
|
+
server = build_server(settings)
|
|
116
|
+
frontend = build_frontend(settings)
|
|
117
|
+
try:
|
|
118
|
+
asyncio.run(run(server, frontend))
|
|
119
|
+
except KeyboardInterrupt:
|
|
120
|
+
pass
|