gestureplaykit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gestureplaykit/__init__.py +16 -0
- gestureplaykit/camera.py +62 -0
- gestureplaykit/models.py +38 -0
- gestureplaykit/pictures.py +106 -0
- gestureplaykit/vision.py +215 -0
- gestureplaykit-0.1.0.dist-info/METADATA +132 -0
- gestureplaykit-0.1.0.dist-info/RECORD +10 -0
- gestureplaykit-0.1.0.dist-info/WHEEL +5 -0
- gestureplaykit-0.1.0.dist-info/licenses/LICENSE +21 -0
- gestureplaykit-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""GesturePlayKit: small reusable building blocks for gesture-controlled apps.
|
|
2
|
+
|
|
3
|
+
This package supplies tracking and rendering primitives, not a finished game.
|
|
4
|
+
Applications decide what pinches or mouth gestures mean.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from .camera import Camera
|
|
8
|
+
from .vision import Vision, Hand, Face
|
|
9
|
+
from .pictures import load_picture, show_picture, show_text, show_point
|
|
10
|
+
from .models import download_models
|
|
11
|
+
|
|
12
|
+
__version__ = "0.1.0"
|
|
13
|
+
__all__ = [
|
|
14
|
+
"Camera", "Vision", "Hand", "Face", "load_picture", "show_picture",
|
|
15
|
+
"show_text", "show_point", "download_models",
|
|
16
|
+
]
|
gestureplaykit/camera.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Simple webcam capture and window display."""
|
|
2
|
+
|
|
3
|
+
import cv2
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Camera:
|
|
7
|
+
"""Read webcam frames and show them in a window.
|
|
8
|
+
|
|
9
|
+
Example::
|
|
10
|
+
|
|
11
|
+
with Camera() as camera:
|
|
12
|
+
while (frame := camera.read()) is not None:
|
|
13
|
+
if not camera.show(frame):
|
|
14
|
+
break
|
|
15
|
+
|
|
16
|
+
On Windows, camera indices may be 0, 1, 2, etc.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
def __init__(self, index: int = 0, width: int = 640, height: int = 480,
|
|
20
|
+
mirror: bool = True):
|
|
21
|
+
self.index = index
|
|
22
|
+
self.width = width
|
|
23
|
+
self.height = height
|
|
24
|
+
self.mirror = mirror
|
|
25
|
+
self._capture = None
|
|
26
|
+
|
|
27
|
+
def __enter__(self):
|
|
28
|
+
self._capture = cv2.VideoCapture(self.index)
|
|
29
|
+
if not self._capture.isOpened():
|
|
30
|
+
self._capture.release()
|
|
31
|
+
self._capture = None
|
|
32
|
+
raise RuntimeError(f"Couldn't open camera {self.index}; check permissions or other apps.")
|
|
33
|
+
self._capture.set(cv2.CAP_PROP_FRAME_WIDTH, self.width)
|
|
34
|
+
self._capture.set(cv2.CAP_PROP_FRAME_HEIGHT, self.height)
|
|
35
|
+
return self
|
|
36
|
+
|
|
37
|
+
def read(self):
|
|
38
|
+
"""Return a BGR image (NumPy array) or None when the webcam stops."""
|
|
39
|
+
if self._capture is None:
|
|
40
|
+
raise RuntimeError("Camera is not open. Use 'with Camera() as camera:'.")
|
|
41
|
+
ok, frame = self._capture.read()
|
|
42
|
+
if not ok:
|
|
43
|
+
return None
|
|
44
|
+
return cv2.flip(frame, 1) if self.mirror else frame
|
|
45
|
+
|
|
46
|
+
def show(self, frame, title: str = "GesturePlayKit", quit_key: str = "q") -> bool:
|
|
47
|
+
"""Display frame; return False if Q/quit_key or Escape is pressed."""
|
|
48
|
+
if not isinstance(quit_key, str) or len(quit_key) != 1:
|
|
49
|
+
raise ValueError("quit_key must be a single character")
|
|
50
|
+
cv2.imshow(title, frame)
|
|
51
|
+
key = cv2.waitKey(1) & 0xFF
|
|
52
|
+
return key not in (ord(quit_key.lower()), 27)
|
|
53
|
+
|
|
54
|
+
def close(self):
|
|
55
|
+
if self._capture is not None:
|
|
56
|
+
self._capture.release()
|
|
57
|
+
self._capture = None
|
|
58
|
+
cv2.destroyAllWindows()
|
|
59
|
+
|
|
60
|
+
def __exit__(self, exc_type, exc, tb):
|
|
61
|
+
self.close()
|
|
62
|
+
return False
|
gestureplaykit/models.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Download Google's model files to a local folder before going offline."""
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from urllib.request import Request, urlopen
|
|
4
|
+
|
|
5
|
+
MODELS = {
|
|
6
|
+
"hand_landmarker.task": "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task",
|
|
7
|
+
"face_landmarker.task": "https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task",
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def model_paths(folder="models"):
|
|
12
|
+
folder = Path(folder).expanduser()
|
|
13
|
+
return folder / "hand_landmarker.task", folder / "face_landmarker.task"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def download_models(folder="models", *, overwrite=False):
|
|
17
|
+
"""Download MediaPipe hand/face models once, then run offline thereafter.
|
|
18
|
+
|
|
19
|
+
Returns paths to both files. Does not overwrite existing files by default.
|
|
20
|
+
Model files are from Google, not generated by GesturePlayKit.
|
|
21
|
+
"""
|
|
22
|
+
folder = Path(folder).expanduser()
|
|
23
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
24
|
+
paths = []
|
|
25
|
+
for filename, url in MODELS.items():
|
|
26
|
+
path = folder / filename
|
|
27
|
+
if not path.is_file() or overwrite:
|
|
28
|
+
temp = folder / (filename + ".part")
|
|
29
|
+
try:
|
|
30
|
+
with urlopen(Request(url, headers={"User-Agent":"GesturePlayKit/0.1"}), timeout=30) as response:
|
|
31
|
+
with temp.open("wb") as output:
|
|
32
|
+
while chunk := response.read(1024*1024):
|
|
33
|
+
output.write(chunk)
|
|
34
|
+
temp.replace(path)
|
|
35
|
+
finally:
|
|
36
|
+
temp.unlink(missing_ok=True)
|
|
37
|
+
paths.append(path)
|
|
38
|
+
return tuple(paths)
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""Load images (including URLs) and draw transparent overlays."""
|
|
2
|
+
|
|
3
|
+
from functools import lru_cache
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from urllib.parse import urlsplit
|
|
6
|
+
from urllib.request import Request, urlopen
|
|
7
|
+
import cv2
|
|
8
|
+
import numpy as np
|
|
9
|
+
|
|
10
|
+
MAX_IMAGE_BYTES = 15 * 1024 * 1024
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@lru_cache(maxsize=32)
|
|
14
|
+
def _load_source(source: str):
|
|
15
|
+
"""Cache downloads/decodes so URLs aren't requested on every video frame."""
|
|
16
|
+
parsed = urlsplit(source)
|
|
17
|
+
if parsed.scheme.lower() in {"http", "https"}:
|
|
18
|
+
request = Request(source, headers={"User-Agent": "GesturePlayKit/0.1"})
|
|
19
|
+
with urlopen(request, timeout=10) as response:
|
|
20
|
+
data = response.read(MAX_IMAGE_BYTES + 1)
|
|
21
|
+
if len(data) > MAX_IMAGE_BYTES:
|
|
22
|
+
raise ValueError("Image is too large (>15 MB)")
|
|
23
|
+
image = cv2.imdecode(np.frombuffer(data, dtype=np.uint8), cv2.IMREAD_UNCHANGED)
|
|
24
|
+
else:
|
|
25
|
+
path = Path(source).expanduser()
|
|
26
|
+
if not path.is_file():
|
|
27
|
+
raise FileNotFoundError(f"Picture not found: {path}")
|
|
28
|
+
image = cv2.imread(str(path), cv2.IMREAD_UNCHANGED)
|
|
29
|
+
if image is None:
|
|
30
|
+
raise ValueError(f"Unsupported or unreadable image: {source}")
|
|
31
|
+
if image.ndim == 2:
|
|
32
|
+
image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR)
|
|
33
|
+
return image
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def load_picture(source, size=None):
|
|
37
|
+
"""Read a PNG/JPG from a file path or direct HTTP(S) image URL.
|
|
38
|
+
|
|
39
|
+
Args:
|
|
40
|
+
source: filename, pathlib.Path, or direct image URL.
|
|
41
|
+
size: optional (width, height) tuple. Defaults to original dimensions.
|
|
42
|
+
|
|
43
|
+
Call this once before your camera loop for best performance. An LRU cache
|
|
44
|
+
also avoids re-downloading the same URL repeatedly in a single process.
|
|
45
|
+
"""
|
|
46
|
+
image = _load_source(str(source))
|
|
47
|
+
if size is None:
|
|
48
|
+
return image.copy()
|
|
49
|
+
width, height = _validate_size(size)
|
|
50
|
+
return cv2.resize(image, (width, height), interpolation=cv2.INTER_AREA)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _validate_size(size):
|
|
54
|
+
if len(size) != 2:
|
|
55
|
+
raise ValueError("size must be (width, height)")
|
|
56
|
+
width, height = int(size[0]), int(size[1])
|
|
57
|
+
if width <= 0 or height <= 0:
|
|
58
|
+
raise ValueError("size dimensions must be positive")
|
|
59
|
+
return width, height
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def show_picture(frame, picture, at, size=None):
|
|
63
|
+
"""Overlay a picture at pixel center (x, y), clipping to frame edges.
|
|
64
|
+
|
|
65
|
+
Transparency (PNG alpha) is preserved. Mutates and returns `frame`.
|
|
66
|
+
`picture` can be a loaded image array, a local filename, or a URL.
|
|
67
|
+
"""
|
|
68
|
+
if isinstance(picture, (str, Path)):
|
|
69
|
+
picture = load_picture(picture, size=size)
|
|
70
|
+
elif size is not None:
|
|
71
|
+
width, height = _validate_size(size)
|
|
72
|
+
picture = cv2.resize(picture, (width, height), interpolation=cv2.INTER_AREA)
|
|
73
|
+
if not isinstance(picture, np.ndarray) or picture.ndim != 3:
|
|
74
|
+
raise ValueError("picture must be a 3- or 4-channel image")
|
|
75
|
+
if picture.shape[2] not in (3, 4):
|
|
76
|
+
raise ValueError("picture must have 3 (BGR) or 4 (BGRA) channels")
|
|
77
|
+
h, w = picture.shape[:2]
|
|
78
|
+
fh, fw = frame.shape[:2]
|
|
79
|
+
x, y = int(at[0]) - w // 2, int(at[1]) - h // 2
|
|
80
|
+
left, top = max(0, x), max(0, y)
|
|
81
|
+
right, bottom = min(fw, x + w), min(fh, y + h)
|
|
82
|
+
if left >= right or top >= bottom:
|
|
83
|
+
return frame
|
|
84
|
+
patch = picture[top-y:bottom-y, left-x:right-x]
|
|
85
|
+
background = frame[top:bottom, left:right]
|
|
86
|
+
if patch.shape[2] == 4:
|
|
87
|
+
a = patch[:, :, 3:4].astype(np.float32) / 255.0
|
|
88
|
+
background[:] = (a * patch[:, :, :3].astype(np.float32)
|
|
89
|
+
+ (1-a) * background.astype(np.float32)).astype(np.uint8)
|
|
90
|
+
else:
|
|
91
|
+
background[:] = patch[:, :, :3]
|
|
92
|
+
return frame
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def show_text(frame, text, at=(15, 35), color=(255, 255, 255),
|
|
96
|
+
scale=0.7, thickness=2):
|
|
97
|
+
"""Draw text with OpenCV BGR colors; mutates/returns the frame."""
|
|
98
|
+
cv2.putText(frame, str(text), tuple(map(int, at)), cv2.FONT_HERSHEY_SIMPLEX,
|
|
99
|
+
float(scale), color, int(thickness), cv2.LINE_AA)
|
|
100
|
+
return frame
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def show_point(frame, at, color=(0, 255, 255), radius=7):
|
|
104
|
+
"""Mark a point on screen, useful when exploring hand landmarks."""
|
|
105
|
+
cv2.circle(frame, tuple(map(int, at)), int(radius), color, -1)
|
|
106
|
+
return frame
|
gestureplaykit/vision.py
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
"""MediaPipe VIDEO-mode tracking with simple frame-to-frame gesture events."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import time
|
|
6
|
+
import cv2
|
|
7
|
+
import numpy as np
|
|
8
|
+
|
|
9
|
+
from .models import model_paths
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _distance(a, b):
|
|
13
|
+
return float(np.hypot(a[0] - b[0], a[1] - b[1]))
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class Hand:
|
|
18
|
+
"""One tracked hand for one frame. Landmark indices 0..20."""
|
|
19
|
+
points: tuple[tuple[int, int], ...]
|
|
20
|
+
pinching: bool
|
|
21
|
+
pinch_started: bool
|
|
22
|
+
pinch_ended: bool
|
|
23
|
+
double_pinched: bool
|
|
24
|
+
|
|
25
|
+
def landmark(self, number: int) -> tuple[int, int]:
|
|
26
|
+
"""Get pixel (x, y), e.g. landmark(8) is the index fingertip."""
|
|
27
|
+
return self.points[number]
|
|
28
|
+
|
|
29
|
+
def distance(self, a: int, b: int) -> float:
|
|
30
|
+
return _distance(self.landmark(a), self.landmark(b))
|
|
31
|
+
|
|
32
|
+
def touching(self, a: int, b: int, ratio: float = 0.35) -> bool:
|
|
33
|
+
"""Estimate whether points are touching, relative to palm width."""
|
|
34
|
+
palm_width = self.distance(5, 17)
|
|
35
|
+
return palm_width > 0 and self.distance(a, b) < palm_width * ratio
|
|
36
|
+
|
|
37
|
+
@property
|
|
38
|
+
def pinch_center(self) -> tuple[int, int]:
|
|
39
|
+
a, b = self.landmark(4), self.landmark(8)
|
|
40
|
+
return ((a[0] + b[0]) // 2, (a[1] + b[1]) // 2)
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def palm_center(self) -> tuple[int, int]:
|
|
44
|
+
five = [self.landmark(i) for i in (0, 5, 9, 13, 17)]
|
|
45
|
+
return (int(sum(p[0] for p in five)/5), int(sum(p[1] for p in five)/5))
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True)
|
|
49
|
+
class Face:
|
|
50
|
+
"""One tracked face. Mouth state is estimated from lip landmarks."""
|
|
51
|
+
points: tuple[tuple[int, int], ...]
|
|
52
|
+
mouth_open: bool
|
|
53
|
+
mouth_just_opened: bool
|
|
54
|
+
mouth_just_closed: bool
|
|
55
|
+
mouth_ratio: float
|
|
56
|
+
|
|
57
|
+
def landmark(self, number: int) -> tuple[int, int]:
|
|
58
|
+
return self.points[number]
|
|
59
|
+
|
|
60
|
+
@property
|
|
61
|
+
def mouth_center(self) -> tuple[int, int]:
|
|
62
|
+
a, b = self.landmark(13), self.landmark(14)
|
|
63
|
+
return ((a[0]+b[0])//2, (a[1]+b[1])//2)
|
|
64
|
+
|
|
65
|
+
@property
|
|
66
|
+
def mouth_width(self) -> float:
|
|
67
|
+
return _distance(self.landmark(61), self.landmark(291))
|
|
68
|
+
|
|
69
|
+
def near_mouth(self, position, max_distance: float = 70) -> bool:
|
|
70
|
+
"""Check whether a screen pixel point is near the mouth center."""
|
|
71
|
+
return _distance(self.mouth_center, position) <= max_distance
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class Vision:
|
|
75
|
+
"""Get easy hand and mouth information from a live webcam frame.
|
|
76
|
+
|
|
77
|
+
Example::
|
|
78
|
+
|
|
79
|
+
with Vision() as vision:
|
|
80
|
+
vision.update(frame)
|
|
81
|
+
if vision.hand and vision.hand.pinching:
|
|
82
|
+
print("Pinching!", vision.hand.pinch_center)
|
|
83
|
+
|
|
84
|
+
Models are downloaded *separately* via download_models(). That lets your
|
|
85
|
+
application work fully offline after the initial download.
|
|
86
|
+
"""
|
|
87
|
+
|
|
88
|
+
def __init__(self, models_dir="models", hands=True, faces=True,
|
|
89
|
+
max_hands=2, pinch_start=0.35, pinch_release=0.50,
|
|
90
|
+
double_pinch_window=0.60, mouth_open_ratio=0.20,
|
|
91
|
+
mouth_closed_ratio=0.12):
|
|
92
|
+
self.models_dir = Path(models_dir)
|
|
93
|
+
self.detect_hands = bool(hands)
|
|
94
|
+
self.detect_faces = bool(faces)
|
|
95
|
+
self.max_hands = max_hands
|
|
96
|
+
self.pinch_start = pinch_start
|
|
97
|
+
self.pinch_release = pinch_release
|
|
98
|
+
self.double_pinch_window = double_pinch_window
|
|
99
|
+
self.mouth_open_ratio = mouth_open_ratio
|
|
100
|
+
self.mouth_closed_ratio = mouth_closed_ratio
|
|
101
|
+
self.hands: list[Hand] = []
|
|
102
|
+
self.hand: Hand | None = None
|
|
103
|
+
self.face: Face | None = None
|
|
104
|
+
self._hands_tracker = None
|
|
105
|
+
self._faces_tracker = None
|
|
106
|
+
self._hand_history = {}
|
|
107
|
+
self._face_open_before = False
|
|
108
|
+
self._start = None
|
|
109
|
+
self._last_timestamp = -1
|
|
110
|
+
|
|
111
|
+
def __enter__(self):
|
|
112
|
+
if not (self.detect_hands or self.detect_faces):
|
|
113
|
+
raise ValueError("Enable hand and/or face tracking")
|
|
114
|
+
try:
|
|
115
|
+
import mediapipe as mp
|
|
116
|
+
except ImportError as exc:
|
|
117
|
+
raise ImportError("Install GesturePlayKit dependencies: python -m pip install -e .") from exc
|
|
118
|
+
self._mp = mp
|
|
119
|
+
hand_path, face_path = model_paths(self.models_dir)
|
|
120
|
+
for enabled, path in ((self.detect_hands, hand_path), (self.detect_faces, face_path)):
|
|
121
|
+
if enabled and not path.is_file():
|
|
122
|
+
raise FileNotFoundError(
|
|
123
|
+
f"Model not found: {path}. First run: "
|
|
124
|
+
"from gestureplaykit import download_models; download_models('models')"
|
|
125
|
+
)
|
|
126
|
+
base = mp.tasks.BaseOptions
|
|
127
|
+
mode = mp.tasks.vision.RunningMode.VIDEO
|
|
128
|
+
try:
|
|
129
|
+
if self.detect_hands:
|
|
130
|
+
opts = mp.tasks.vision.HandLandmarkerOptions(
|
|
131
|
+
base_options=base(model_asset_path=str(hand_path)),
|
|
132
|
+
running_mode=mode, num_hands=self.max_hands)
|
|
133
|
+
self._hands_tracker = mp.tasks.vision.HandLandmarker.create_from_options(opts)
|
|
134
|
+
if self.detect_faces:
|
|
135
|
+
opts = mp.tasks.vision.FaceLandmarkerOptions(
|
|
136
|
+
base_options=base(model_asset_path=str(face_path)),
|
|
137
|
+
running_mode=mode, num_faces=1)
|
|
138
|
+
self._faces_tracker = mp.tasks.vision.FaceLandmarker.create_from_options(opts)
|
|
139
|
+
except Exception:
|
|
140
|
+
self.close()
|
|
141
|
+
raise
|
|
142
|
+
self._start = time.monotonic()
|
|
143
|
+
self._last_timestamp = -1
|
|
144
|
+
return self
|
|
145
|
+
|
|
146
|
+
def update(self, frame):
|
|
147
|
+
"""Process a BGR webcam frame and update .hand, .hands and .face.
|
|
148
|
+
|
|
149
|
+
Use the returned Vision object or read its attributes directly.
|
|
150
|
+
"""
|
|
151
|
+
if self._start is None:
|
|
152
|
+
raise RuntimeError("Vision not started; use 'with Vision() as vision:'.")
|
|
153
|
+
height, width = frame.shape[:2]
|
|
154
|
+
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
|
155
|
+
mp_image = self._mp.Image(image_format=self._mp.ImageFormat.SRGB, data=rgb)
|
|
156
|
+
now = time.monotonic()
|
|
157
|
+
timestamp = max(self._last_timestamp + 1,
|
|
158
|
+
int((now - self._start) * 1000))
|
|
159
|
+
self._last_timestamp = timestamp
|
|
160
|
+
|
|
161
|
+
self.hands = []
|
|
162
|
+
if self._hands_tracker is not None:
|
|
163
|
+
result = self._hands_tracker.detect_for_video(mp_image, timestamp)
|
|
164
|
+
for index, landmarks in enumerate(result.hand_landmarks):
|
|
165
|
+
points = tuple((round(p.x * width), round(p.y * height)) for p in landmarks)
|
|
166
|
+
history = self._hand_history.setdefault(index, {"down": False, "last_start": -100.0})
|
|
167
|
+
palm_width = _distance(points[5], points[17])
|
|
168
|
+
threshold = self.pinch_release if history["down"] else self.pinch_start
|
|
169
|
+
down = palm_width > 0 and _distance(points[4], points[8]) < palm_width * threshold
|
|
170
|
+
started = down and not history["down"]
|
|
171
|
+
ended = not down and history["down"]
|
|
172
|
+
double = False
|
|
173
|
+
if started:
|
|
174
|
+
elapsed = now - history["last_start"]
|
|
175
|
+
double = 0.08 <= elapsed <= self.double_pinch_window
|
|
176
|
+
history["last_start"] = -100.0 if double else now
|
|
177
|
+
history["down"] = down
|
|
178
|
+
self.hands.append(Hand(points, down, started, ended, double))
|
|
179
|
+
for index in list(self._hand_history):
|
|
180
|
+
if index >= len(self.hands):
|
|
181
|
+
self._hand_history[index]["down"] = False
|
|
182
|
+
self.hand = self.hands[0] if self.hands else None
|
|
183
|
+
|
|
184
|
+
self.face = None
|
|
185
|
+
if self._faces_tracker is not None:
|
|
186
|
+
result = self._faces_tracker.detect_for_video(mp_image, timestamp)
|
|
187
|
+
if result.face_landmarks:
|
|
188
|
+
points = tuple((round(p.x * width), round(p.y * height))
|
|
189
|
+
for p in result.face_landmarks[0])
|
|
190
|
+
width_px = _distance(points[61], points[291])
|
|
191
|
+
ratio = (_distance(points[13], points[14]) / width_px) if width_px > 0 else 0.0
|
|
192
|
+
is_open = self._face_open_before
|
|
193
|
+
if ratio >= self.mouth_open_ratio:
|
|
194
|
+
is_open = True
|
|
195
|
+
elif ratio <= self.mouth_closed_ratio:
|
|
196
|
+
is_open = False
|
|
197
|
+
self.face = Face(points, is_open,
|
|
198
|
+
is_open and not self._face_open_before,
|
|
199
|
+
not is_open and self._face_open_before, ratio)
|
|
200
|
+
self._face_open_before = is_open
|
|
201
|
+
else:
|
|
202
|
+
self._face_open_before = False
|
|
203
|
+
return self
|
|
204
|
+
|
|
205
|
+
def close(self):
|
|
206
|
+
for name in ("_hands_tracker", "_faces_tracker"):
|
|
207
|
+
tracker = getattr(self, name)
|
|
208
|
+
if tracker is not None:
|
|
209
|
+
tracker.close()
|
|
210
|
+
setattr(self, name, None)
|
|
211
|
+
self._start = None
|
|
212
|
+
|
|
213
|
+
def __exit__(self, exc_type, exc, tb):
|
|
214
|
+
self.close()
|
|
215
|
+
return False
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: gestureplaykit
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Friendly webcam, hand gesture, facial landmark and picture-overlay tools for creative Python projects
|
|
5
|
+
Author: GesturePlayKit contributors
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Keywords: gestures,webcam,hand tracking,mediapipe,AR,computer vision
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Classifier: Topic :: Multimedia :: Graphics
|
|
11
|
+
Requires-Python: >=3.10
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE
|
|
14
|
+
Requires-Dist: mediapipe>=0.10.20
|
|
15
|
+
Requires-Dist: numpy<3,>=1.26
|
|
16
|
+
Requires-Dist: opencv-contrib-python>=4.8
|
|
17
|
+
Provides-Extra: test
|
|
18
|
+
Requires-Dist: pytest>=8; extra == "test"
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# GesturePlayKit 🖐️
|
|
22
|
+
|
|
23
|
+
**Write interactive webcam apps in plain Python without rewriting MediaPipe setup or OpenCV image-overlays each time.**
|
|
24
|
+
|
|
25
|
+
GesturePlayKit is an **open-source-ready library scaffold** (MIT licensed). It is **not published on PyPI or GitHub yet**. Its job is to supply reusable webcam, gesture and image primitives—not your application rules.
|
|
26
|
+
|
|
27
|
+
## Installation (Windows PowerShell)
|
|
28
|
+
|
|
29
|
+
```powershell
|
|
30
|
+
cd gestureplaykit
|
|
31
|
+
python -m pip install -e .
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
One-time download of Google's hand/face AI model files:
|
|
35
|
+
|
|
36
|
+
```powershell
|
|
37
|
+
python -c "from gestureplaykit import download_models; print(download_models('models'))"
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
After this download, **tracking works offline**. If you use pictures from internet URLs, download those in advance too for offline use.
|
|
41
|
+
|
|
42
|
+
## A tiny program
|
|
43
|
+
|
|
44
|
+
Save this as `try_it.py` in the project folder:
|
|
45
|
+
|
|
46
|
+
```python
|
|
47
|
+
from gestureplaykit import Camera, Vision, load_picture, show_picture, show_text
|
|
48
|
+
|
|
49
|
+
chip = load_picture("examples/chip.png", size=(100, 100))
|
|
50
|
+
|
|
51
|
+
with Camera() as camera, Vision(models_dir="models") as vision:
|
|
52
|
+
while True:
|
|
53
|
+
frame = camera.read()
|
|
54
|
+
if frame is None:
|
|
55
|
+
break
|
|
56
|
+
vision.update(frame)
|
|
57
|
+
|
|
58
|
+
hand = vision.hand
|
|
59
|
+
face = vision.face
|
|
60
|
+
if hand and hand.pinching:
|
|
61
|
+
show_picture(frame, chip, at=hand.pinch_center)
|
|
62
|
+
if hand and hand.double_pinched:
|
|
63
|
+
print("DOUBLE PINCH!")
|
|
64
|
+
if face and face.mouth_open:
|
|
65
|
+
show_text(frame, "MOUTH OPEN", at=(15, 70))
|
|
66
|
+
|
|
67
|
+
if not camera.show(frame, title="My Project"):
|
|
68
|
+
break
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Press **Q** or Escape to exit.
|
|
72
|
+
|
|
73
|
+
## Quick reference
|
|
74
|
+
|
|
75
|
+
| What you want | What to write |
|
|
76
|
+
|---|---|
|
|
77
|
+
| Open webcam | `with Camera() as camera:` |
|
|
78
|
+
| Get video frame | `frame = camera.read()` |
|
|
79
|
+
| Track hand and face | `vision.update(frame)` |
|
|
80
|
+
| First hand found? | `if vision.hand:` |
|
|
81
|
+
| Are thumb/index touching? | `if vision.hand.pinching:` |
|
|
82
|
+
| Pinch began **this frame**? | `if vision.hand.pinch_started:` |
|
|
83
|
+
| Double pinch **this frame**? | `if vision.hand.double_pinched:` |
|
|
84
|
+
| Thumb tip coordinate | `hand.landmark(4)` |
|
|
85
|
+
| Index tip coordinate | `hand.landmark(8)` |
|
|
86
|
+
| Center between fingertips | `hand.pinch_center` |
|
|
87
|
+
| Palm center | `hand.palm_center` |
|
|
88
|
+
| Any landmarks touching? | `hand.touching(4, 12)` |
|
|
89
|
+
| Is mouth open? | `if face and face.mouth_open:` |
|
|
90
|
+
| Mouth just closed? | `if face and face.mouth_just_closed:` |
|
|
91
|
+
| Mouth screen position | `face.mouth_center` |
|
|
92
|
+
| Is pixel point near mouth? | `face.near_mouth((x, y), max_distance=70)` |
|
|
93
|
+
| Local / web picture | `chip = load_picture("image.png")` or `load_picture("https://example.com/picture.png")` |
|
|
94
|
+
| Draw picture | `show_picture(frame, chip, at=(x, y))` |
|
|
95
|
+
| Draw text | `show_text(frame, "Hello!", at=(10, 30))` |
|
|
96
|
+
| Display webcam | `if not camera.show(frame): break` |
|
|
97
|
+
|
|
98
|
+
## Your own app logic
|
|
99
|
+
|
|
100
|
+
The library **does not** decide whether someone ate a chip, award points, freeze objects, manage skins, implement a shop, or save game state. Those are the interesting program features for you to design. Look at `examples/03_build_your_own_game.py` for an intentionally unfinished starter.
|
|
101
|
+
|
|
102
|
+
## Images
|
|
103
|
+
|
|
104
|
+
`load_picture(...)` accepts local PNG/JPG or a direct HTTP(S) image URL. It downloads and decodes the URL only once per unique URL in the current Python process. For speed, load before entering the camera loop. Transparent PNGs are supported; pictures clip safely when partly offscreen.
|
|
105
|
+
|
|
106
|
+
The included `examples/chip.png` is a simple bundled example graphic; change it however you like.
|
|
107
|
+
|
|
108
|
+
## Models and privacy
|
|
109
|
+
|
|
110
|
+
Hand and face detection runs **on your computer**, using Google MediaPipe Tasks. The library does **not upload webcam frames**. Calling `download_models()` or `load_picture("https://...")` connects to those respective web hosts. You may need to adjust gesture ratios in different lighting, with occlusions or different camera angles. The API only estimates apparent fingertip contact and mouth opening—it cannot prove actual touch or eating.
|
|
111
|
+
|
|
112
|
+
## Build your own published package
|
|
113
|
+
|
|
114
|
+
```powershell
|
|
115
|
+
python -m pip install build
|
|
116
|
+
python -m build
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
The generated `.whl` and `.tar.gz` appear under `dist/`. Read Python Packaging's publishing guidance before releasing to PyPI. You can also place this folder in a **public GitHub repository**, including `LICENSE` and `README.md`.
|
|
120
|
+
|
|
121
|
+
## Attribution and competition honesty
|
|
122
|
+
|
|
123
|
+
This initial GesturePlayKit implementation and parts of its documentation were produced with AI assistance. Record that attribution if you reuse it. Open-source libraries may be cited, but your competition project's **significant code must still be your own work**. It is not enough to repackage an AI-written application behind a library. Build your original game rules, state transitions, interface, and other meaningful software yourself.
|
|
124
|
+
|
|
125
|
+
## Testing
|
|
126
|
+
|
|
127
|
+
```powershell
|
|
128
|
+
python -m pip install pytest
|
|
129
|
+
python -m pytest -q
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Camera/model integration requires testing on your own hardware. Automated tests cover image overlays and public helper behavior using fake tracking results.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
gestureplaykit/__init__.py,sha256=kNKnpbySzsFxpK0elMj96y3oUTiU4AoMv7HvGg4TeS0,553
|
|
2
|
+
gestureplaykit/camera.py,sha256=O8RriFqpciOkdM_EHU0s5KwfCAsopKvjLaizV4SYxhU,2094
|
|
3
|
+
gestureplaykit/models.py,sha256=s0QDW-dJqTYIOTUJeofzvjbPovWKG7ytq4EooNOX4og,1600
|
|
4
|
+
gestureplaykit/pictures.py,sha256=EsvaSVxmDQK-RLjB4mo4RbATkyd_kutEYMVIwlacFTw,4144
|
|
5
|
+
gestureplaykit/vision.py,sha256=Xu7qTzYG9x8Is4zAxoef1Wzc2eY5RFNZtktBdb9w68I,8749
|
|
6
|
+
gestureplaykit-0.1.0.dist-info/licenses/LICENSE,sha256=36ne0tpDukEgdwRdih0Onk4zgOUZfAFtPPaxNsBuSLM,1084
|
|
7
|
+
gestureplaykit-0.1.0.dist-info/METADATA,sha256=yYZAtxMiUvBQPXdKf9G4meHsALob-PNIaD8eSh9Pq6I,5749
|
|
8
|
+
gestureplaykit-0.1.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
9
|
+
gestureplaykit-0.1.0.dist-info/top_level.txt,sha256=rBNVFma5b0r_CxzwnUAwqTx_hCHfR_T0I_iTlnGO6_w,15
|
|
10
|
+
gestureplaykit-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 GesturePlayKit contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
gestureplaykit
|