gesto 0.1.0__py3-none-any.whl

gesto/__init__.py

@@ -0,0 +1,71 @@
+"""
+gesto — train and run gesture recognition models from Gesto Labeller datasets.
+
+Two model types:
+    static    single frame; the gesture is a held shape or posture
+    sequence  a window of frames; the gesture is a motion
+
+Five regions: hands_one, hands_two, pose, legs, full.
+
+Quick start (Python):
+
+    import gesto
+
+    run = gesto.train("./gesto_projects/signs", region="hands_one",
+                      mode="static")
+    gesto.detect("static", "hands_one")
+
+Quick start (command line):
+
+    gesto train static hands_one ./gesto_projects/signs
+    gesto detect static hands_one
+
+Models are saved under artifacts/<mode>/<region>/, versioned so a new run never
+overwrites an old one (pose, pose_2, pose_3, ...).
+"""
+
+from __future__ import annotations
+
+__version__ = "0.1.0"
+
+from .regions import REGION_KEYS, REGION_INFO, feature_dim, extract, normalize
+
+__all__ = [
+    "__version__",
+    "REGION_KEYS",
+    "REGION_INFO",
+    "feature_dim",
+    "extract",
+    "normalize",
+    "train",
+    "detect",
+    "Predictor",
+    "artifacts",
+    "data",
+]
+
+
+def train(project_dir, region: str, mode: str = "static", **kwargs):
+    """Train a model. See gesto.train.train for the full signature."""
+    from .train import train as _train
+    return _train(project_dir, region, mode, **kwargs)
+
+
+def detect(mode: str, region: str, **kwargs):
+    """Run live detection. See gesto.detect.run for the full signature."""
+    from .detect import run as _run
+    return _run(mode, region, **kwargs)
+
+
+def Predictor(*args, **kwargs):  # noqa: N802 - re-exported class-like helper
+    """Load a trained model for programmatic use."""
+    from .detect import Predictor as _Predictor
+    return _Predictor(*args, **kwargs)
+
+
+def __getattr__(name: str):
+    # lazy submodule access: gesto.artifacts / gesto.data without import cost
+    if name in ("artifacts", "data"):
+        import importlib
+        return importlib.import_module(f".{name}", __name__)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

gesto/artifacts.py

@@ -0,0 +1,132 @@
+"""
+Artifact storage.
+
+Everything a training run produces lives under one root folder, split by mode
+and then region:
+
+    artifacts/
+        static/
+            pose/            <- first static pose model
+                model.keras
+                labels.json
+            pose_2/          <- next one doesn't overwrite; it's versioned
+            hands_one/
+        sequence/
+            pose/
+            legs/
+
+`new_run()` never overwrites: if `artifacts/static/pose` exists it returns
+`pose_2`, then `pose_3`, and so on. `resolve()` finds the newest version when
+you don't name one explicitly, so `gesto detect static pose` picks up the model
+you just trained.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+MODES = ("static", "sequence")
+DEFAULT_ROOT = "artifacts"
+
+_VERSION_RE = re.compile(r"^(?P<base>.+?)(?:_(?P<n>\d+))?$")
+
+
+def _check_mode(mode: str) -> None:
+    if mode not in MODES:
+        raise ValueError(f"mode must be one of {MODES}, got {mode!r}")
+
+
+def mode_dir(root: str | Path, mode: str) -> Path:
+    _check_mode(mode)
+    return Path(root) / mode
+
+
+def _version_of(name: str, base: str) -> int:
+    """Return the version number a folder name represents for `base`.
+
+    "pose" -> 1, "pose_2" -> 2, anything else -> 0 (not a version of base).
+    """
+    m = _VERSION_RE.match(name)
+    if not m or m.group("base") != base:
+        return 0
+    n = m.group("n")
+    return int(n) if n else 1
+
+
+def list_runs(root: str | Path, mode: str, region: str) -> list[Path]:
+    """Every run folder for this mode+region, oldest version first."""
+    d = mode_dir(root, mode)
+    if not d.exists():
+        return []
+    runs = [(p, _version_of(p.name, region)) for p in d.iterdir() if p.is_dir()]
+    runs = [(p, v) for p, v in runs if v > 0]
+    runs.sort(key=lambda t: t[1])
+    return [p for p, _ in runs]
+
+
+def new_run(root: str | Path, mode: str, region: str) -> Path:
+    """Create and return a fresh run folder, versioning instead of overwriting.
+
+    First call -> artifacts/<mode>/<region>
+    Later calls -> artifacts/<mode>/<region>_2, _3, ...
+    """
+    d = mode_dir(root, mode)
+    d.mkdir(parents=True, exist_ok=True)
+    existing = list_runs(root, mode, region)
+    if not existing:
+        run = d / region
+    else:
+        highest = max(_version_of(p.name, region) for p in existing)
+        run = d / f"{region}_{highest + 1}"
+    run.mkdir(parents=True, exist_ok=False)
+    return run
+
+
+def resolve(root: str | Path, mode: str, region: str,
+            version: int | str | None = None) -> Path:
+    """Locate an existing run folder.
+
+    version=None  -> newest version
+    version=2     -> artifacts/<mode>/<region>_2
+    version="pose_2" (a folder name) -> that exact folder
+    """
+    d = mode_dir(root, mode)
+    if isinstance(version, str) and not version.isdigit():
+        run = d / version
+        if not run.exists():
+            raise FileNotFoundError(f"No such run: {run}")
+        return run
+
+    runs = list_runs(root, mode, region)
+    if not runs:
+        raise FileNotFoundError(
+            f"No {mode} model for region {region!r} under {d}. "
+            f"Train one first: gesto train {mode} {region} <project_dir>")
+    if version is None:
+        return runs[-1]                      # newest
+    want = int(version)
+    for p in runs:
+        if _version_of(p.name, region) == want:
+            return p
+    have = ", ".join(p.name for p in runs)
+    raise FileNotFoundError(
+        f"No version {want} for {mode}/{region}. Available: {have}")
+
+
+def save_meta(run: Path, meta: dict) -> Path:
+    path = Path(run) / "labels.json"
+    path.write_text(json.dumps(meta, indent=2), encoding="utf-8")
+    return path
+
+
+def load_meta(run: Path) -> dict:
+    path = Path(run) / "labels.json"
+    if not path.exists():
+        raise FileNotFoundError(f"No labels.json in {run}")
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def model_path(run: Path) -> Path:
+    return Path(run) / "model.keras"

gesto/cli.py

@@ -0,0 +1,195 @@
+"""
+Command line interface.
+
+    gesto train static pose ./gesto_projects/postures
+    gesto train sequence hands_one ./gesto_projects/signs --seq-len 30
+    gesto detect static pose
+    gesto detect sequence hands_one --source clip.mp4 --version 2
+    gesto list
+    gesto inspect ./gesto_projects/signs
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from . import __version__, artifacts
+from .regions import REGION_KEYS
+
+
+def _add_common(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--root", default=artifacts.DEFAULT_ROOT,
+                   help="artifact root folder (default: artifacts)")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="gesto",
+        description="Train and run gesture models from Gesto Labeller datasets.")
+    parser.add_argument("--version", action="version",
+                        version=f"gesto {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    # ---- train ----
+    t = sub.add_parser("train", help="train a model")
+    t.add_argument("mode", choices=artifacts.MODES)
+    t.add_argument("region", choices=REGION_KEYS)
+    t.add_argument("project_dir", help="Gesto project folder (Copy path button)")
+    t.add_argument("--seq-len", type=int, default=30,
+                   help="frames per sequence (sequence mode, default 30)")
+    t.add_argument("--epochs", type=int, default=None)
+    t.add_argument("--batch-size", type=int, default=16)
+    t.add_argument("--val-split", type=float, default=0.2)
+    t.add_argument("--small", dest="small", action="store_true", default=None,
+                   help="force the lighter model")
+    t.add_argument("--large", dest="small", action="store_false",
+                   help="force the full-size model")
+    t.add_argument("--raw", action="store_true",
+                   help="data was captured with Gesto 'Normalise' unchecked")
+    _add_common(t)
+
+    # ---- detect ----
+    d = sub.add_parser("detect", help="run live detection")
+    d.add_argument("mode", choices=artifacts.MODES)
+    d.add_argument("region", choices=REGION_KEYS)
+    d.add_argument("--source", default="0", help="webcam index or video path")
+    d.add_argument("--version", dest="run_version", default=None,
+                   help="model version (default: newest)")
+    d.add_argument("--threshold", type=float, default=0.5)
+    d.add_argument("--smooth", type=int, default=5,
+                   help="frames of agreement before committing a label")
+    d.add_argument("--width", type=int, default=960, help="display width")
+    d.add_argument("--no-mirror", dest="mirror", action="store_false",
+                   default=None, help="don't mirror the webcam")
+    _add_common(d)
+
+    # ---- list ----
+    ls = sub.add_parser("list", help="show trained models")
+    _add_common(ls)
+
+    # ---- inspect ----
+    i = sub.add_parser("inspect", help="summarise a Gesto project's data")
+    i.add_argument("project_dir")
+    i.add_argument("--seq-len", type=int, default=30)
+
+    return parser
+
+
+def _cmd_train(args) -> int:
+    from .train import train
+    run = train(args.project_dir, args.region, args.mode, root=args.root,
+                seq_len=args.seq_len, epochs=args.epochs,
+                batch_size=args.batch_size, val_split=args.val_split,
+                small=args.small, normalized=not args.raw)
+    print(f"\nDetect with:  gesto detect {args.mode} {args.region}"
+          f"{'' if args.root == artifacts.DEFAULT_ROOT else f' --root {args.root}'}")
+    return 0
+
+
+def _cmd_detect(args) -> int:
+    from .detect import run as run_detect
+    version = args.run_version
+    if version is not None and str(version).isdigit():
+        version = int(version)
+    run_detect(args.mode, args.region, root=args.root, version=version,
+               source=args.source, threshold=args.threshold,
+               smooth=args.smooth, width=args.width, mirror=args.mirror)
+    return 0
+
+
+def _cmd_list(args) -> int:
+    root = Path(args.root)
+    if not root.exists():
+        print(f"No artifacts yet under {root}/")
+        return 0
+    found = False
+    for mode in artifacts.MODES:
+        d = root / mode
+        if not d.exists():
+            continue
+        runs = sorted(p for p in d.iterdir() if p.is_dir())
+        if not runs:
+            continue
+        found = True
+        print(f"{mode}/")
+        for run in runs:
+            try:
+                meta = artifacts.load_meta(run)
+                classes = ", ".join(meta.get("labels", []))
+                extra = (f", seq_len={meta['seq_len']}"
+                         if "seq_len" in meta else "")
+                print(f"  {run.name:20} {meta.get('region','?'):10} "
+                      f"{meta.get('samples','?')} samples{extra}  [{classes}]")
+            except FileNotFoundError:
+                print(f"  {run.name:20} (no labels.json)")
+    if not found:
+        print(f"No trained models under {root}/")
+    return 0
+
+
+def _cmd_inspect(args) -> int:
+    import numpy as np
+    from .data import project_meta
+
+    project = Path(args.project_dir)
+    meta = project_meta(project)
+    if meta:
+        print(f"Project: {meta.get('name', project.name)}  "
+              f"region={meta.get('region')}  hands={meta.get('hands')}")
+        print(f"Classes: {meta.get('classes', [])}")
+    else:
+        print(f"Project: {project.name}  (no project.json)")
+
+    for mode in artifacts.MODES:
+        root = project / "data" / mode
+        if not root.exists():
+            print(f"\n{mode}: none")
+            continue
+        labels = sorted(d.name for d in root.iterdir() if d.is_dir())
+        print(f"\n{mode}:")
+        dims = set()
+        for name in labels:
+            files = sorted((root / name).glob("*.npy"))
+            if not files:
+                print(f"  {name:16} 0")
+                continue
+            lengths = []
+            for f in files:
+                a = np.load(f)
+                if a.ndim == 1:
+                    a = a[None, :]
+                lengths.append(a.shape[0])
+                dims.add(a.shape[1])
+            if mode == "sequence":
+                usable = sum(1 for n in lengths if n >= args.seq_len)
+                print(f"  {name:16} {len(files):4} clips, "
+                      f"{usable} usable at seq_len={args.seq_len}, "
+                      f"frames {min(lengths)}-{max(lengths)}")
+            else:
+                print(f"  {name:16} {len(files):4} samples")
+        if dims:
+            print(f"  feature dim: {sorted(dims)}"
+                  + ("   MIXED — recapture consistently!" if len(dims) > 1 else ""))
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    handlers = {
+        "train": _cmd_train,
+        "detect": _cmd_detect,
+        "list": _cmd_list,
+        "inspect": _cmd_inspect,
+    }
+    try:
+        return handlers[args.command](args)
+    except (FileNotFoundError, ValueError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

gesto/data.py

@@ -0,0 +1,113 @@
+"""
+Loading Gesto Labeller datasets.
+
+A Gesto project folder looks like:
+
+    <project>/
+        project.json
+        data/
+            static/<label>/<uid>.npy      each (D,)
+            sequence/<label>/<uid>.npy    each (T, D)
+
+Use the "Copy path" button on a project card in Gesto Labeller to get the path.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import numpy as np
+
+from .regions import feature_dim
+
+
+def project_meta(project_dir: str | Path) -> dict:
+    """Contents of project.json, or {} when it isn't there."""
+    p = Path(project_dir) / "project.json"
+    if not p.exists():
+        return {}
+    try:
+        return json.loads(p.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return {}
+
+
+def _class_dirs(project_dir: str | Path, mode: str) -> tuple[Path, list[str]]:
+    root = Path(project_dir) / "data" / mode
+    if not root.exists():
+        other = "sequence" if mode == "static" else "static"
+        hint = ""
+        if (Path(project_dir) / "data" / other).exists():
+            hint = (f"\nThis project has {other} data instead — either capture "
+                    f"in {mode.capitalize()} mode in Gesto Labeller, or train "
+                    f"the {other} model.")
+        raise FileNotFoundError(f"No {mode} data at {root}{hint}")
+    labels = sorted(d.name for d in root.iterdir() if d.is_dir())
+    if not labels:
+        raise ValueError(f"No class folders under {root}")
+    return root, labels
+
+
+def load_static(project_dir: str | Path, region: str):
+    """-> X (N, D) float32, y (N,) int, labels list[str]."""
+    dim = feature_dim(region)
+    root, labels = _class_dirs(project_dir, "static")
+    index = {name: i for i, name in enumerate(labels)}
+
+    X: list[np.ndarray] = []
+    y: list[int] = []
+    for name in labels:
+        for f in sorted((root / name).glob("*.npy")):
+            arr = np.load(f).astype(np.float32).reshape(-1)
+            if arr.shape[0] != dim:
+                raise ValueError(
+                    f"{f} has dim {arr.shape[0]} but region {region!r} expects "
+                    f"{dim}. Wrong region for this project?")
+            X.append(arr)
+            y.append(index[name])
+    if not X:
+        raise ValueError(f"No .npy samples found under {root}")
+    return np.asarray(X, np.float32), np.asarray(y), labels
+
+
+def load_sequence(project_dir: str | Path, region: str, seq_len: int = 30):
+    """-> X (N, seq_len, D) float32, y (N,) int, labels list[str].
+
+    Sequences shorter than seq_len are skipped (they can't fill the window);
+    longer ones are trimmed to the first seq_len frames. Capture at a consistent
+    length in Gesto Labeller ("Max frames") for best results.
+    """
+    dim = feature_dim(region)
+    root, labels = _class_dirs(project_dir, "sequence")
+    index = {name: i for i, name in enumerate(labels)}
+
+    X: list[np.ndarray] = []
+    y: list[int] = []
+    skipped = 0
+    for name in labels:
+        for f in sorted((root / name).glob("*.npy")):
+            arr = np.load(f).astype(np.float32)
+            if arr.ndim == 1:
+                arr = arr[None, :]
+            if arr.shape[1] != dim:
+                raise ValueError(
+                    f"{f} has dim {arr.shape[1]} but region {region!r} expects "
+                    f"{dim}. Wrong region for this project?")
+            if arr.shape[0] < seq_len:
+                skipped += 1
+                continue
+            X.append(arr[:seq_len])
+            y.append(index[name])
+    if not X:
+        raise ValueError(
+            f"No sequences with at least {seq_len} frames under {root}. "
+            f"Lower seq_len or capture longer clips.")
+    if skipped:
+        print(f"Note: skipped {skipped} sequence(s) shorter than {seq_len} frames.")
+    return np.asarray(X, np.float32), np.asarray(y), labels
+
+
+def class_summary(y, labels) -> dict[str, int]:
+    counts = np.bincount(np.asarray(y), minlength=len(labels))
+    return {name: int(c) for name, c in zip(labels, counts)}

gesto/detect.py

@@ -0,0 +1,191 @@
+"""
+Live detection from a webcam or video file.
+
+Matches Gesto Labeller's capture exactly, which is what keeps predictions
+correct: the same MediaPipe Holistic engine, the same landmark ordering, the
+same normalization, and the same webcam mirroring.
+
+static   -> classifies every frame on its own; instant, no warm-up.
+sequence -> keeps a rolling window of the last seq_len frames.
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from pathlib import Path
+
+import numpy as np
+
+from . import artifacts
+from .regions import draw, extract, make_holistic
+
+HEADER_BG = (245, 117, 16)
+WHITE = (255, 255, 255)
+
+
+class Predictor:
+    """A trained model plus everything needed to feed it correctly."""
+
+    def __init__(self, run: str | Path):
+        from tensorflow import keras
+        self.run = Path(run)
+        self.meta = artifacts.load_meta(self.run)
+        self.model = keras.models.load_model(artifacts.model_path(self.run))
+        self.labels: list[str] = self.meta["labels"]
+        self.region: str = self.meta["region"]
+        self.mode: str = self.meta["mode"]
+        self.dim: int = self.meta["input_dim"]
+        self.normalized: bool = self.meta.get("normalized", True)
+        self.seq_len: int = int(self.meta.get("seq_len", 1))
+        self._window: deque = deque(maxlen=self.seq_len)
+
+    @classmethod
+    def load(cls, mode: str, region: str, *,
+             root: str | Path = artifacts.DEFAULT_ROOT,
+             version: int | str | None = None) -> "Predictor":
+        return cls(artifacts.resolve(root, mode, region, version))
+
+    def features(self, res) -> np.ndarray | None:
+        """Landmark vector for a frame, normalized the way training expected."""
+        vec = extract(res, self.region)
+        if vec is None:
+            return None
+        if self.normalized:
+            from .regions import normalize
+            vec = normalize(vec, self.region)
+        return vec
+
+    def reset(self) -> None:
+        self._window.clear()
+
+    def predict(self, vec: np.ndarray | None) -> np.ndarray | None:
+        """Probabilities for one frame, or None when there's nothing to say yet.
+
+        For sequence models this returns None until the window is full.
+        """
+        if self.mode == "static":
+            if vec is None:
+                return None
+            return self.model.predict(vec[None, :], verbose=0)[0]
+
+        if vec is not None:
+            self._window.append(vec)
+        if len(self._window) < self.seq_len:
+            return None
+        batch = np.stack(self._window)[None, :, :]
+        return self.model.predict(batch, verbose=0)[0]
+
+    @property
+    def progress(self) -> tuple[int, int]:
+        return len(self._window), self.seq_len
+
+
+def _palette(n: int) -> list[tuple[int, int, int]]:
+    rng = np.random.RandomState(7)
+    return [(int(rng.randint(60, 256)), int(rng.randint(60, 256)),
+             int(rng.randint(60, 256))) for _ in range(n)]
+
+
+def _overlay(frame, labels, probs, current, colors, progress=None) -> None:
+    import cv2
+    cv2.rectangle(frame, (0, 0), (frame.shape[1], 46), HEADER_BG, -1)
+    text = current
+    if progress and progress[1] > 1:
+        text += f"   [{progress[0]}/{progress[1]}]"
+    cv2.putText(frame, text, (10, 32), cv2.FONT_HERSHEY_SIMPLEX, 0.9, WHITE, 2,
+                cv2.LINE_AA)
+    for i, p in enumerate(probs):
+        y = 60 + i * 34
+        cv2.rectangle(frame, (10, y), (270, y + 26), (50, 50, 50), -1)
+        cv2.rectangle(frame, (10, y), (10 + int(260 * float(p)), y + 26),
+                      colors[i], -1)
+        cv2.putText(frame, f"{labels[i]}  {float(p) * 100:4.1f}%", (16, y + 19),
+                    cv2.FONT_HERSHEY_SIMPLEX, 0.55, WHITE, 1, cv2.LINE_AA)
+
+
+def run(mode: str, region: str, *, root: str | Path = artifacts.DEFAULT_ROOT,
+        version: int | str | None = None, source: str = "0",
+        threshold: float = 0.5, smooth: int = 5, width: int = 960,
+        mirror: bool | None = None) -> None:
+    """Open a camera or video and show live predictions."""
+    import cv2
+
+    predictor = Predictor.load(mode, region, root=root, version=version)
+    labels = predictor.labels
+    print(f"{predictor.mode} / {predictor.region}: {labels}")
+    print(f"model: {predictor.run}")
+
+    src: int | str = int(source) if str(source).isdigit() else source
+    is_webcam = isinstance(src, int)
+    if mirror is None:
+        mirror = is_webcam          # Gesto mirrors the webcam, not video files
+
+    cap = cv2.VideoCapture(src)
+    if not cap.isOpened():
+        raise SystemExit(f"Could not open source: {src}")
+
+    if is_webcam:
+        wait_ms = 1
+    else:
+        fps = cap.get(cv2.CAP_PROP_FPS)
+        wait_ms = int(1000 / fps) if fps and fps > 1 else 33
+
+    window_name = f"gesto {mode}/{region} (q to quit)"
+    cv2.namedWindow(window_name, cv2.WINDOW_AUTOSIZE)
+
+    holistic = make_holistic()
+    colors = _palette(len(labels))
+    recent: deque = deque(maxlen=max(1, smooth))
+    current = "-"
+    probs = np.zeros(len(labels), np.float32)
+
+    quit_pressed = False
+    frame = None
+    try:
+        while cap.isOpened():
+            ok, frame = cap.read()
+            if not ok:
+                break
+            if mirror:
+                frame = cv2.flip(frame, 1)
+
+            rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            rgb.flags.writeable = False
+            res = holistic.process(rgb)
+
+            h0, w0 = frame.shape[:2]
+            if w0 != width:
+                frame = cv2.resize(frame, (width, int(h0 * width / w0)))
+            draw(frame, res, predictor.region)
+
+            vec = predictor.features(res)
+            out = predictor.predict(vec)
+            if out is not None:
+                probs = out
+                top = int(np.argmax(probs))
+                recent.append(top)
+                if (len(recent) == recent.maxlen and len(set(recent)) == 1
+                        and float(probs[top]) >= threshold):
+                    current = labels[top]
+            elif vec is None and predictor.mode == "static":
+                recent.clear()
+                current = "-"
+                probs = np.zeros(len(labels), np.float32)
+
+            _overlay(frame, labels, probs, current, colors,
+                     predictor.progress if predictor.mode == "sequence" else None)
+            cv2.imshow(window_name, frame)
+            if cv2.waitKey(wait_ms) & 0xFF == ord("q"):
+                quit_pressed = True
+                break
+
+        if not is_webcam and not quit_pressed and frame is not None:
+            cv2.putText(frame, "video ended - press any key",
+                        (10, frame.shape[0] - 15), cv2.FONT_HERSHEY_SIMPLEX,
+                        0.6, WHITE, 1, cv2.LINE_AA)
+            cv2.imshow(window_name, frame)
+            cv2.waitKey(0)
+    finally:
+        cap.release()
+        cv2.destroyAllWindows()
+        holistic.close()

gesto/regions.py

@@ -0,0 +1,216 @@
+"""
+Region definitions — extraction, normalization and drawing for each region.
+
+These mirror Gesto Labeller's capture exactly, so a model trained on Gesto data
+receives identically-shaped, identically-ordered, identically-normalized vectors
+at inference time.
+
+    hands_one : one hand (prefer right, else left)  ->  63   (21 x 3)
+    hands_two : left hand + right hand              -> 126   (2 x 21 x 3)
+    pose      : full body                           -> 132   (33 x 4: x,y,z,vis)
+    legs      : lower body subset                   ->  32   (8 x 4)
+    full      : pose + both hands                   -> 258   (132 + 126)
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+LEG_POSE_IDX = [23, 24, 25, 26, 27, 28, 31, 32]
+
+REGION_KEYS = ("hands_one", "hands_two", "pose", "legs", "full")
+
+# region -> (feature dim, Gesto project "region", Gesto project "hands")
+REGION_INFO = {
+    "hands_one": (63, "Hands", "one"),
+    "hands_two": (126, "Hands", "two"),
+    "pose": (132, "Pose", "two"),
+    "legs": (32, "Legs", "two"),
+    "full": (258, "Full", "two"),
+}
+
+
+def feature_dim(region: str) -> int:
+    _check(region)
+    return REGION_INFO[region][0]
+
+
+def _check(region: str) -> None:
+    if region not in REGION_INFO:
+        raise ValueError(
+            f"Unknown region {region!r}. Choose one of: {', '.join(REGION_KEYS)}"
+        )
+
+
+# --------------------------------------------------------------------------
+# extraction
+# --------------------------------------------------------------------------
+
+def _hand(landmarks) -> np.ndarray:
+    if landmarks is None:
+        return np.zeros(63, np.float32)
+    return np.array([[p.x, p.y, p.z] for p in landmarks.landmark],
+                    dtype=np.float32).reshape(-1)
+
+
+def _pose_all(res) -> np.ndarray:
+    if not res.pose_landmarks:
+        return np.zeros(132, np.float32)
+    return np.array([[p.x, p.y, p.z, p.visibility]
+                     for p in res.pose_landmarks.landmark],
+                    dtype=np.float32).reshape(-1)
+
+
+def _pose_legs(res) -> np.ndarray:
+    if not res.pose_landmarks:
+        return np.zeros(len(LEG_POSE_IDX) * 4, np.float32)
+    lm = res.pose_landmarks.landmark
+    return np.array([[lm[i].x, lm[i].y, lm[i].z, lm[i].visibility]
+                     for i in LEG_POSE_IDX], dtype=np.float32).reshape(-1)
+
+
+def extract(res, region: str) -> np.ndarray | None:
+    """Feature vector for `region`, or None when nothing was detected.
+
+    Returning None (rather than zeros) lets callers skip empty frames instead of
+    feeding the model meaningless all-zero input.
+    """
+    _check(region)
+    if region == "hands_one":
+        hand = res.right_hand_landmarks or res.left_hand_landmarks
+        if hand is None:
+            return None
+        return _hand(hand)
+    if region == "hands_two":
+        if not (res.left_hand_landmarks or res.right_hand_landmarks):
+            return None
+        return np.concatenate([_hand(res.left_hand_landmarks),
+                               _hand(res.right_hand_landmarks)])
+    if region == "pose":
+        return _pose_all(res) if res.pose_landmarks else None
+    if region == "legs":
+        return _pose_legs(res) if res.pose_landmarks else None
+    # full
+    if not (res.pose_landmarks or res.left_hand_landmarks
+            or res.right_hand_landmarks):
+        return None
+    return np.concatenate([_pose_all(res),
+                           _hand(res.left_hand_landmarks),
+                           _hand(res.right_hand_landmarks)])
+
+
+# --------------------------------------------------------------------------
+# normalization — byte-for-byte identical to Gesto Labeller's normalize_vector
+# --------------------------------------------------------------------------
+
+def _norm_hand(pts: np.ndarray) -> np.ndarray:
+    """(21,3) -> wrist-relative, scale-normalised."""
+    if np.any(pts):
+        pts = pts - pts[0]
+        scale = np.linalg.norm(pts, axis=1).max()
+        if scale > 1e-6:
+            pts = pts / scale
+    return pts
+
+
+def _norm_body(pts: np.ndarray) -> np.ndarray:
+    """(N,4) x,y,z,visibility -> centred and scale-normalised; visibility kept."""
+    xyz = pts[:, :3]
+    mask = np.any(xyz != 0, axis=1)
+    if np.any(mask):
+        xyz -= xyz[mask].mean(axis=0)
+        scale = np.linalg.norm(xyz, axis=1).max()
+        if scale > 1e-6:
+            xyz /= scale
+    pts[:, :3] = xyz
+    return pts
+
+
+def normalize(vec, region: str) -> np.ndarray:
+    """Translation/scale-invariant normalization, matching Gesto exactly."""
+    _check(region)
+    vec = np.asarray(vec, np.float32)
+    if region == "hands_one":
+        return _norm_hand(vec.reshape(21, 3).copy()).reshape(-1)
+    if region == "hands_two":
+        pts = vec.reshape(2, 21, 3).copy()
+        for h in range(2):
+            pts[h] = _norm_hand(pts[h])
+        return pts.reshape(-1)
+    if region in ("pose", "legs"):
+        n = 33 if region == "pose" else len(LEG_POSE_IDX)
+        return _norm_body(vec.reshape(n, 4).copy()).reshape(-1)
+    # full: pose part as body, hands part per-hand
+    pose = _norm_body(vec[:132].reshape(33, 4).copy()).reshape(-1)
+    hands = vec[132:].reshape(2, 21, 3).copy()
+    for h in range(2):
+        hands[h] = _norm_hand(hands[h])
+    return np.concatenate([pose, hands.reshape(-1)])
+
+
+# --------------------------------------------------------------------------
+# drawing
+# --------------------------------------------------------------------------
+
+HAND_BONES = [(0, 1), (1, 2), (2, 3), (3, 4), (0, 5), (5, 6), (6, 7), (7, 8),
+              (5, 9), (9, 10), (10, 11), (11, 12), (9, 13), (13, 14), (14, 15),
+              (15, 16), (13, 17), (17, 18), (18, 19), (19, 20), (0, 17)]
+POSE_BONES = [(11, 12), (11, 13), (13, 15), (12, 14), (14, 16), (11, 23),
+              (12, 24), (23, 24), (23, 25), (25, 27), (27, 29), (27, 31),
+              (24, 26), (26, 28), (28, 30), (28, 32)]
+LEG_BONES = [(0, 2), (2, 4), (4, 6), (1, 3), (3, 5), (5, 7), (0, 1)]
+
+_BONE_COLOR = (60, 180, 75)
+_JOINT_COLOR = (40, 160, 230)
+
+
+def _plot(frame, pts, bones) -> None:
+    import cv2
+    h, w = frame.shape[:2]
+    for a, b in bones:
+        if a < len(pts) and b < len(pts):
+            ax, ay = pts[a]
+            bx, by = pts[b]
+            if (ax or ay) and (bx or by):
+                cv2.line(frame, (int(ax * w), int(ay * h)),
+                         (int(bx * w), int(by * h)), _BONE_COLOR, 2, cv2.LINE_AA)
+    for x, y in pts:
+        if x or y:
+            cv2.circle(frame, (int(x * w), int(y * h)), 3, _JOINT_COLOR, -1,
+                       cv2.LINE_AA)
+
+
+def draw(frame, res, region: str) -> None:
+    """Draw the region's skeleton onto a BGR frame, in place."""
+    _check(region)
+    xy = lambda h: [(p.x, p.y) for p in h.landmark] if h else []
+
+    if region == "hands_one":
+        hand = res.right_hand_landmarks or res.left_hand_landmarks
+        if hand:
+            _plot(frame, xy(hand), HAND_BONES)
+    elif region == "hands_two":
+        for hand in (res.left_hand_landmarks, res.right_hand_landmarks):
+            if hand:
+                _plot(frame, xy(hand), HAND_BONES)
+    elif region == "pose":
+        if res.pose_landmarks:
+            _plot(frame, xy(res.pose_landmarks), POSE_BONES)
+    elif region == "legs":
+        if res.pose_landmarks:
+            lm = res.pose_landmarks.landmark
+            _plot(frame, [(lm[i].x, lm[i].y) for i in LEG_POSE_IDX], LEG_BONES)
+    else:  # full
+        if res.pose_landmarks:
+            _plot(frame, xy(res.pose_landmarks), POSE_BONES)
+        for hand in (res.left_hand_landmarks, res.right_hand_landmarks):
+            if hand:
+                _plot(frame, xy(hand), HAND_BONES)
+
+
+def make_holistic():
+    """MediaPipe Holistic with the same settings Gesto Labeller captures with."""
+    import mediapipe as mp
+    return mp.solutions.holistic.Holistic(
+        static_image_mode=False, model_complexity=1,
+        min_detection_confidence=0.6, min_tracking_confidence=0.5)

gesto/train.py

@@ -0,0 +1,171 @@
+"""
+Training.
+
+Two model types, matched to what the gesture actually is:
+
+- static   : one frame per sample. The gesture IS a held shape or posture
+             (hand signs, letters, stances). A small Dense network.
+- sequence : a window of frames. The gesture is a MOTION (waving, clapping,
+             jogging). A stacked LSTM.
+
+Both auto-select a lighter architecture on small datasets — an oversized model
+on a few dozen samples overfits and collapses to predicting one class.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import numpy as np
+
+from . import artifacts, data
+from .regions import REGION_INFO, feature_dim
+
+# datasets smaller than this get the lighter model
+SMALL_STATIC = 300
+SMALL_SEQUENCE = 100
+
+
+def _keras():
+    """Imported lazily so `import gesto` stays cheap and TF is optional."""
+    from tensorflow import keras
+    return keras
+
+
+def _class_weights(y, n_classes) -> dict[int, float]:
+    counts = np.bincount(np.asarray(y), minlength=n_classes).astype(float)
+    total = counts.sum()
+    return {i: float(total / (n_classes * c)) if c else 1.0
+            for i, c in enumerate(counts)}
+
+
+def build_static(dim: int, n_classes: int, small: bool):
+    keras = _keras()
+    L = keras.layers
+    if small:
+        layers = [L.Dense(64, activation="relu"), L.Dropout(0.3),
+                  L.Dense(32, activation="relu"), L.Dropout(0.2)]
+    else:
+        layers = [L.Dense(256, activation="relu"), L.Dropout(0.4),
+                  L.Dense(128, activation="relu"), L.Dropout(0.3),
+                  L.Dense(64, activation="relu"), L.Dropout(0.2)]
+    model = keras.Sequential([keras.Input((dim,)), *layers,
+                              L.Dense(n_classes, activation="softmax")])
+    model.compile(optimizer="adam", loss="sparse_categorical_crossentropy",
+                  metrics=["accuracy"])
+    return model
+
+
+def build_sequence(seq_len: int, dim: int, n_classes: int, small: bool):
+    keras = _keras()
+    L = keras.layers
+    if small:
+        layers = [L.LSTM(32, activation="tanh"), L.Dropout(0.4),
+                  L.Dense(32, activation="relu"), L.Dropout(0.3)]
+    else:
+        layers = [L.LSTM(64, return_sequences=True, activation="tanh"),
+                  L.LSTM(128, return_sequences=True, activation="tanh"),
+                  L.LSTM(64, activation="tanh"),
+                  L.Dense(64, activation="relu"),
+                  L.Dense(32, activation="relu")]
+    model = keras.Sequential([keras.Input((seq_len, dim)), *layers,
+                              L.Dense(n_classes, activation="softmax")])
+    model.compile(optimizer="adam", loss="sparse_categorical_crossentropy",
+                  metrics=["accuracy"])
+    return model
+
+
+def _fit(model, X, y, labels, *, epochs, batch_size, val_split, quiet):
+    keras = _keras()
+    stop = keras.callbacks.EarlyStopping(
+        monitor="val_loss", patience=25, restore_best_weights=True)
+    model.fit(X, y, validation_split=val_split, epochs=epochs,
+              batch_size=batch_size, callbacks=[stop],
+              class_weight=_class_weights(y, len(labels)),
+              verbose=0 if quiet else 2)
+    return model
+
+
+def _report(counts: dict[str, int], quiet: bool) -> None:
+    if quiet:
+        return
+    for name, c in counts.items():
+        print(f"   {name:16} {c}")
+    values = list(counts.values())
+    if values and min(values) and max(values) >= 3 * min(values):
+        print("Note: classes are imbalanced — applying class weights.")
+
+
+def train(project_dir: str | Path, region: str, mode: str, *,
+          root: str | Path = artifacts.DEFAULT_ROOT,
+          seq_len: int = 30, epochs: int | None = None, batch_size: int = 16,
+          val_split: float = 0.2, small: bool | None = None,
+          normalized: bool = True, quiet: bool = False) -> Path:
+    """Train a model and save it to a fresh, versioned artifact folder.
+
+    Returns the run folder, e.g. artifacts/static/pose_2.
+    """
+    if mode not in artifacts.MODES:
+        raise ValueError(f"mode must be one of {artifacts.MODES}")
+    dim = feature_dim(region)
+
+    if mode == "static":
+        X, y, labels = data.load_static(project_dir, region)
+        auto_small = len(X) < SMALL_STATIC
+        epochs = 200 if epochs is None else epochs
+    else:
+        X, y, labels = data.load_sequence(project_dir, region, seq_len)
+        auto_small = len(X) < SMALL_SEQUENCE
+        epochs = 300 if epochs is None else epochs
+
+    use_small = auto_small if small is None else small
+    counts = data.class_summary(y, labels)
+
+    if not quiet:
+        shape = f"{len(X)} samples" if mode == "static" else f"{len(X)} sequences"
+        print(f"{mode} / {region} (dim {dim}): {shape}, classes={labels}")
+    _report(counts, quiet)
+    if min(counts.values()) == 0:
+        raise ValueError("A class has no usable samples — capture some for "
+                         "every class, or lower seq_len.")
+    if use_small and small is None and not quiet:
+        print(f"Note: small dataset — using the lighter model (a large one "
+              f"overfits and collapses on this much data).")
+
+    rng = np.random.default_rng(42)
+    order = rng.permutation(len(X))
+    X, y = X[order], np.asarray(y)[order]
+
+    if mode == "static":
+        model = build_static(dim, len(labels), use_small)
+    else:
+        model = build_sequence(seq_len, dim, len(labels), use_small)
+    if not quiet:
+        model.summary()
+
+    _fit(model, X, y, labels, epochs=epochs, batch_size=batch_size,
+         val_split=val_split, quiet=quiet)
+
+    run = artifacts.new_run(root, mode, region)
+    model.save(artifacts.model_path(run))
+    gesto_region, hands = REGION_INFO[region][1], REGION_INFO[region][2]
+    meta = {
+        "labels": labels,
+        "region": region,
+        "mode": mode,
+        "input_dim": dim,
+        "gesto_region": gesto_region,
+        "hands": hands,
+        "normalized": normalized,
+        "samples": int(len(X)),
+        "counts": counts,
+    }
+    if mode == "sequence":
+        meta["seq_len"] = int(seq_len)
+    artifacts.save_meta(run, meta)
+
+    if not quiet:
+        loss, acc = model.evaluate(X, y, verbose=0)
+        print(f"\nTraining-set accuracy: {acc:.3f}")
+        print(f"Saved -> {run}")
+    return run

gesto-0.1.0.dist-info/METADATA

@@ -0,0 +1,160 @@
+Metadata-Version: 2.4
+Name: gesto
+Version: 0.1.0
+Summary: Train and run gesture recognition models from Gesto Labeller datasets
+Author: Sundar Balamurugan
+License: MIT
+Project-URL: Homepage, https://github.com/TheMadrasTechie/gesto
+Project-URL: Issues, https://github.com/TheMadrasTechie/gesto/issues
+Keywords: gesture-recognition,mediapipe,pose-estimation,computer-vision,lstm,sign-language
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Scientific/Engineering :: Image Recognition
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.23
+Requires-Dist: tensorflow>=2.15
+Requires-Dist: opencv-python>=4.8
+Requires-Dist: mediapipe>=0.10
+Provides-Extra: dev
+Requires-Dist: pytest>=7; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+Dynamic: license-file
+
+# gesto
+
+Train and run gesture recognition models from [Gesto Labeller](https://github.com/TheMadrasTechie) datasets.
+
+Point it at a project folder, pick a mode and a region, and it handles the rest —
+loading, training, versioned model storage, and live detection that matches how
+the data was captured.
+
+```bash
+pip install gesto
+```
+
+## Two modes
+
+| mode | the gesture is… | model | example |
+|---|---|---|---|
+| `static` | a held shape or posture | Dense network | thumbs up, alphabet letters, a stance |
+| `sequence` | a motion over time | stacked LSTM | waving, clapping, jogging |
+
+Static needs far less data (every captured frame is a training sample) and
+predicts instantly with no warm-up. Reach for `sequence` only when two gestures
+share the same shape and differ by movement.
+
+## Five regions
+
+| region | dim | tracks |
+|---|---|---|
+| `hands_one` | 63 | one hand, 21 joints |
+| `hands_two` | 126 | both hands |
+| `pose` | 132 | full body, 33 points |
+| `legs` | 32 | lower body, 8 points |
+| `full` | 258 | body + both hands |
+
+The region must match how the project was captured — `gesto` checks the feature
+dimension and tells you if it doesn't.
+
+## Command line
+
+```bash
+# what's in this dataset?
+gesto inspect ./gesto_projects/signs
+
+# train
+gesto train static hands_one ./gesto_projects/signs
+gesto train sequence pose ./gesto_projects/jogging --seq-len 30
+
+# detect (newest model by default)
+gesto detect static hands_one
+gesto detect sequence pose --source clip.mp4
+gesto detect static hands_one --version 2
+
+# what have I trained?
+gesto list
+```
+
+## Python
+
+```python
+import gesto
+
+run = gesto.train("./gesto_projects/signs", region="hands_one", mode="static")
+gesto.detect("static", "hands_one")
+```
+
+Or drive a model yourself:
+
+```python
+from gesto.detect import Predictor
+
+predictor = Predictor.load("static", "hands_one")
+vector = predictor.features(holistic_result)   # extract + normalize
+probs = predictor.predict(vector)
+```
+
+## Where models go
+
+Everything lands under one `artifacts/` folder, split by mode then region.
+Training never overwrites an earlier run — it versions:
+
+```
+artifacts/
+    static/
+        hands_one/          model.keras, labels.json
+        hands_one_2/        the next run
+        pose/
+    sequence/
+        pose/
+        pose_2/
+```
+
+`gesto detect static pose` picks the newest version; `--version 1` picks a
+specific one.
+
+## Matching your capture
+
+Predictions are only correct when detection feeds the model the same kind of
+vector it trained on. `gesto` mirrors Gesto Labeller exactly:
+
+- **the same engine** — MediaPipe Holistic, same confidence settings
+- **the same landmark order** — including one-hand mode preferring the right
+  hand and falling back to the left
+- **the same normalization** — translation/scale-invariant, verified identical
+- **the same mirroring** — webcam frames are flipped, video files are not
+
+If you captured with Gesto's **Normalise** unchecked, pass `--raw` when training
+so detection knows to skip it.
+
+## Getting good results
+
+- **Balance your classes.** Similar sample counts per class; `gesto` applies
+  class weights but balanced data is better.
+- **Enough samples.** ~20–30 static frames per class, or ~15–30 sequences.
+  Small datasets automatically get a lighter model, since an oversized network
+  on little data overfits and collapses to predicting one class.
+- **Consistent clip length** for sequence mode — set "Max frames" in Gesto
+  Labeller so every capture is the same length.
+
+Run `gesto inspect <project>` to check all of this before training.
+
+## Requirements
+
+Python 3.9+, TensorFlow, OpenCV, MediaPipe, NumPy — all installed with the
+package.
+
+## License
+
+MIT

gesto-0.1.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+gesto/__init__.py,sha256=r4CH5XvWh967F4snt0DX34-p2asr2DE1GdvRxwZwYBs,2023
+gesto/artifacts.py,sha256=DwM6z9gjqUUeMgjehK0QN27mPFvaMVltZj4lyQe4QRc,4003
+gesto/cli.py,sha256=D_j2UdfiWpUK5JqoGFDcxc1aqMozq3xplcLPIS6VFWU,7142
+gesto/data.py,sha256=RKHstVwBz4dFlG4PZsZajPPKFwRBrpzgEtZzc3LYqBw,4003
+gesto/detect.py,sha256=xyCbXof6K2IUBf4f85tsOtQuv3kkhcGSJ-f9mj_E_sk,6874
+gesto/regions.py,sha256=meMb4oaaamv9b5vKYj_Xtf6bVZQQx9butTDyN1Snk7I,7929
+gesto/train.py,sha256=KUiRGsNWXyna8jC6ial4cuUdYKCEiqYhtjjxI6U8_vY,6236
+gesto-0.1.0.dist-info/licenses/LICENSE,sha256=XFJjfKTnDwOnOxZ1rFh96YppdNU9sFw3Kb455hK3-q8,1075
+gesto-0.1.0.dist-info/METADATA,sha256=XKTL5b89P5T8GjENIR5tGQxYsHlxTtjMxIjcYKNdli0,5190
+gesto-0.1.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+gesto-0.1.0.dist-info/entry_points.txt,sha256=6viZErcidQAscKN9S-ks6aaEH37CIRujaAeht0_qyig,41
+gesto-0.1.0.dist-info/top_level.txt,sha256=PmyDmZ9EQmvbTSlj9YAL5nljnkb1dBwJtK8eOvzajog,6
+gesto-0.1.0.dist-info/RECORD,,

gesto-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

gesto-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+gesto = gesto.cli:main

gesto-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Sundar Balamurugan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

gesto-0.1.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+gesto