vd3 0.1.0__tar.gz

vd3-0.1.0/.gitignore

@@ -0,0 +1,29 @@
+# Python
+__pycache__/
+*.pyc
+*.pyo
+*.egg-info/
+dist/
+build/
+.venv/
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+
+# Linting
+.ruff_cache/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+#
+.pypirc

vd3-0.1.0/PKG-INFO

@@ -0,0 +1,208 @@
+Metadata-Version: 2.4
+Name: vd3
+Version: 0.1.0
+Summary: Content database library and CLI for VisData 3
+Project-URL: Homepage, https://github.com/jmuncaster/vd3
+Project-URL: Repository, https://github.com/jmuncaster/vd3
+Project-URL: Issues, https://github.com/jmuncaster/vd3/issues
+Author-email: Justin Muncaster <justin@muncasterconsulting.com>
+License-Expression: MIT
+Keywords: computer-vision,dataset,dvc,foxglove,mcap,video
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Multimedia :: Video
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.12
+Requires-Dist: duckdb>=1.1.0
+Requires-Dist: dvc-azure>=3.0
+Requires-Dist: dvc-gs>=3.0
+Requires-Dist: dvc-s3>=3.0
+Requires-Dist: dvc>=3.50.0
+Requires-Dist: ffmpeg-python>=0.2.0
+Requires-Dist: foxglove-schemas-protobuf>=0.3.0
+Requires-Dist: mcap-protobuf-support>=0.5.4
+Requires-Dist: mcap>=1.1.0
+Requires-Dist: orjson>=3.10.0
+Requires-Dist: pillow>=10.0.0
+Requires-Dist: protobuf>=5.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: typer>=0.12.0
+Description-Content-Type: text/markdown
+
+# VD3Storage
+
+Content database library and CLI for VisData 3. Manages video assets, annotations, and workspaces backed by MCAP files and CSV-based metadata.
+
+## Installation
+
+```bash
+uv sync
+```
+
+To use as a dependency in a content database project:
+
+```toml
+# pyproject.toml
+[project]
+dependencies = ["vd3storage"]
+
+[tool.uv.sources]
+vd3storage = { path = "./vd3storage", editable = true }
+```
+
+## Quick Start
+
+```bash
+# Initialize a new content database
+vd3 init /path/to/mydb
+
+# Add a video
+vd3 add video clip.mp4 -c my-collection -p /path/to/mydb
+
+# Add multiple videos with a glob
+vd3 add video '*.mp4' -c my-collection -p /path/to/mydb
+
+# List assets
+vd3 list assets -p /path/to/mydb
+
+# Check status
+vd3 status -p /path/to/mydb
+```
+
+## Remote Storage
+
+VD3Storage uses [DVC](https://dvc.org) under the hood to push and pull large media files to/from remote storage.
+
+### Google Cloud Storage
+
+1. Authenticate with GCS:
+
+   ```bash
+   gcloud auth application-default login
+   ```
+
+2. Add a GCS remote:
+
+   ```bash
+   vd3 remote add gcs gs://my-bucket/vd3-data -p /path/to/mydb
+   ```
+
+3. Push media to remote:
+
+   ```bash
+   vd3 push --all -p /path/to/mydb
+   ```
+
+4. Pull media from remote (e.g. on another machine):
+
+   ```bash
+   vd3 pull --all -p /path/to/mydb
+   ```
+
+### Amazon S3
+
+```bash
+# Ensure AWS credentials are configured (aws configure, env vars, etc.)
+vd3 remote add s3remote s3://my-bucket/vd3-data -p /path/to/mydb
+```
+
+### Local / NAS
+
+```bash
+vd3 remote add backup /mnt/nas/vd3-backup -p /path/to/mydb
+```
+
+### Listing Remotes
+
+```bash
+vd3 remote list -p /path/to/mydb
+```
+
+## Workspaces
+
+Workspaces let you organize subsets of assets into named groups with optional packages (folders).
+
+```bash
+# Create a workspace
+vd3 workspace create "My Experiment" -p /path/to/mydb
+
+# Add assets by name, ID, or media file glob
+vd3 workspace add-asset my-experiment 'db/media/videos/fc/*.mcap' -p /path/to/mydb
+
+# List workspaces
+vd3 workspace list -p /path/to/mydb
+
+# Show workspace details
+vd3 workspace show my-experiment -p /path/to/mydb
+```
+
+## Adding Videos
+
+```bash
+# Single file
+vd3 add video clip.mp4 -c dashcam
+
+# Glob pattern (quote to prevent shell expansion)
+vd3 add video '*.mp4' -c dashcam
+
+# Recursive glob
+vd3 add video 'rawdata/**/*.mp4' -c dashcam
+
+# Force re-import of a duplicate
+vd3 add video clip.mp4 -c dashcam --force
+
+# Add and assign to a workspace
+vd3 add video clip.mp4 -c dashcam -w my-workspace -k batch1
+```
+
+Duplicate detection uses SHA-256 hashing of the source file. Use `--force` to overwrite an existing asset with the same content.
+
+## Adding Results
+
+Import VD3 JSON detection/track results into an asset's MCAP:
+
+```bash
+vd3 add result results.json -a clip -p /path/to/mydb
+```
+
+## Viewing in Foxglove
+
+MCAP files are written with `foxglove.CompressedImage` protobuf messages on the `/video/frames` topic, compatible with [Foxglove Studio](https://foxglove.dev) and Lichtblick.
+
+Open any `.mcap` file under `db/media/videos/` directly in Foxglove to view the video and any annotation overlays.
+
+## CLI Reference
+
+```
+vd3 --help              Top-level help
+vd3 <command> --help    Help for a specific command
+```
+
+| Command | Description |
+|---|---|
+| `init` | Initialize a new content database |
+| `info` | Show database information |
+| `add video` | Import video files |
+| `add result` | Import detection/track results |
+| `list assets` | List assets |
+| `list collections` | List collections |
+| `show` | Show asset details |
+| `query` | Run raw SQL against the database |
+| `remove` | Delete an asset |
+| `workspace create` | Create a workspace |
+| `workspace add-asset` | Add assets to a workspace |
+| `workspace remove-asset` | Remove an asset from a workspace |
+| `workspace list` | List workspaces |
+| `workspace show` | Show workspace details |
+| `workspace delete` | Delete a workspace |
+| `status` | Show media availability (use `-l` to list all) |
+| `push` | Push media to remote storage |
+| `pull` | Pull media from remote storage |
+| `remote add` | Add a remote storage backend |
+| `remote list` | List configured remotes |
+| `export` | Export data from the database |

vd3-0.1.0/README.md

@@ -0,0 +1,172 @@
+# VD3Storage
+
+Content database library and CLI for VisData 3. Manages video assets, annotations, and workspaces backed by MCAP files and CSV-based metadata.
+
+## Installation
+
+```bash
+uv sync
+```
+
+To use as a dependency in a content database project:
+
+```toml
+# pyproject.toml
+[project]
+dependencies = ["vd3storage"]
+
+[tool.uv.sources]
+vd3storage = { path = "./vd3storage", editable = true }
+```
+
+## Quick Start
+
+```bash
+# Initialize a new content database
+vd3 init /path/to/mydb
+
+# Add a video
+vd3 add video clip.mp4 -c my-collection -p /path/to/mydb
+
+# Add multiple videos with a glob
+vd3 add video '*.mp4' -c my-collection -p /path/to/mydb
+
+# List assets
+vd3 list assets -p /path/to/mydb
+
+# Check status
+vd3 status -p /path/to/mydb
+```
+
+## Remote Storage
+
+VD3Storage uses [DVC](https://dvc.org) under the hood to push and pull large media files to/from remote storage.
+
+### Google Cloud Storage
+
+1. Authenticate with GCS:
+
+   ```bash
+   gcloud auth application-default login
+   ```
+
+2. Add a GCS remote:
+
+   ```bash
+   vd3 remote add gcs gs://my-bucket/vd3-data -p /path/to/mydb
+   ```
+
+3. Push media to remote:
+
+   ```bash
+   vd3 push --all -p /path/to/mydb
+   ```
+
+4. Pull media from remote (e.g. on another machine):
+
+   ```bash
+   vd3 pull --all -p /path/to/mydb
+   ```
+
+### Amazon S3
+
+```bash
+# Ensure AWS credentials are configured (aws configure, env vars, etc.)
+vd3 remote add s3remote s3://my-bucket/vd3-data -p /path/to/mydb
+```
+
+### Local / NAS
+
+```bash
+vd3 remote add backup /mnt/nas/vd3-backup -p /path/to/mydb
+```
+
+### Listing Remotes
+
+```bash
+vd3 remote list -p /path/to/mydb
+```
+
+## Workspaces
+
+Workspaces let you organize subsets of assets into named groups with optional packages (folders).
+
+```bash
+# Create a workspace
+vd3 workspace create "My Experiment" -p /path/to/mydb
+
+# Add assets by name, ID, or media file glob
+vd3 workspace add-asset my-experiment 'db/media/videos/fc/*.mcap' -p /path/to/mydb
+
+# List workspaces
+vd3 workspace list -p /path/to/mydb
+
+# Show workspace details
+vd3 workspace show my-experiment -p /path/to/mydb
+```
+
+## Adding Videos
+
+```bash
+# Single file
+vd3 add video clip.mp4 -c dashcam
+
+# Glob pattern (quote to prevent shell expansion)
+vd3 add video '*.mp4' -c dashcam
+
+# Recursive glob
+vd3 add video 'rawdata/**/*.mp4' -c dashcam
+
+# Force re-import of a duplicate
+vd3 add video clip.mp4 -c dashcam --force
+
+# Add and assign to a workspace
+vd3 add video clip.mp4 -c dashcam -w my-workspace -k batch1
+```
+
+Duplicate detection uses SHA-256 hashing of the source file. Use `--force` to overwrite an existing asset with the same content.
+
+## Adding Results
+
+Import VD3 JSON detection/track results into an asset's MCAP:
+
+```bash
+vd3 add result results.json -a clip -p /path/to/mydb
+```
+
+## Viewing in Foxglove
+
+MCAP files are written with `foxglove.CompressedImage` protobuf messages on the `/video/frames` topic, compatible with [Foxglove Studio](https://foxglove.dev) and Lichtblick.
+
+Open any `.mcap` file under `db/media/videos/` directly in Foxglove to view the video and any annotation overlays.
+
+## CLI Reference
+
+```
+vd3 --help              Top-level help
+vd3 <command> --help    Help for a specific command
+```
+
+| Command | Description |
+|---|---|
+| `init` | Initialize a new content database |
+| `info` | Show database information |
+| `add video` | Import video files |
+| `add result` | Import detection/track results |
+| `list assets` | List assets |
+| `list collections` | List collections |
+| `show` | Show asset details |
+| `query` | Run raw SQL against the database |
+| `remove` | Delete an asset |
+| `workspace create` | Create a workspace |
+| `workspace add-asset` | Add assets to a workspace |
+| `workspace remove-asset` | Remove an asset from a workspace |
+| `workspace list` | List workspaces |
+| `workspace show` | Show workspace details |
+| `workspace delete` | Delete a workspace |
+| `status` | Show media availability (use `-l` to list all) |
+| `push` | Push media to remote storage |
+| `pull` | Pull media from remote storage |
+| `remote add` | Add a remote storage backend |
+| `remote list` | List configured remotes |
+| `export` | Export data from the database |

vd3-0.1.0/pyproject.toml

@@ -0,0 +1,78 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "vd3"
+version = "0.1.0"
+description = "Content database library and CLI for VisData 3"
+requires-python = ">=3.12"
+license = "MIT"
+authors = [
+    { name = "Justin Muncaster", email = "justin@muncasterconsulting.com" },
+]
+readme = "README.md"
+keywords = ["video", "dataset", "mcap", "foxglove", "dvc", "computer-vision"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Multimedia :: Video",
+]
+dependencies = [
+    "duckdb>=1.1.0",
+    "pydantic>=2.0",
+    "mcap>=1.1.0",
+    "ffmpeg-python>=0.2.0",
+    "pillow>=10.0.0",
+    "typer>=0.12.0",
+    "rich>=13.0.0",
+    "orjson>=3.10.0",
+    "foxglove-schemas-protobuf>=0.3.0",
+    "protobuf>=5.0",
+    "mcap-protobuf-support>=0.5.4",
+    "dvc>=3.50.0",
+    "dvc-gs>=3.0",
+    "dvc-s3>=3.0",
+    "dvc-azure>=3.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/jmuncaster/vd3"
+Repository = "https://github.com/jmuncaster/vd3"
+Issues = "https://github.com/jmuncaster/vd3/issues"
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0",
+    "ruff>=0.8.0",
+    "ty>=0.0.1a1",
+    "prek>=0.0.1",
+    "hatch-vcs>=0.5.0",
+    "hatchling>=1.28.0",
+    "build>=1.0",
+    "twine>=6.0",
+]
+
+[project.scripts]
+vd3 = "vd3storage.cli:cli"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/vd3storage"]
+
+[tool.ruff]
+line-length = 120
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "SIM"]
+ignore = ["UP046", "UP047"]  # Allow Generic[T] pattern (more readable than PEP 695 type params)
+
+[tool.ruff.lint.per-file-ignores]
+"src/vd3storage/cli/**" = ["B008"]  # typer.Argument/Option in defaults is the standard pattern
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

vd3-0.1.0/scripts/publish.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# Build and publish to PyPI.
+#
+# Usage:
+#   ./scripts/publish.sh          # publish to PyPI
+#   ./scripts/publish.sh --test   # publish to Test PyPI
+#
+# Requires ~/.pypirc or TWINE_USERNAME/TWINE_PASSWORD env vars.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
+cd "$PROJECT_DIR"
+
+REPO_FLAG=()
+if [[ "${1:-}" == "--test" ]]; then
+    REPO_FLAG=(--repository testpypi)
+    echo "Publishing to Test PyPI"
+else
+    echo "Publishing to PyPI"
+fi
+
+# Clean previous builds
+rm -rf dist/
+
+# Lint
+echo "Running ruff..."
+uv run ruff check src/
+
+# Build
+echo "Building..."
+uv run python -m build
+
+# Show what we built
+echo ""
+echo "Built:"
+ls -lh dist/
+
+# Confirm
+PKG_INFO=$(uv run python -c "import tomllib; t=tomllib.load(open('pyproject.toml','rb'))['project']; print(t['name'], t['version'])")
+PKG_NAME="${PKG_INFO% *}"
+VERSION="${PKG_INFO#* }"
+echo ""
+read -rp "Publish ${PKG_NAME} v${VERSION}? [y/N] " confirm
+if [[ "$confirm" != [yY] ]]; then
+    echo "Aborted."
+    exit 1
+fi
+
+# Upload
+uv run twine upload "${REPO_FLAG[@]}" dist/*
+
+echo ""
+echo "Published ${PKG_NAME} v${VERSION}"

vd3-0.1.0/src/vd3storage/__init__.py

@@ -0,0 +1,10 @@
+"""VD3Storage — Content database library for VisData 3."""
+
+try:
+    from vd3storage._version import __version__
+except ImportError:
+    __version__ = "0.0.0.dev0"
+
+from vd3storage.storage import VD3Storage
+
+__all__ = ["VD3Storage", "__version__"]

vd3-0.1.0/src/vd3storage/cli/__init__.py

@@ -0,0 +1,5 @@
+"""VD3Storage CLI."""
+
+from vd3storage.cli.app import cli
+
+__all__ = ["cli"]

vd3-0.1.0/src/vd3storage/cli/app.py

@@ -0,0 +1,43 @@
+"""VD3Storage CLI — Typer application."""
+
+import typer
+
+from vd3storage.cli.commands.add import add_app
+from vd3storage.cli.commands.export import export_app
+from vd3storage.cli.commands.info import info_command
+from vd3storage.cli.commands.init import init_command
+from vd3storage.cli.commands.list_ import list_app
+from vd3storage.cli.commands.query import query_command
+from vd3storage.cli.commands.remote import pull_command, push_command, remote_app, status_command
+from vd3storage.cli.commands.remove import remove_command
+from vd3storage.cli.commands.show import show_command
+from vd3storage.cli.commands.workspace import workspace_app
+
+app = typer.Typer(
+    name="vd3",
+    help="VD3Storage — Content database management for VisData 3",
+    add_completion=False,
+    rich_markup_mode="rich",
+    no_args_is_help=True,
+)
+
+# Top-level commands
+app.command("init")(init_command)
+app.command("info")(info_command)
+app.command("show")(show_command)
+app.command("query")(query_command)
+app.command("remove")(remove_command)
+app.command("pull")(pull_command)
+app.command("push")(push_command)
+app.command("status")(status_command)
+
+# Sub-command groups
+app.add_typer(add_app, name="add")
+app.add_typer(list_app, name="list")
+app.add_typer(workspace_app, name="workspace")
+app.add_typer(export_app, name="export")
+app.add_typer(remote_app, name="remote")
+
+
+def cli() -> None:
+    app()