image-annex 0.1.2__tar.gz

image_annex-0.1.2/PKG-INFO

@@ -0,0 +1,18 @@
+Metadata-Version: 2.4
+Name: image-annex
+Version: 0.1.2
+Summary: High-performance, Git-Annex powered photo archive with AWS S3.
+Requires-Python: <4,>=3.9
+Requires-Dist: aws-cdk-lib==2.202.0
+Requires-Dist: constructs<11.0.0,>=10.0.0
+Requires-Dist: pillow>=11.3.0
+Requires-Dist: platformdirs>=4.4.0
+Requires-Dist: psutil>=6.0.0
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pyexiftool>=0.5.6
+Requires-Dist: faster-whisper>=1.1.0
+Requires-Dist: sentence-transformers>=3.0.0
+Requires-Dist: sqlite-vec>=0.1.0
+Requires-Dist: textual>=1.0.0
+Requires-Dist: toml
+Requires-Dist: torch>=2.0.0

image_annex-0.1.2/README.md

@@ -0,0 +1,86 @@
+
+# Image Annex
+
+This project contains the AWS CDK infrastructure and client-side tooling for the Image Annex system, a bulletproof, deduplicating, and cost-effective photo archive.
+
+The core concept is based on the blog post [Your Photos Are a Mess](blog-post.md), which details a workflow for using `git-annex` to manage a master photo repository backed by cloud storage.
+
+## Architecture
+
+The system consists of two primary components:
+
+1.  **Cloud Infrastructure (AWS CDK):** Defines an S3 bucket used as a `git-annex` special remote. It is managed via Python-based AWS CDK and deployed via GitHub Actions.
+2.  **Client-Side Tooling:**
+    *   `scripts/setup_client.py`: Automates the initialization of a local `git-annex` repository and configuration of the S3 remote.
+    *   `scripts/hook.py`: A pre-commit hook that extracts EXIF metadata and generates local thumbnails upon `git commit`.
+    *   `scripts/search_metadata.py`: A CLI tool to search the local SQLite metadata database.
+
+## Core Components
+
+*   **[Blog Post](blog-post.md):** The original article outlining the philosophy and the user-facing workflow for managing photos.
+*   **[Security Architecture](docs/SECURITY.md):** A detailed document explaining the security model for deployment, application runtime, and local development.
+*   **AWS CDK Application:** Located in `cdk_image_annex/`.
+*   **Scripts:** Located in `scripts/`.
+
+## Usage & Migration Workflow
+
+The general workflow for migrating images to the Annex is as follows:
+
+1.  **Initialize the Client:**
+    Run the setup script to prepare a local directory as your Annex repository.
+    ```bash
+    python scripts/setup_client.py --repo-path ~/my-photo-annex --s3-bucket-name my-annex-bucket --s3-region us-east-1
+    ```
+2.  **Add Photos:**
+    Move your photos into the repository directory.
+    ```bash
+    cd ~/my-photo-annex
+    cp /path/to/my/photos/*.jpg .
+    git add .
+    ```
+3.  **Commit (Triggers Metadata & Thumbnails):**
+    When you commit, the pre-commit hook automatically extracts EXIF data and generates thumbnails (stored locally in `.git-annex/thumbnails/`).
+    ```bash
+    git commit -m "Add summer vacation photos"
+    ```
+4.  **Sync to S3:**
+    Upload the actual file content to AWS S3.
+    ```bash
+    git annex sync --content
+    ```
+
+## Metadata & Thumbnails
+
+*   **Metadata:** Extracted using `exiftool` and stored in a local SQLite database (`metadata.db`).
+*   **Thumbnails:** Generated as 256x256 JPEGs using `Pillow-SIMD`. They are stored locally and are *not* uploaded to S3, keeping the cloud storage focused on the original assets.
+
+## Current Status (Dev/UAT)
+
+This project is currently in a **Development/UAT phase**. Key points:
+*   Automated testing (UAT) is performed via Docker (`Dockerfile.uat`).
+*   **Limitation:** `git-annex` currently places an `annex-uuid` file at the root of the S3 bucket during `initremote`, even if a prefix is configured. This makes multi-tenant or multi-prefix use of a single bucket slightly more complex.
+*   The production infrastructure is defined but should be used with caution until UAT is finalized.
+
+## Deployment
+
+This project is deployed automatically via a GitHub Actions workflow. The deployment pipeline is configured to use a secure OIDC connection to AWS.
+
+For details, see [docs/SECURITY.md](docs/SECURITY.md).
+
+## Local Development
+
+To work with the CDK infrastructure locally:
+
+1.  **Environment Setup:**
+    *   Copy `.env.template` to `.env`.
+    *   Set up a Python virtual environment:
+        ```bash
+        python3 -m venv .venv
+        source .venv/bin/activate
+        pip install .
+        ```
+2.  **CDK Commands:**
+    ```bash
+    op run --env-file=.env -- cdk synth
+    op run --env-file=.env -- cdk deploy
+    ```

image_annex-0.1.2/pyproject.toml

@@ -0,0 +1,59 @@
+[project]
+name = "image-annex"
+version = "0.1.2"
+description = "High-performance, Git-Annex powered photo archive with AWS S3."
+dependencies = [
+    "aws-cdk-lib==2.202.0",
+    "constructs>=10.0.0,<11.0.0",
+    "pillow>=11.3.0",
+    "platformdirs>=4.4.0",
+    "psutil>=6.0.0",
+    "pydantic>=2.0.0",
+    "pyexiftool>=0.5.6",
+    "faster-whisper>=1.1.0",
+    "sentence-transformers>=3.0.0",
+    "sqlite-vec>=0.1.0",
+    "textual>=1.0.0",
+    "toml",
+    "torch>=2.0.0",
+]
+requires-python = ">=3.9,<4"
+
+[project.scripts]
+ia = "image_annex.cli.main:main"
+image-annex = "image_annex.tui.app:main"
+ia-scan = "image_annex.core.scanner:main"
+ia-gallery = "image_annex.core.gallery:main"
+
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[dependency-groups]
+dev = [
+    "mypy>=1.19.1",
+    "pytest",
+    "pytest-asyncio>=1.2.0",
+    "ruff>=0.15.2",
+    "types-toml>=0.10.8.20240310",
+    "verkit",
+]
+
+[tool.pytest.ini_options]
+norecursedirs = [".venv", "build", "cdk.out", "appdata"]
+asyncio_mode = "strict"
+
+[tool.mypy]
+exclude = ["^cdk\\.out/"]
+ignore_missing_imports = true
+
+[tool.ruff]
+exclude = [
+    ".git",
+    ".mypy_cache",
+    ".ruff_cache",
+    ".venv",
+    "cdk.out",
+    "build",
+    "dist",
+]

image_annex-0.1.2/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

image_annex-0.1.2/src/image_annex/cli/main.py

@@ -0,0 +1,188 @@
+import argparse
+import sys
+import subprocess
+from rich.console import Console
+from image_annex.tui.app import ImageAnnexApp
+from image_annex.core.config import ConfigManager
+
+console = Console()
+
+def run_tui(args):
+    app = ImageAnnexApp()
+    app.run()
+
+def run_gallery_start(args):
+    console.print("[bold yellow]Starting Gallery server...[/]")
+    gallery_log = open(".logs/gallery.log", "w", encoding="utf-8")
+    process = subprocess.Popen(
+        [sys.executable, "-m", "image_annex.core.gallery"],
+        stdout=gallery_log,
+        stderr=gallery_log
+    )
+    console.print(f"[bold green]Gallery started (PID: {process.pid})[/]")
+
+def run_gallery_stop(args):
+    import psutil # type: ignore
+    killed = False
+    for proc in psutil.process_iter(['pid', 'name', 'cmdline']):
+        if proc.info['cmdline'] and 'image_annex.core.gallery' in str(proc.info['cmdline']):
+            proc.terminate()
+            killed = True
+    if killed:
+        console.print("[bold green]Gallery stopped.[/]")
+    else:
+        console.print("[bold yellow]Gallery not running.[/]")
+
+def run_gallery_status(args):
+    import psutil # type: ignore
+    running = False
+    for proc in psutil.process_iter(['name', 'cmdline']):
+        if proc.info['cmdline'] and 'image_annex.core.gallery' in str(proc.info['cmdline']):
+            running = True
+            break
+    if running:
+        console.print("[bold green]Gallery is running.[/]")
+    else:
+        console.print("[bold yellow]Gallery is stopped.[/]")
+
+def run_ai_scan(args):
+    from image_annex.core.ai import AIScraper
+    cfg = ConfigManager().get_config()
+    thumb_dir = cfg.git_annex_repo.expanded_path / ".git-annex" / "thumbnails"
+    scraper = AIScraper(cfg.db_path, thumb_dir)
+    console.print("[bold cyan]Starting AI scraper...[/]")
+    scraper.start()
+    scraper.join()
+    console.print("[bold green]AI scan finished.[/]")
+
+def run_storage_status(args):
+    console.print("[bold]Storage Metrics[/]")
+    cfg = ConfigManager().get_config()
+    from image_annex.core.service import ImageAnnexService
+    service = ImageAnnexService(cfg)
+    metrics = service.get_storage_metrics()
+    console.print(metrics["raw_output"])
+
+def run_storage_lock(args):
+    cfg = ConfigManager().get_config()
+    from image_annex.core.service import ImageAnnexService
+    service = ImageAnnexService(cfg)
+    console.print("[bold yellow]Locking archive...[/]")
+    console.print("[dim]Note: This converts working files to symlinks/pointers. Ensure you have run 'ia sync' first.[/]")
+    service.lock_archive()
+    console.print("[bold green]Archive locked.[/]")
+
+def run_storage_prune(args):
+    cfg = ConfigManager().get_config()
+    from image_annex.core.service import ImageAnnexService
+    service = ImageAnnexService(cfg)
+    console.print("[bold yellow]Pruning local content...[/]")
+    console.print("[bold red]DANGER:[/] This will delete local content for files confirmed to exist on S3.")
+    if console.input("Are you sure? [y/N]: ").lower() == 'y':
+        service.prune_annex()
+        console.print("[bold green]Prune complete.[/]")
+    else:
+        console.print("[bold]Operation cancelled.[/]")
+
+def run_storage_regen_thumbs(args):
+    cfg = ConfigManager().get_config()
+    from image_annex.core.service import ImageAnnexService
+    service = ImageAnnexService(cfg)
+    console.print("[bold yellow]Regenerating missing thumbnails...[/]")
+    service.regenerate_missing_thumbnails(console.print)
+    console.print("[bold green]Missing thumbnail regeneration complete.[/]")
+
+def run_hub_connect(args):
+    ssh_uri = args.uri
+    # Parse out hostname for the remote name
+    hostname = ssh_uri.split('@')[-1].split(':')[0]
+    
+    console.print(f"[bold cyan]Setting up hub connection to {hostname}...[/]")
+    
+    cfg = ConfigManager().get_config()
+    repo_path = cfg.git_annex_repo.expanded_path
+    
+    console.print("[dim]Adding git remote...[/]")
+    # We ignore errors here in case the remote already exists
+    subprocess.run(["git", "remote", "add", hostname, ssh_uri], cwd=repo_path, capture_output=True)
+    
+    console.print("[dim]Syncing annex state with hub (this may take a moment)...[/]")
+    subprocess.run(["git", "annex", "sync", hostname], cwd=repo_path)
+    
+    console.print(f"[bold green]Successfully connected to {hostname}![/]")
+    console.print(f"\nTo transfer your photos to the hub, run: [bold]git annex copy --to {hostname}[/]")
+    console.print("Once confirmed, you can free up local space with: [bold]git annex drop .[/]")
+
+def main():
+    parser = argparse.ArgumentParser(description="Image Annex CLI", formatter_class=argparse.RawDescriptionHelpFormatter)
+    subparsers = parser.add_subparsers(dest="command", help="Available subcommands")
+
+    subparsers.add_parser("tui", help="Launch the Text User Interface")
+    
+    gallery_parser = subparsers.add_parser("gallery", help="Manage Gallery web server")
+    gallery_sub = gallery_parser.add_subparsers(dest="subcommand")
+    gallery_sub.add_parser("start", help="Start the Gallery server")
+    gallery_sub.add_parser("stop", help="Stop the Gallery server")
+    gallery_sub.add_parser("status", help="Check Gallery server status")
+
+    subparsers.add_parser("ai-scan", help="Run AI image indexing")
+    
+    storage_parser = subparsers.add_parser("storage", help="Manage storage and S3 sync")
+    storage_sub = storage_parser.add_subparsers(dest="subcommand")
+    storage_sub.add_parser("status", help="Show archive storage metrics")
+    storage_sub.add_parser("lock", help="Lock files locally to reclaim space (requires S3 sync)")
+    storage_sub.add_parser("prune", help="Drop local content for files already backed up on S3")
+    storage_sub.add_parser("regen-thumbs", help="Regenerate missing thumbnails")
+
+    hub_parser = subparsers.add_parser("hub", help="Manage connections to a central image-annex hub/appliance")
+    hub_sub = hub_parser.add_subparsers(dest="subcommand")
+    
+    connect_parser = hub_sub.add_parser("connect", help="Connect to an image-annex hub")
+    connect_parser.add_argument("uri", help="SSH URI to the hub (e.g. user@nuc02:/path/to/repo)")
+
+    if len(sys.argv) == 1:
+        parser.print_help()
+        return
+
+    args = parser.parse_args()
+
+    if args.command == "tui":
+        run_tui(args)
+    elif args.command == "gallery":
+        if hasattr(args, 'subcommand'):
+            if args.subcommand == 'start':
+                run_gallery_start(args)
+            elif args.subcommand == 'stop':
+                run_gallery_stop(args)
+            elif args.subcommand == 'status':
+                run_gallery_status(args)
+            else:
+                gallery_parser.print_help()
+        else:
+            gallery_parser.print_help()
+    elif args.command == "ai-scan":
+        run_ai_scan(args)
+    elif args.command == "storage":
+        if hasattr(args, 'subcommand'):
+            if args.subcommand == 'status':
+                run_storage_status(args)
+            elif args.subcommand == 'lock':
+                run_storage_lock(args)
+            elif args.subcommand == 'prune':
+                run_storage_prune(args)
+            elif args.subcommand == 'regen-thumbs':
+                run_storage_regen_thumbs(args)
+            else:
+                storage_parser.print_help()
+        else:
+            storage_parser.print_help()
+    elif args.command == "hub":
+        if hasattr(args, 'subcommand') and args.subcommand == 'connect':
+            run_hub_connect(args)
+        else:
+            hub_parser.print_help()
+    else:
+        parser.print_help()
+
+if __name__ == "__main__":
+    main()

image_annex-0.1.2/src/image_annex/core/ai.py

@@ -0,0 +1,257 @@
+import os
+import time
+import logging
+import sqlite3
+import threading
+from datetime import datetime
+from pathlib import Path
+from typing import List, Optional, Callable
+import psutil # type: ignore
+from PIL import Image
+
+# Setup logging
+ai_logger = logging.getLogger("ai")
+
+class AIService:
+    """Handles model loading and vector embedding generation."""
+    _instance = None
+    _model = None
+    _lock = threading.Lock()
+
+    def __init__(self):
+        # Model name for CLIP Pro (High Accuracy)
+        self.model_name = "clip-ViT-L-14"
+        self.dimensions = 768
+
+    @classmethod
+    def get_instance(cls):
+        with cls._lock:
+            if cls._instance is None:
+                cls._instance = cls()
+            return cls._instance
+
+    def _load_model(self):
+        """Lazy load the model to save startup time and memory if not used."""
+        if self._model is None:
+            from sentence_transformers import SentenceTransformer
+            import torch
+            
+            device = "cuda" if torch.cuda.is_available() else "cpu"
+            ai_logger.info(f"Loading {self.model_name} on {device}...")
+            self._model = SentenceTransformer(self.model_name, device=device)
+            ai_logger.info(f"Model {self.model_name} loaded.")
+        return self._model
+
+    def generate_image_embedding(self, image_path: str) -> List[float]:
+        """Generates a vector embedding for a single image."""
+        model = self._load_model()
+        img = Image.open(image_path)
+        # Sentence-transformers handles the CLIP image processing
+        embedding = model.encode(img)
+        return embedding.tolist()
+
+    def generate_text_embedding(self, text: str) -> List[float]:
+        """Generates a vector embedding for a text query."""
+        model = self._load_model()
+        embedding = model.encode(text)
+        return embedding.tolist()
+
+class AIScraper(threading.Thread):
+    """Background worker that processes images when resources are available."""
+    def __init__(self, db_path: Path, thumb_dir: Path, status_callback: Optional[Callable[[int, int], None]] = None):
+        super().__init__(daemon=True)
+        self.db_path = db_path
+        self.thumb_dir = thumb_dir
+        self.status_callback = status_callback
+        self._stop_event = threading.Event()
+        self.scan_type = "clip_v1"
+        self.model_name = "clip-ViT-L-14"
+
+    def stop(self):
+        self._stop_event.set()
+
+    def _is_system_busy(self) -> bool:
+        """Checks if overall CPU usage is too high (>50%)."""
+        return psutil.cpu_percent(interval=0.1) > 50
+
+    def _set_low_priority(self):
+        """Sets the current process to background/idle priority."""
+        try:
+            p = psutil.Process(os.getpid())
+            if os.name == 'nt':
+                p.nice(psutil.IDLE_PRIORITY_CLASS)
+            else:
+                p.nice(19)
+        except Exception as e:
+            ai_logger.warning(f"Could not set low priority: {e}")
+
+    def run(self):
+        self._set_low_priority()
+        ai_service = AIService.get_instance()
+        
+        while not self._stop_event.is_set():
+            if self._is_system_busy():
+                time.sleep(10)
+                continue
+
+            try:
+                from .config import ConfigManager
+                cfg = ConfigManager().get_config()
+                exclude_paths = [p.lower() for p in cfg.exclude_paths]
+
+                conn = sqlite3.connect(str(self.db_path))
+                conn.row_factory = sqlite3.Row
+                cursor = conn.cursor()
+                
+                cursor.execute("""
+                    SELECT a.key, a.thumbnail_path, i.full_path 
+                    FROM assets a
+                    JOIN instances i ON a.key = i.key
+                    LEFT JOIN asset_scans s ON a.key = s.key AND s.scan_type = ?
+                    WHERE s.key IS NULL AND a.thumbnail_path IS NOT NULL AND a.thumbnail_path != ''
+                    LIMIT 5
+                """, (self.scan_type,))
+                
+                rows = cursor.fetchall()
+                
+                valid_rows = []
+                for row in rows:
+                    full_path = row["full_path"].lower()
+                    if any(full_path.startswith(ex) for ex in exclude_paths):
+                        cursor.execute("INSERT OR REPLACE INTO asset_scans (key, scan_type, status, scanned_at) VALUES (?, ?, ?, ?)",
+                                     (row["key"], self.scan_type, "excluded", datetime.now().isoformat()))
+                        continue
+                    valid_rows.append(row)
+                
+                cursor.execute("SELECT count(*) FROM assets")
+                total_assets = cursor.fetchone()[0]
+                cursor.execute("SELECT count(*) FROM asset_scans WHERE scan_type = ?", (self.scan_type,))
+                processed_assets = cursor.fetchone()[0]
+                
+                if self.status_callback:
+                    self.status_callback(processed_assets, total_assets)
+
+                if not valid_rows:
+                    conn.close()
+                    time.sleep(30)
+                    continue
+
+                for row in valid_rows:
+                    if self._stop_event.is_set():
+                        break
+                    
+                    key = row["key"]
+                    thumb_path = row["thumbnail_path"]
+                    
+                    if not os.path.exists(thumb_path):
+                        cursor.execute("INSERT OR REPLACE INTO asset_scans (key, scan_type, status, scanned_at) VALUES (?, ?, ?, ?)",
+                                     (key, self.scan_type, "error_missing_thumb", datetime.now().isoformat()))
+                        continue
+
+                    try:
+                        embedding = ai_service.generate_image_embedding(thumb_path)
+                        import struct
+                        buf = struct.pack(f"{len(embedding)}f", *embedding)
+                        cursor.execute("INSERT OR REPLACE INTO asset_embeddings (key, embedding) VALUES (?, ?)", (key, buf))
+                        cursor.execute("INSERT OR REPLACE INTO asset_scans (key, scan_type, status, scanned_at, model_name) VALUES (?, ?, ?, ?, ?)",
+                                     (key, self.scan_type, "completed", datetime.now().isoformat(), self.model_name))
+                    except Exception as e:
+                        ai_logger.error(f"AI Scan error for {key}: {e}")
+                        cursor.execute("INSERT OR REPLACE INTO asset_scans (key, scan_type, status, scanned_at) VALUES (?, ?, ?, ?)",
+                                     (key, self.scan_type, f"error: {str(e)}", datetime.now().isoformat()))
+
+                conn.commit()
+                conn.close()
+                time.sleep(1)
+            except Exception as e:
+                ai_logger.error(f"AIScraper loop error: {e}")
+                time.sleep(5)
+
+class TranscriptionService:
+    """Handles audio transcription using faster-whisper."""
+    _instance = None
+    _model = None
+    _lock = threading.Lock()
+
+    def __init__(self):
+        self.model_size = 'base' 
+        self._model = None
+
+    @classmethod
+    def get_instance(cls):
+        with cls._lock:
+            if cls._instance is None:
+                cls._instance = cls()
+            return cls._instance
+
+    def _load_model(self):
+        if self._model is None:
+            from faster_whisper import WhisperModel # type: ignore
+            ai_logger.info(f'Loading Whisper model {self.model_size}...')
+            self._model = WhisperModel(self.model_size, device='cpu', compute_type='int8')
+        return self._model
+
+    def transcribe(self, media_path: str) -> str:
+        model = self._load_model()
+        segments, _ = model.transcribe(media_path, beam_size=5)
+        return ' '.join([segment.text for segment in segments])
+
+class TranscriptionScraper(threading.Thread):
+    """Background worker for transcribing audio/video files."""
+    def __init__(self, db_path: Path, status_callback: Optional[Callable[[int, int], None]] = None):
+        super().__init__(daemon=True)
+        self.db_path = db_path
+        self.status_callback = status_callback
+        self._stop_event = threading.Event()
+        self.scan_type = 'whisper_v1'
+
+    def stop(self):
+        self._stop_event.set()
+
+    def run(self):
+        try:
+            p = psutil.Process(os.getpid())
+            if os.name == 'nt':
+                p.nice(psutil.IDLE_PRIORITY_CLASS)
+        except Exception:
+            pass
+            
+        transcriber = TranscriptionService.get_instance()
+        
+        while not self._stop_event.is_set():
+            time.sleep(10)
+            try:
+                conn = sqlite3.connect(str(self.db_path))
+                conn.row_factory = sqlite3.Row
+                cursor = conn.cursor()
+                
+                cursor.execute('''
+                    SELECT a.key, i.full_path 
+                    FROM assets a
+                    JOIN instances i ON a.key = i.key
+                    LEFT JOIN media_transcripts t ON a.key = t.key
+                    WHERE t.key IS NULL AND (i.full_path LIKE '%.mp4' OR i.full_path LIKE '%.mp3' OR i.full_path LIKE '%.m4a')
+                    LIMIT 1
+                ''')
+                
+                row = cursor.fetchone()
+                if not row:
+                    conn.close()
+                    continue
+                
+                key = row['key']
+                path = row['full_path']
+                
+                transcript = transcriber.transcribe(path)
+                
+                cursor.execute('''
+                    INSERT INTO media_transcripts (key, transcript, language, processed_at)
+                    VALUES (?, ?, ?, ?)
+                ''', (key, transcript, 'en', datetime.now().isoformat()))
+                
+                conn.commit()
+                conn.close()
+                
+            except Exception as e:
+                ai_logger.error(f'Transcription error: {e}')
+                time.sleep(10)