valscanner 0.1.0__py3-none-any.whl

valscanner/__init__.py

@@ -0,0 +1,3 @@
+"""ValScanner — recursive file scanner with metadata, tagging, and duplicate detection."""
+
+__version__ = "0.1.0"

valscanner/cli.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+"""
+ValScanner CLI — scan a directory and build a searchable file database.
+
+Usage:
+    valscanner /path/to/scan [options]
+    valscanner --list-scans --db my.db
+    valscanner --delete-scan 3 --db my.db
+"""
+
+import sys
+import time
+import argparse
+from pathlib import Path
+
+from .core.metadata import PIL_AVAILABLE, MUTAGEN_AVAILABLE, PYPDF_AVAILABLE
+from .core.scanner import scan
+from .core.export import export_csv, export_json
+from .core.db import query_db, print_summary, list_scans, delete_scan
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Scan a directory and build a searchable file database.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument("path",           nargs="?", help="Root directory to scan")
+    parser.add_argument("--db",           default="file_index.db",
+                        help="SQLite database path (default: file_index.db)")
+    parser.add_argument("--label",        metavar="NAME", default="",
+                        help="Human-readable label for this scan")
+    parser.add_argument("--export-csv",   action="store_true", help="Export results to CSV")
+    parser.add_argument("--export-json",  action="store_true", help="Export results to JSON")
+    parser.add_argument("--no-hash",      action="store_true", help="Skip SHA-256 hashing")
+    parser.add_argument("--verbose",      action="store_true", help="Print each file as indexed")
+    parser.add_argument("--query",        metavar="TERM", help="Query the database after scanning")
+    parser.add_argument("--list-scans",   action="store_true", help="List all scans in the database")
+    parser.add_argument("--delete-scan",  type=int, metavar="ID", help="Delete a scan by ID")
+    args = parser.parse_args()
+
+    if args.list_scans:
+        scans = list_scans(args.db)
+        if not scans:
+            print("No scans in database.")
+        for s in scans:
+            print(f"  [{s['id']:3d}]  {s['label'] or s['root']:40s}  "
+                  f"{s['file_count']:>8,} files  {s['total_human']:>10s}  {s['scanned_at']}")
+        sys.exit(0)
+
+    if args.delete_scan is not None:
+        delete_scan(args.db, args.delete_scan)
+        print(f"✓ Scan {args.delete_scan} deleted.")
+        sys.exit(0)
+
+    if not args.path:
+        parser.error("path is required unless using --list-scans or --delete-scan")
+
+    root = Path(args.path).expanduser().resolve()
+    if not root.exists():
+        print(f"Error: path does not exist: {root}")
+        sys.exit(1)
+
+    print(f"\n🔎 Scanning: {root}")
+    print(f"   Database: {args.db}")
+    if args.label:
+        print(f"   Label:    {args.label}")
+    if not PIL_AVAILABLE:
+        print("   ⚠  Pillow not installed — image EXIF metadata skipped")
+    if not MUTAGEN_AVAILABLE:
+        print("   ⚠  mutagen not installed — audio metadata skipped")
+    if not PYPDF_AVAILABLE:
+        print("   ⚠  PyPDF2 not installed — PDF metadata skipped")
+    print()
+
+    t0      = time.time()
+    stats   = scan(root, args.db, compute_hash=not args.no_hash,
+                   verbose=args.verbose, label=args.label)
+    elapsed = time.time() - t0
+
+    print(f"\n✅ Done in {elapsed:.1f}s — "
+          f"scan #{stats['scan_id']}, "
+          f"{stats['scanned']:,} indexed, "
+          f"{stats['errors']:,} errors, "
+          f"{stats['skipped']:,} skipped")
+
+    print_summary(args.db)
+
+    if args.export_csv:
+        export_csv(args.db, args.db.replace(".db", ".csv"), scan_id=stats["scan_id"])
+    if args.export_json:
+        export_json(args.db, args.db.replace(".db", ".json"), scan_id=stats["scan_id"])
+    if args.query:
+        query_db(args.db, args.query)
+
+    print(f"  💡 Run with --query photos, --list-scans, or open the GUI:\n"
+          f"     valscanner-gui\n")
+
+
+if __name__ == "__main__":
+    main()

valscanner/core/categories.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+MIME_CATEGORY: dict[str, str] = {
+    "image":       "photo",
+    "video":       "video",
+    "audio":       "audio",
+    "text":        "document",
+    "application": "application",
+}
+
+EXT_CATEGORY: dict[str, str] = {
+    # Documents
+    ".pdf": "document", ".doc": "document", ".docx": "document",
+    ".odt": "document", ".rtf": "document", ".txt": "document",
+    ".md":  "document", ".rst": "document", ".tex": "document",
+    # Spreadsheets
+    ".xls": "spreadsheet", ".xlsx": "spreadsheet", ".ods": "spreadsheet",
+    ".csv": "spreadsheet",
+    # Presentations
+    ".ppt": "presentation", ".pptx": "presentation", ".odp": "presentation",
+    # Photos / images
+    ".jpg": "photo", ".jpeg": "photo", ".png": "photo", ".gif": "photo",
+    ".bmp": "photo", ".tiff": "photo", ".tif": "photo", ".webp": "photo",
+    ".heic": "photo", ".heif": "photo", ".raw": "photo", ".cr2": "photo",
+    ".nef": "photo", ".svg": "image",
+    # Video
+    ".mp4": "video", ".mov": "video", ".avi": "video", ".mkv": "video",
+    ".wmv": "video", ".flv": "video", ".webm": "video", ".m4v": "video",
+    # Audio
+    ".mp3": "audio", ".flac": "audio", ".wav": "audio", ".aac": "audio",
+    ".ogg": "audio", ".m4a": "audio", ".wma": "audio",
+    # Code
+    ".py": "code",  ".js": "code",  ".ts": "code",  ".java": "code",
+    ".c":  "code",  ".cpp": "code", ".h":  "code",  ".cs": "code",
+    ".go": "code",  ".rs": "code",  ".rb": "code",  ".php": "code",
+    ".sh": "code",  ".bat": "code", ".ps1": "code",
+    # Data
+    ".json": "data", ".xml": "data", ".yaml": "data", ".yml": "data",
+    ".toml": "data", ".ini": "data", ".sql": "data",
+    # Archives
+    ".zip": "archive", ".tar": "archive", ".gz": "archive", ".bz2": "archive",
+    ".7z": "archive",  ".rar": "archive", ".xz": "archive",
+    # Executables / installers
+    ".exe": "executable", ".msi": "executable", ".dmg": "executable",
+    ".deb": "executable", ".rpm": "executable", ".appimage": "executable",
+    # Fonts
+    ".ttf": "font", ".otf": "font", ".woff": "font", ".woff2": "font",
+    # Ebooks
+    ".epub": "ebook", ".mobi": "ebook", ".azw": "ebook",
+}

valscanner/core/db.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+import sqlite3
+from collections import Counter
+
+from .schema import human_size
+
+
+def list_scans(db_path: str) -> list[dict]:
+    conn = sqlite3.connect(db_path)
+    conn.row_factory = sqlite3.Row
+    try:
+        rows = conn.execute(
+            "SELECT id, label, root, scanned_at, file_count, total_bytes, total_human "
+            "FROM scans ORDER BY id"
+        ).fetchall()
+    except sqlite3.OperationalError:
+        rows = []
+    conn.close()
+    return [dict(r) for r in rows]
+
+
+def delete_scan(db_path: str, scan_id: int) -> None:
+    conn = sqlite3.connect(db_path)
+    conn.execute("PRAGMA foreign_keys = ON")
+    conn.execute("DELETE FROM scans WHERE id = ?", (scan_id,))
+    conn.commit()
+    conn.close()
+
+
+def query_db(db_path: str, term: str) -> None:
+    conn = sqlite3.connect(db_path)
+    conn.row_factory = sqlite3.Row
+    print(f"\n🔍 Searching for: '{term}'\n{'─'*60}")
+
+    try:
+        fts_rows = conn.execute(
+            "SELECT path, category, size_human, tags FROM files "
+            "WHERE id IN (SELECT rowid FROM files_fts WHERE files_fts MATCH ?) "
+            "ORDER BY path LIMIT 50",
+            (term,),
+        ).fetchall()
+    except sqlite3.OperationalError:
+        fts_rows = []
+
+    like = f"%{term}%"
+    like_rows = conn.execute(
+        "SELECT path, category, size_human, tags FROM files "
+        "WHERE path LIKE ? OR category LIKE ? OR tags LIKE ? ORDER BY path LIMIT 50",
+        (like, like, like),
+    ).fetchall()
+
+    seen: set[str] = set()
+    total = 0
+    for row in list(fts_rows) + list(like_rows):
+        p = row["path"]
+        if p in seen:
+            continue
+        seen.add(p)
+        total += 1
+        print(f"  {row['category']:14s}  {row['size_human']:>10s}  {p}")
+        print(f"  {'':14s}  tags: {row['tags']}\n")
+
+    print(f"{'─'*60}\n  Found {total} result(s).")
+    conn.close()
+
+
+def print_summary(db_path: str) -> None:
+    conn = sqlite3.connect(db_path)
+    total,      = conn.execute("SELECT COUNT(*) FROM files").fetchone()
+    total_size, = conn.execute("SELECT SUM(size_bytes) FROM files").fetchone()
+    total_size  = total_size or 0
+
+    scans = conn.execute(
+        "SELECT id, label, root, scanned_at, file_count, total_human FROM scans ORDER BY id"
+    ).fetchall()
+
+    print(f"\n{'═'*60}")
+    if len(scans) > 1:
+        print(f"  📦  Scans in database: {len(scans)}")
+        for sid, slabel, sroot, sat, sfc, sth in scans:
+            print(f"    [{sid}] {slabel or sroot}  —  {sfc:,} files  {sth}  ({sat})")
+        print()
+
+    print(f"  📁  Total files indexed : {total:,}")
+    print(f"  💾  Total size          : {human_size(total_size)}")
+    print(f"\n  Files by category:")
+    for cat, cnt, sz in conn.execute(
+        "SELECT category, COUNT(*), SUM(size_bytes) FROM files "
+        "GROUP BY category ORDER BY 2 DESC"
+    ):
+        print(f"    {cat:20s} {cnt:>6,} files   {human_size(sz or 0):>10s}")
+
+    print(f"\n  Top 10 most common extensions:")
+    for ext, cnt in conn.execute(
+        "SELECT extension, COUNT(*) AS cnt FROM files "
+        "GROUP BY extension ORDER BY 2 DESC LIMIT 10"
+    ):
+        print(f"    {ext:12s}  {cnt:>6,}")
+
+    print(f"\n  Top 10 tags:")
+    all_tags = conn.execute("SELECT tags FROM files WHERE tags != ''").fetchall()
+    tag_counter: Counter = Counter()
+    for (tag_str,) in all_tags:
+        for t in tag_str.split(", "):
+            tag_counter[t.strip()] += 1
+    for tag, cnt in tag_counter.most_common(10):
+        print(f"    {tag:30s}  {cnt:>6,}")
+
+    print(f"\n  Top 10 largest folders (cumulative):")
+    for fpath_str, tb, fc in conn.execute(
+        "SELECT path, SUM(total_bytes), SUM(file_count) FROM folders "
+        "GROUP BY path ORDER BY 2 DESC LIMIT 10"
+    ):
+        print(f"    {human_size(tb):>10s}  ({fc:,} files)  {fpath_str}")
+
+    print(f"{'═'*60}\n")
+    conn.close()

valscanner/core/export.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+import csv
+import json
+import sqlite3
+
+
+def export_csv(db_path: str, out_path: str, scan_id: int | None = None) -> None:
+    conn = sqlite3.connect(db_path)
+    conn.row_factory = sqlite3.Row
+    q    = "SELECT * FROM files" + (" WHERE scan_id=?" if scan_id else "") + " ORDER BY path"
+    rows = conn.execute(q, (scan_id,) if scan_id else ()).fetchall()
+    if not rows:
+        print("No rows to export.")
+        conn.close()
+        return
+    with open(out_path, "w", newline="", encoding="utf-8") as f:
+        w = csv.DictWriter(f, fieldnames=rows[0].keys())
+        w.writeheader()
+        w.writerows([dict(r) for r in rows])
+    conn.close()
+    print(f"✓ CSV exported → {out_path}")
+
+
+def export_json(db_path: str, out_path: str, scan_id: int | None = None) -> None:
+    conn = sqlite3.connect(db_path)
+    conn.row_factory = sqlite3.Row
+    q    = "SELECT * FROM files" + (" WHERE scan_id=?" if scan_id else "") + " ORDER BY path"
+    rows = [dict(r) for r in conn.execute(q, (scan_id,) if scan_id else ())]
+    conn.close()
+    with open(out_path, "w", encoding="utf-8") as f:
+        json.dump(rows, f, indent=2, ensure_ascii=False)
+    print(f"✓ JSON exported → {out_path}")

valscanner/core/metadata.py

@@ -0,0 +1,175 @@
+from __future__ import annotations
+import io
+import hashlib
+import shutil
+import subprocess
+from pathlib import Path
+
+try:
+    from PIL import Image
+    from PIL.ExifTags import TAGS
+    PIL_AVAILABLE = True
+except ImportError:
+    PIL_AVAILABLE = False
+
+try:
+    import mutagen
+    MUTAGEN_AVAILABLE = True
+except ImportError:
+    MUTAGEN_AVAILABLE = False
+
+try:
+    import PyPDF2
+    PYPDF_AVAILABLE = True
+except ImportError:
+    PYPDF_AVAILABLE = False
+
+FFMPEG_AVAILABLE = bool(shutil.which("ffmpeg"))
+
+
+def extract_image_metadata(path: Path) -> dict:
+    if not PIL_AVAILABLE:
+        return {}
+    try:
+        with Image.open(path) as img:
+            meta = {
+                "img_width":  img.width,
+                "img_height": img.height,
+                "img_mode":   img.mode,
+                "img_format": img.format,
+            }
+            exif_data = img._getexif() if hasattr(img, "_getexif") else None
+            if exif_data:
+                readable = {}
+                for tag_id, value in exif_data.items():
+                    tag = TAGS.get(tag_id, str(tag_id))
+                    if isinstance(value, (str, int, float)):
+                        readable[tag] = value
+                if "DateTime" in readable:
+                    meta["exif_datetime"] = readable["DateTime"]
+                if "Make" in readable:
+                    meta["exif_camera_make"] = readable["Make"]
+                if "Model" in readable:
+                    meta["exif_camera_model"] = readable["Model"]
+                if "GPSInfo" in exif_data:
+                    meta["has_gps"] = True
+            return meta
+    except Exception:
+        return {}
+
+
+def extract_audio_metadata(path: Path) -> dict:
+    if not MUTAGEN_AVAILABLE:
+        return {}
+    try:
+        audio = mutagen.File(path, easy=True)
+        if audio is None:
+            return {}
+        meta = {}
+        for key in ("title", "artist", "album", "date", "genre", "tracknumber"):
+            if key in audio:
+                meta[f"audio_{key}"] = ", ".join(audio[key])
+        if hasattr(audio, "info"):
+            info = audio.info
+            if hasattr(info, "length"):
+                meta["audio_duration_sec"] = round(info.length, 2)
+            if hasattr(info, "bitrate"):
+                meta["audio_bitrate"] = info.bitrate
+        return meta
+    except Exception:
+        return {}
+
+
+def extract_pdf_metadata(path: Path) -> dict:
+    if not PYPDF_AVAILABLE:
+        return {}
+    try:
+        with open(path, "rb") as f:
+            reader = PyPDF2.PdfReader(f)
+            meta = {"pdf_pages": len(reader.pages)}
+            info = reader.metadata
+            if info:
+                for k in ("/Title", "/Author", "/Subject", "/Creator", "/CreationDate"):
+                    v = info.get(k)
+                    if v:
+                        meta[f"pdf_{k.strip('/').lower()}"] = str(v)
+            return meta
+    except Exception:
+        return {}
+
+
+def file_sha256(path: Path, block: int = 65536) -> str:
+    h = hashlib.sha256()
+    try:
+        with open(path, "rb") as f:
+            while chunk := f.read(block):
+                h.update(chunk)
+        return h.hexdigest()
+    except Exception:
+        return ""
+
+
+def _thumb_image(fpath: Path, max_size: int, quality: int) -> bytes | None:
+    if not PIL_AVAILABLE:
+        return None
+    try:
+        img = Image.open(fpath)
+        if img.mode == "P" and "transparency" in img.info:
+            img = img.convert("RGBA")
+        img.thumbnail((max_size, max_size), Image.LANCZOS)
+        buf = io.BytesIO()
+        img.convert("RGB").save(buf, format="JPEG", quality=quality, optimize=True)
+        return buf.getvalue()
+    except Exception:
+        return None
+
+
+def _thumb_video(fpath: Path, max_size: int, quality: int) -> bytes | None:
+    if not FFMPEG_AVAILABLE:
+        return None
+    try:
+        result = subprocess.run(
+            [
+                "ffmpeg", "-y", "-i", str(fpath),
+                "-ss", "00:00:01",
+                "-vframes", "1",
+                "-vf", f"scale={max_size}:{max_size}:force_original_aspect_ratio=decrease",
+                "-q:v", str(max(1, (100 - quality) // 10)),
+                "-f", "image2pipe", "-vcodec", "mjpeg", "pipe:1",
+            ],
+            capture_output=True, timeout=30,
+        )
+        return result.stdout if result.returncode == 0 and result.stdout else None
+    except Exception:
+        return None
+
+
+def _sample_media(fpath: Path, category: str, duration: int) -> tuple[bytes, str] | None:
+    if not FFMPEG_AVAILABLE:
+        return None
+    try:
+        if category == "video":
+            cmd = [
+                "ffmpeg", "-y", "-i", str(fpath),
+                "-t", str(duration),
+                "-vf", "scale=320:-2",
+                "-c:v", "libx264", "-crf", "35", "-preset", "ultrafast",
+                "-c:a", "aac", "-b:a", "32k",
+                "-movflags", "frag_keyframe+empty_moov",
+                "-f", "mp4", "pipe:1",
+            ]
+            fmt = "mp4"
+        else:
+            cmd = [
+                "ffmpeg", "-y", "-i", str(fpath),
+                "-t", str(duration),
+                "-c:a", "libmp3lame", "-b:a", "32k",
+                "-f", "mp3", "pipe:1",
+            ]
+            fmt = "mp3"
+        result = subprocess.run(cmd, capture_output=True, timeout=60)
+        if result.returncode == 0 and result.stdout:
+            return result.stdout, fmt
+    except Exception:
+        pass
+    return None