alysis-code 0.13.0__py3-none-any.whl

alysis_code/assets/ingestion.py

@@ -0,0 +1,275 @@
+from __future__ import annotations
+
+import hashlib
+import mimetypes
+import os
+import shutil
+import tempfile
+import uuid
+from contextlib import suppress
+from dataclasses import replace
+from pathlib import Path
+from typing import Any, Literal
+
+from ..forge import RunPaths, now_iso
+from .index import AssetIndex
+from .models import AssetAlreadyExistsError, AssetError, AssetRecord
+from .paths import asset_raw_path, asset_thumbnail_path, repo_rel
+
+_TEXT_SUFFIXES = {
+    ".txt",
+    ".md",
+    ".markdown",
+    ".json",
+    ".yaml",
+    ".yml",
+    ".toml",
+    ".ini",
+    ".cfg",
+    ".csv",
+    ".tsv",
+    ".log",
+    ".py",
+    ".js",
+    ".jsx",
+    ".ts",
+    ".tsx",
+    ".html",
+    ".css",
+    ".xml",
+    ".sql",
+    ".sh",
+    ".ps1",
+}
+_IMAGE_THUMB_MAX_EDGE = 512
+
+
+def ingest_asset(
+    path: Path,
+    *,
+    title: str,
+    description: str = "",
+    run_paths: RunPaths,
+    pinned: bool = False,
+    added_by: dict[str, Any] | None = None,
+    dedupe_policy: Literal["reject", "link"] = "reject",
+) -> AssetRecord:
+    source = path.expanduser().resolve()
+    clean_title = title.strip()
+    if not clean_title:
+        raise AssetError("Asset title is required.")
+    if not source.exists():
+        raise AssetError(f"Asset file does not exist: {source}")
+    if not source.is_file():
+        raise AssetError(f"Asset path is not a file: {source}")
+
+    raw = source.read_bytes()
+    digest = hashlib.sha256(raw).hexdigest()
+    index = AssetIndex(run_paths)
+    existing = index.find_by_sha256(digest)
+    if existing is not None:
+        if dedupe_policy == "link":
+            return existing
+        raise AssetAlreadyExistsError(existing.id)
+
+    kind, mime = _classify_asset(source, raw)
+    staging_dir = _staging_dir(run_paths)
+    staged_raw = staging_dir / source.name
+    staged_thumbnail = staging_dir / "thumbnail.png"
+    try:
+        _atomic_write_bytes(staged_raw, raw)
+        if kind == "image":
+            _write_thumbnail(source, staged_thumbnail)
+        return index.add_with_publish(
+            sha256=digest,
+            dedupe_policy=dedupe_policy,
+            build_record=lambda existing_ids: _build_record(
+                run_paths=run_paths,
+                existing_ids=existing_ids,
+                digest=digest,
+                title=clean_title,
+                description=description,
+                kind=kind,
+                mime=mime,
+                original_filename=source.name,
+                size_bytes=len(raw),
+                pinned=pinned,
+                added_by=added_by,
+            ),
+            publish=lambda record: _publish_staged_asset(
+                run_paths=run_paths,
+                record=record,
+                staged_raw=staged_raw,
+                staged_thumbnail=staged_thumbnail if kind == "image" else None,
+            ),
+            rollback=lambda record: _rollback_published_asset(run_paths, record),
+        )
+    finally:
+        with suppress(OSError):
+            shutil.rmtree(staging_dir)
+
+
+def _staging_dir(run_paths: RunPaths) -> Path:
+    return run_paths.assets_raw_dir / ".staging" / uuid.uuid4().hex
+
+
+def _build_record(
+    *,
+    run_paths: RunPaths,
+    existing_ids: set[str],
+    digest: str,
+    title: str,
+    description: str,
+    kind: Literal["text", "image"],
+    mime: str,
+    original_filename: str,
+    size_bytes: int,
+    pinned: bool,
+    added_by: dict[str, Any] | None,
+) -> AssetRecord:
+    asset_id = _asset_id_for_sha256(digest, existing_ids)
+    stored_path = asset_raw_path(run_paths, asset_id, original_filename)
+    record = AssetRecord(
+        id=asset_id,
+        title=title,
+        description=description,
+        kind=kind,
+        mime=mime,
+        original_filename=original_filename,
+        size_bytes=size_bytes,
+        sha256=digest,
+        stored_path=repo_rel(run_paths.root, stored_path),
+        extracted_text_path=None,
+        thumbnail_path=None,
+        pinned=bool(pinned),
+        added_at=now_iso(),
+        added_by=dict(added_by or {}),
+        deleted_at=None,
+        comprehension_status="pending",
+        comprehension_current_version=None,
+    )
+    if kind == "text":
+        return _with_text_paths(record)
+    return _with_thumbnail(
+        record, repo_rel(run_paths.root, asset_thumbnail_path(run_paths, asset_id))
+    )
+
+
+def _publish_staged_asset(
+    *,
+    run_paths: RunPaths,
+    record: AssetRecord,
+    staged_raw: Path,
+    staged_thumbnail: Path | None,
+) -> None:
+    final_raw = run_paths.root / record.stored_path
+    final_raw.parent.mkdir(parents=True, exist_ok=True)
+    os.replace(staged_raw, final_raw)
+    if staged_thumbnail is not None and record.thumbnail_path is not None:
+        final_thumbnail = run_paths.root / record.thumbnail_path
+        final_thumbnail.parent.mkdir(parents=True, exist_ok=True)
+        os.replace(staged_thumbnail, final_thumbnail)
+
+
+def _rollback_published_asset(run_paths: RunPaths, record: AssetRecord) -> None:
+    with suppress(OSError):
+        shutil.rmtree((run_paths.root / record.stored_path).parent)
+    if record.thumbnail_path is not None:
+        with suppress(FileNotFoundError):
+            (run_paths.root / record.thumbnail_path).unlink()
+
+
+def _with_text_paths(record: AssetRecord) -> AssetRecord:
+    return replace(record, extracted_text_path=record.stored_path)
+
+
+def _with_thumbnail(record: AssetRecord, thumbnail_path: str) -> AssetRecord:
+    return replace(record, thumbnail_path=thumbnail_path)
+
+
+def _classify_asset(source: Path, raw: bytes) -> tuple[Literal["text", "image"], str]:
+    mime, _ = mimetypes.guess_type(source.name)
+    suffix = source.suffix.lower()
+    if mime and mime.startswith("image/"):
+        _ensure_pillow_can_open(source)
+        return "image", mime
+    if mime and mime.startswith("text/"):
+        _ensure_utf8_text(raw, source)
+        return "text", mime
+    if suffix in _TEXT_SUFFIXES:
+        _ensure_utf8_text(raw, source)
+        return "text", mime or "text/plain"
+    raise AssetError(f"Unsupported asset file type: {source.name}")
+
+
+def _ensure_utf8_text(raw: bytes, source: Path) -> None:
+    if b"\x00" in raw:
+        raise AssetError(f"Unsupported binary text asset: {source.name}")
+    try:
+        raw.decode("utf-8")
+    except UnicodeDecodeError as e:
+        raise AssetError(f"Text asset must be UTF-8: {source.name}") from e
+
+
+def _ensure_pillow_can_open(source: Path) -> None:
+    try:
+        from PIL import Image
+    except ModuleNotFoundError as e:
+        raise AssetError("Pillow is required for image asset ingestion.") from e
+    try:
+        with Image.open(source) as image:
+            image.verify()
+    except Exception as e:  # noqa: BLE001 - Pillow raises multiple format-specific errors
+        raise AssetError(f"Unsupported image asset: {source.name}") from e
+
+
+def _write_thumbnail(source: Path, destination: Path) -> None:
+    try:
+        from PIL import Image
+    except ModuleNotFoundError as e:
+        raise AssetError("Pillow is required for image asset thumbnails.") from e
+    with Image.open(source) as image:
+        normalized = image.convert("RGB")
+        normalized.thumbnail((_IMAGE_THUMB_MAX_EDGE, _IMAGE_THUMB_MAX_EDGE))
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        fd, temp_name = tempfile.mkstemp(
+            dir=destination.parent,
+            prefix=f".{destination.name}.",
+            suffix=".tmp",
+        )
+        temp_path = Path(temp_name)
+        os.close(fd)
+        try:
+            normalized.save(temp_path, format="PNG")
+            os.replace(temp_path, destination)
+        finally:
+            with suppress(FileNotFoundError):
+                temp_path.unlink()
+
+
+def _atomic_write_bytes(path: Path, data: bytes) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    fd, temp_name = tempfile.mkstemp(
+        dir=path.parent,
+        prefix=f".{path.name}.",
+        suffix=".tmp",
+    )
+    temp_path = Path(temp_name)
+    try:
+        with os.fdopen(fd, "wb") as handle:
+            handle.write(data)
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(temp_path, path)
+    finally:
+        with suppress(FileNotFoundError):
+            temp_path.unlink()
+
+
+def _asset_id_for_sha256(sha256: str, existing_ids: set[str]) -> str:
+    digest = sha256.strip().lower()
+    for offset in range(0, max(len(digest) - 7, 1), 8):
+        candidate = f"ast_{digest[offset : offset + 8]}"
+        if len(candidate) == 12 and candidate not in existing_ids:
+            return candidate
+    raise AssetError("Unable to allocate stable asset id without collision.")

alysis_code/assets/legacy_migration.py

@@ -0,0 +1,413 @@
+from __future__ import annotations
+
+import errno
+import hashlib
+import json
+import logging
+import os
+import socket
+import time
+from contextlib import suppress
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Literal
+
+from ..config import AppConfig
+from ..forge import RunPaths, now_iso
+from .models import AssetAlreadyExistsError, AssetError
+from .surface import AssetSurface
+
+LOGGER = logging.getLogger(__name__)
+
+PLAN_SCHEMA_VERSION_V1 = 1
+PLAN_SCHEMA_VERSION_V2 = 2
+_LOCK_POLL_SECONDS = 0.05
+_LOCK_WAIT_SECONDS = 30.0
+
+
+@dataclass(frozen=True)
+class MigratedLegacyAsset:
+    legacy_stored_path: str
+    legacy_text_copy_path: str | None
+    new_asset_id: str
+    title: str
+    description: str
+    comprehension_handle_running: bool
+
+
+@dataclass(frozen=True)
+class LegacyAssetMigrationResult:
+    schema_version_before: int
+    schema_version_after: int
+    migrated_assets: list[MigratedLegacyAsset]
+    skipped_existing: list[str]
+    failed: list[tuple[str, str]]
+    plan_assets_array_cleared: bool
+    plan_v2_written: bool
+
+
+def migrate_legacy_assets(
+    *,
+    cfg: AppConfig,
+    run_paths: RunPaths,
+    surface: AssetSurface,
+    comprehend_mode: Literal["sync", "async", "skip"] = "async",
+) -> LegacyAssetMigrationResult:
+    plan = _read_plan(run_paths.plan_json_path)
+    schema_before = _schema_version(plan)
+    if schema_before >= PLAN_SCHEMA_VERSION_V2:
+        return LegacyAssetMigrationResult(
+            schema_version_before=schema_before,
+            schema_version_after=schema_before,
+            migrated_assets=[],
+            skipped_existing=[],
+            failed=[],
+            plan_assets_array_cleared=False,
+            plan_v2_written=False,
+        )
+
+    with LegacyMigrationLock(run_paths):
+        plan = _read_plan(run_paths.plan_json_path)
+        schema_before = _schema_version(plan)
+        if schema_before >= PLAN_SCHEMA_VERSION_V2:
+            return LegacyAssetMigrationResult(
+                schema_version_before=schema_before,
+                schema_version_after=schema_before,
+                migrated_assets=[],
+                skipped_existing=[],
+                failed=[],
+                plan_assets_array_cleared=False,
+                plan_v2_written=False,
+            )
+        result = _migrate_locked(
+            cfg=cfg,
+            run_paths=run_paths,
+            surface=surface,
+            plan=plan,
+            schema_before=schema_before,
+            comprehend_mode="skip" if not cfg.assets.enabled else comprehend_mode,
+        )
+        return result
+
+
+class LegacyMigrationLock:
+    def __init__(self, run_paths: RunPaths) -> None:
+        self.path = run_paths.run_dir / "legacy_migration.lock"
+        self._fd: int | None = None
+
+    def __enter__(self) -> LegacyMigrationLock:
+        deadline = time.monotonic() + _LOCK_WAIT_SECONDS
+        while True:
+            try:
+                self.path.parent.mkdir(parents=True, exist_ok=True)
+                fd = os.open(self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600)
+            except FileExistsError:
+                if self._recover_stale_lock():
+                    continue
+                if time.monotonic() >= deadline:
+                    raise AssetError(
+                        f"Timed out waiting for legacy asset migration lock: {self.path}"
+                    ) from None
+                time.sleep(_LOCK_POLL_SECONDS)
+                continue
+            self._fd = fd
+            payload = {
+                "pid": os.getpid(),
+                "hostname": socket.gethostname(),
+                "created_at": now_iso(),
+                "created_epoch": time.time(),
+            }
+            os.write(fd, (json.dumps(payload, sort_keys=True) + "\n").encode("utf-8"))
+            os.fsync(fd)
+            return self
+
+    def __exit__(self, exc_type: object, exc: object, tb: object) -> None:
+        if self._fd is not None:
+            os.close(self._fd)
+            self._fd = None
+        with suppress(FileNotFoundError):
+            self.path.unlink()
+
+    def _recover_stale_lock(self) -> bool:
+        try:
+            payload = json.loads(self.path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError):
+            return False
+        if not isinstance(payload, dict):
+            return False
+        created_epoch = _safe_float(payload.get("created_epoch"))
+        if created_epoch is None or time.time() - created_epoch < _LOCK_WAIT_SECONDS:
+            return False
+        hostname = str(payload.get("hostname") or "").strip()
+        if hostname and hostname != socket.gethostname():
+            return False
+        pid = _safe_int(payload.get("pid"))
+        if pid is not None and _process_is_running(pid) is not False:
+            return False
+        with suppress(FileNotFoundError):
+            self.path.unlink()
+        return True
+
+
+def _migrate_locked(
+    *,
+    cfg: AppConfig,
+    run_paths: RunPaths,
+    surface: AssetSurface,
+    plan: dict[str, Any],
+    schema_before: int,
+    comprehend_mode: Literal["sync", "async", "skip"],
+) -> LegacyAssetMigrationResult:
+    legacy_assets = plan.get("assets")
+    if not isinstance(legacy_assets, list):
+        raise AssetError("Invalid legacy plan: 'assets' must be an array before migration.")
+    title_counts: dict[str, int] = {}
+    migrated: list[MigratedLegacyAsset] = []
+    skipped: list[str] = []
+    failed: list[tuple[str, str]] = []
+
+    for entry in legacy_assets:
+        if not isinstance(entry, dict):
+            failed.append(("(invalid legacy asset entry)", "Legacy asset entry must be an object"))
+            continue
+        stored_path = str(entry.get("stored_path") or "").strip()
+        text_copy_path = _optional_string(entry.get("text_copy_path"))
+        if not stored_path:
+            failed.append(("(missing stored_path)", "Legacy asset is missing stored_path"))
+            continue
+        source = (run_paths.root / stored_path).resolve()
+        try:
+            _ensure_under_root(run_paths.root, source)
+            digest = hashlib.sha256(source.read_bytes()).hexdigest()
+        except OSError as exc:
+            failed.append((stored_path, str(exc)))
+            continue
+        except AssetError as exc:
+            failed.append((stored_path, str(exc)))
+            continue
+
+        existing = surface.index.find_by_sha256(digest, include_deleted=False)
+        if existing is not None:
+            skipped.append(stored_path)
+            LOGGER.info(
+                "legacy_asset_migration skipped existing legacy_stored_path=%s asset_id=%s",
+                stored_path,
+                existing.id,
+            )
+            continue
+
+        title = _deduped_title(_derive_title(source), title_counts)
+        try:
+            add_result = surface.add_asset(
+                source,
+                title=title,
+                description="",
+                pinned=False,
+                added_by={
+                    "phase": "legacy_migration",
+                    "schema_version_before": schema_before,
+                    "legacy_stored_path": stored_path,
+                },
+                comprehend=comprehend_mode,
+                dedupe_policy="link",
+            )
+        except AssetAlreadyExistsError as exc:
+            skipped.append(stored_path)
+            LOGGER.info(
+                "legacy_asset_migration skipped dedupe legacy_stored_path=%s asset_id=%s",
+                stored_path,
+                exc.existing_id,
+            )
+            continue
+        except AssetError as exc:
+            failed.append((stored_path, str(exc)))
+            continue
+        handle = add_result.comprehension_handle
+        migrated.append(
+            MigratedLegacyAsset(
+                legacy_stored_path=stored_path,
+                legacy_text_copy_path=text_copy_path,
+                new_asset_id=add_result.record.id,
+                title=add_result.record.title,
+                description=add_result.record.description,
+                comprehension_handle_running=bool(handle is not None and not handle.is_done()),
+            )
+        )
+        LOGGER.info(
+            "legacy_asset_migration migrated legacy_stored_path=%s asset_id=%s comprehend=%s",
+            stored_path,
+            add_result.record.id,
+            comprehend_mode,
+        )
+
+    if failed:
+        return LegacyAssetMigrationResult(
+            schema_version_before=schema_before,
+            schema_version_after=schema_before,
+            migrated_assets=migrated,
+            skipped_existing=skipped,
+            failed=failed,
+            plan_assets_array_cleared=False,
+            plan_v2_written=False,
+        )
+
+    plan["schema_version"] = PLAN_SCHEMA_VERSION_V2
+    plan["assets"] = []
+    plan["legacy_assets_migrated_at"] = now_iso()
+    _bind_migrated_assets_to_matching_tasks(plan=plan, surface=surface, migrated=migrated)
+    from ..forge import save_plan
+
+    save_plan(run_paths, plan)
+    return LegacyAssetMigrationResult(
+        schema_version_before=schema_before,
+        schema_version_after=PLAN_SCHEMA_VERSION_V2,
+        migrated_assets=migrated,
+        skipped_existing=skipped,
+        failed=[],
+        plan_assets_array_cleared=True,
+        plan_v2_written=True,
+    )
+
+
+def _bind_migrated_assets_to_matching_tasks(
+    *,
+    plan: dict[str, Any],
+    surface: AssetSurface,
+    migrated: list[MigratedLegacyAsset],
+) -> None:
+    if not migrated:
+        return
+    from .plan_binding import bind_asset_to_matching_tasks
+
+    try:
+        active_records = surface.index.records(include_deleted=False)
+    except AssetError:
+        return
+    by_id = {record.id: record for record in active_records}
+    for item in migrated:
+        record = by_id.get(item.new_asset_id)
+        if record is None:
+            continue
+        bind_asset_to_matching_tasks(
+            plan=plan,
+            record=record,
+            active_records=active_records,
+        )
+
+
+def _read_plan(path: Path) -> dict[str, Any]:
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except OSError as exc:
+        raise AssetError(f"Failed to read plan for legacy migration: {path}") from exc
+    except json.JSONDecodeError as exc:
+        raise AssetError(f"Invalid plan JSON for legacy migration: {path}") from exc
+    if not isinstance(payload, dict):
+        raise AssetError(f"Invalid plan structure for legacy migration: {path}")
+    return payload
+
+
+def _schema_version(plan: dict[str, Any]) -> int:
+    try:
+        return int(plan.get("schema_version", PLAN_SCHEMA_VERSION_V1) or PLAN_SCHEMA_VERSION_V1)
+    except (TypeError, ValueError):
+        return PLAN_SCHEMA_VERSION_V1
+
+
+def _derive_title(source: Path) -> str:
+    stem = source.stem.replace("_", " ").replace("-", " ").strip().casefold()
+    stem = " ".join(stem.split())
+    if not stem:
+        return "Legacy asset"
+    return stem[:1].upper() + stem[1:]
+
+
+def _deduped_title(title: str, counts: dict[str, int]) -> str:
+    count = counts.get(title, 0) + 1
+    counts[title] = count
+    return title if count == 1 else f"{title} {count}"
+
+
+def _optional_string(value: Any) -> str | None:
+    if value is None:
+        return None
+    text = str(value).strip()
+    return text or None
+
+
+def _safe_int(value: Any) -> int | None:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _safe_float(value: Any) -> float | None:
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _process_is_running(pid: int) -> bool | None:
+    try:
+        if pid <= 0:
+            return False
+    except TypeError:
+        return None
+    if os.name == "nt":
+        return _windows_process_is_running(pid)
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True
+    except OSError as exc:
+        if getattr(exc, "errno", None) == errno.ESRCH:
+            return False
+        return None
+    return True
+
+
+def _windows_process_is_running(pid: int) -> bool | None:
+    try:
+        import ctypes
+        from ctypes import wintypes
+
+        process_query_limited_information = 0x1000
+        still_active = 259
+        error_invalid_parameter = 87
+        kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+        kernel32.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD]
+        kernel32.OpenProcess.restype = wintypes.HANDLE
+        kernel32.GetExitCodeProcess.argtypes = [
+            wintypes.HANDLE,
+            ctypes.POINTER(wintypes.DWORD),
+        ]
+        kernel32.GetExitCodeProcess.restype = wintypes.BOOL
+        kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
+        kernel32.CloseHandle.restype = wintypes.BOOL
+
+        handle = kernel32.OpenProcess(process_query_limited_information, False, pid)
+        if not handle:
+            error = ctypes.get_last_error()
+            if error == error_invalid_parameter:
+                return False
+            return None
+        exit_code = wintypes.DWORD()
+        try:
+            if not kernel32.GetExitCodeProcess(handle, ctypes.byref(exit_code)):
+                return None
+            return exit_code.value == still_active
+        finally:
+            kernel32.CloseHandle(handle)
+    except Exception:
+        return None
+
+
+def _ensure_under_root(root: Path, path: Path) -> None:
+    try:
+        path.resolve().relative_to(root.resolve())
+    except ValueError as exc:
+        raise AssetError(f"Legacy asset path escapes workspace root: {path}") from exc