furu 0.0.1__py3-none-any.whl

furu/serialization/serializer.py

@@ -0,0 +1,233 @@
+import datetime
+import enum
+import hashlib
+import importlib
+import json
+import pathlib
+import textwrap
+from pathlib import Path
+from typing import Any
+
+import chz
+
+from ..errors import _FuruMissing
+from pydantic import BaseModel as PydanticBaseModel
+
+
+# Type alias for JSON-serializable values. We use Any here because this serialization
+# library handles arbitrary user-defined objects that we cannot know at compile time.
+JsonValue = Any
+
+
+class FuruSerializer:
+    """Handles serialization, deserialization, and hashing of Furu objects."""
+
+    CLASS_MARKER = "__class__"
+
+    @staticmethod
+    def get_classname(obj: object) -> str:
+        """Get fully qualified class name."""
+        classname = obj.__class__.__module__
+        if classname == "__main__":
+            raise ValueError("Cannot serialize objects from __main__ module")
+
+        if isinstance(obj, enum.Enum):
+            return f"{classname}.{obj.__class__.__qualname__}:{obj.name}"
+        return f"{classname}.{obj.__class__.__qualname__}"
+
+    @classmethod
+    def to_dict(cls, obj: object) -> JsonValue:
+        """Convert object to JSON-serializable dictionary."""
+        if isinstance(obj, _FuruMissing):
+            raise ValueError("Cannot serialize Furu.MISSING")
+
+        if chz.is_chz(obj):
+            result = {cls.CLASS_MARKER: cls.get_classname(obj)}
+            for field_name in chz.chz_fields(obj):
+                result[field_name] = cls.to_dict(getattr(obj, field_name))
+            return result
+
+        if isinstance(obj, pathlib.Path):
+            return str(obj)
+
+        if isinstance(obj, (list, tuple)):
+            return [cls.to_dict(v) for v in obj]
+
+        if isinstance(obj, dict):
+            return {k: cls.to_dict(v) for k, v in obj.items()}
+
+        return obj
+
+    @classmethod
+    def from_dict(cls, data: JsonValue) -> JsonValue:
+        """Reconstruct object from dictionary."""
+        if isinstance(data, dict) and cls.CLASS_MARKER in data:
+            module_path, _, class_name = data[cls.CLASS_MARKER].rpartition(".")
+            data_class = getattr(importlib.import_module(module_path), class_name)
+
+            kwargs = {
+                k: cls.from_dict(v) for k, v in data.items() if k != cls.CLASS_MARKER
+            }
+
+            path_types = (Path, pathlib.Path)
+
+            if chz.is_chz(data_class):
+                for name, field in chz.chz_fields(data_class).items():
+                    if field.final_type in path_types and isinstance(
+                        kwargs.get(name), str
+                    ):
+                        kwargs[name] = pathlib.Path(kwargs[name])
+            return data_class(**kwargs)
+
+        if isinstance(data, list):
+            return [cls.from_dict(v) for v in data]
+
+        if isinstance(data, dict):
+            return {k: cls.from_dict(v) for k, v in data.items()}
+
+        return data
+
+    @classmethod
+    def compute_hash(cls, obj: object, verbose: bool = False) -> str:
+        """Compute deterministic hash of object."""
+
+        def canonicalize(item: object) -> JsonValue:
+            if isinstance(item, _FuruMissing):
+                raise ValueError("Cannot hash Furu.MISSING")
+
+            if chz.is_chz(item):
+                fields = chz.chz_fields(item)
+                return {
+                    "__class__": cls.get_classname(item),
+                    **{
+                        name: canonicalize(getattr(item, name))
+                        for name in fields
+                        if not name.startswith("_")
+                    },
+                }
+
+            if isinstance(item, dict):
+                filtered = item
+                if cls.CLASS_MARKER in item:
+                    filtered = {
+                        k: v
+                        for k, v in item.items()
+                        if not (isinstance(k, str) and k.startswith("_"))
+                        or k == cls.CLASS_MARKER
+                    }
+                return {k: canonicalize(v) for k, v in sorted(filtered.items())}
+
+            if isinstance(item, (list, tuple)):
+                return [canonicalize(v) for v in item]
+
+            if isinstance(item, Path):
+                return str(item)
+
+            if isinstance(item, enum.Enum):
+                return {"__enum__": cls.get_classname(item)}
+
+            if isinstance(item, (set, frozenset)):
+                return sorted(canonicalize(v) for v in item)
+
+            if isinstance(item, (bytes, bytearray, memoryview)):
+                return {"__bytes__": hashlib.sha256(item).hexdigest()}
+
+            if isinstance(item, datetime.datetime):
+                return item.astimezone(datetime.timezone.utc).isoformat(
+                    timespec="microseconds"
+                )
+
+            if isinstance(item, (str, int, float, bool)) or item is None:
+                return item
+
+            if isinstance(item, PydanticBaseModel):
+                return {
+                    "__class__": cls.get_classname(item),
+                    **{k: canonicalize(v) for k, v in item.model_dump().items()},
+                }
+
+            raise TypeError(f"Cannot hash type: {type(item)}")
+
+        canonical = canonicalize(obj)
+        json_str = json.dumps(canonical, sort_keys=True, separators=(",", ":"))
+
+        if verbose:
+            print(json_str)
+
+        return hashlib.blake2s(json_str.encode(), digest_size=10).hexdigest()
+
+    @classmethod
+    def to_python(cls, obj: object, multiline: bool = True) -> str:
+        """Convert object to Python code representation."""
+
+        def to_py_recursive(item: object, indent: int = 0) -> str:
+            if isinstance(item, _FuruMissing):
+                raise ValueError("Cannot convert Furu.MISSING to Python")
+
+            pad = "" if not multiline else " " * indent
+            next_indent = indent + (4 if multiline else 0)
+
+            if chz.is_chz(item):
+                cls_path = cls.get_classname(item)
+                fields = []
+                for name, field in chz.chz_fields(item).items():
+                    fields.append(
+                        f"{name}={to_py_recursive(getattr(item, name), next_indent)}"
+                    )
+
+                if multiline:
+                    inner = (",\n" + " " * next_indent).join(fields)
+                    return f"{cls_path}(\n{pad}    {inner}\n{pad})"
+                return f"{cls_path}({', '.join(fields)})"
+
+            if isinstance(item, enum.Enum):
+                return cls.get_classname(item)
+
+            if isinstance(item, pathlib.Path):
+                return f"pathlib.Path({str(item)!r})"
+
+            if isinstance(item, datetime.datetime):
+                iso = item.astimezone(datetime.timezone.utc).isoformat(
+                    timespec="microseconds"
+                )
+                return f"datetime.datetime.fromisoformat({iso!r})"
+
+            if isinstance(item, (bytes, bytearray, memoryview)):
+                hex_str = hashlib.sha256(item).hexdigest()
+                return f"bytes.fromhex({hex_str!r})"
+
+            if isinstance(item, list):
+                items = ", ".join(to_py_recursive(v, next_indent) for v in item)
+                return f"[{items}]"
+
+            if isinstance(item, tuple):
+                items = ", ".join(to_py_recursive(v, next_indent) for v in item)
+                comma = "," if len(item) == 1 else ""
+                return f"({items}{comma})"
+
+            if isinstance(item, set):
+                items = ", ".join(to_py_recursive(v, next_indent) for v in item)
+                return f"{{{items}}}"
+
+            if isinstance(item, frozenset):
+                items = ", ".join(to_py_recursive(v, next_indent) for v in item)
+                return f"frozenset({{{items}}})"
+
+            if isinstance(item, dict):
+                kv_pairs = [
+                    f"{to_py_recursive(k, next_indent)}: {to_py_recursive(v, next_indent)}"
+                    for k, v in item.items()
+                ]
+
+                if multiline:
+                    joined = (",\n" + " " * (indent + 4)).join(kv_pairs)
+                    return f"{{\n{pad}    {joined}\n{pad}}}"
+                else:
+                    return "{" + ", ".join(kv_pairs) + "}"
+
+            return repr(item)
+
+        result = to_py_recursive(obj, indent=0)
+        if multiline:
+            result = textwrap.dedent(result).strip()
+        return result

furu/storage/__init__.py

@@ -0,0 +1,32 @@
+from .metadata import (
+    EnvironmentInfo,
+    GitInfo,
+    FuruMetadata,
+    MetadataManager,
+    clear_metadata_cache,
+)
+from .migration import MigrationManager, MigrationRecord
+from .state import (
+    ComputeLockContext,
+    FuruErrorState,
+    StateAttempt,
+    StateManager,
+    StateOwner,
+    compute_lock,
+)
+
+__all__ = [
+    "ComputeLockContext",
+    "EnvironmentInfo",
+    "GitInfo",
+    "FuruErrorState",
+    "FuruMetadata",
+    "MetadataManager",
+    "MigrationManager",
+    "MigrationRecord",
+    "StateAttempt",
+    "StateManager",
+    "StateOwner",
+    "clear_metadata_cache",
+    "compute_lock",
+]

furu/storage/metadata.py

@@ -0,0 +1,282 @@
+import datetime
+import getpass
+import json
+import os
+import platform
+import socket
+import subprocess
+import sys
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from pydantic import BaseModel, ConfigDict
+
+from ..config import FURU_CONFIG
+from ..serialization import BaseModel as PydanticBaseModel
+from ..serialization import FuruSerializer
+from ..serialization.serializer import JsonValue
+
+if TYPE_CHECKING:
+    from ..core.furu import Furu
+
+# Module-level cache for metadata (controlled via FURU_CACHE_METADATA)
+_cached_git_info: "GitInfo | None" = None
+_cached_git_info_time: float = 0.0
+
+
+def clear_metadata_cache() -> None:
+    """Clear the cached metadata. Useful for testing or long-running processes."""
+    global _cached_git_info, _cached_git_info_time
+    _cached_git_info = None
+    _cached_git_info_time = 0.0
+
+
+class GitInfo(BaseModel):
+    """Git repository information."""
+
+    model_config = ConfigDict(extra="forbid", strict=True)
+
+    git_commit: str
+    git_branch: str
+    git_remote: str | None
+    git_patch: str
+    git_submodules: dict[str, str]
+
+
+class EnvironmentInfo(BaseModel):
+    """Runtime environment information."""
+
+    model_config = ConfigDict(extra="forbid", strict=True)
+
+    timestamp: str
+    command: str
+    python_version: str
+    executable: str
+    platform: str
+    hostname: str
+    user: str
+    pid: int
+
+
+class FuruMetadata(BaseModel):
+    """Complete metadata for a Furu experiment."""
+
+    model_config = ConfigDict(extra="forbid", strict=True)
+
+    # Furu-specific fields
+    furu_python_def: str
+    furu_obj: JsonValue  # Serialized Furu object from FuruSerializer.to_dict()
+    furu_hash: str
+    furu_path: str
+
+    # Git info
+    git_commit: str
+    git_branch: str
+    git_remote: str | None
+    git_patch: str
+    git_submodules: dict[str, str]
+
+    # Environment info
+    timestamp: str
+    command: str
+    python_version: str
+    executable: str
+    platform: str
+    hostname: str
+    user: str
+    pid: int
+
+
+class MetadataManager:
+    """Handles metadata collection and storage."""
+
+    INTERNAL_DIR = ".furu"
+    METADATA_FILE = "metadata.json"
+
+    @classmethod
+    def get_metadata_path(cls, directory: Path) -> Path:
+        return directory / cls.INTERNAL_DIR / cls.METADATA_FILE
+
+    @staticmethod
+    def run_git_command(args: list[str]) -> str:
+        """Run git command, return output."""
+        proc = subprocess.run(
+            ["git", *args], text=True, capture_output=True, timeout=10
+        )
+        if proc.returncode not in (0, 1):
+            proc.check_returncode()
+        return proc.stdout.strip()
+
+    @classmethod
+    def collect_git_info(cls, ignore_diff: bool = False) -> GitInfo:
+        """Collect git repository information."""
+        global _cached_git_info, _cached_git_info_time
+        import time
+
+        ttl = FURU_CONFIG.cache_metadata_ttl_sec
+        # Return cached result if caching is enabled and not expired
+        if ttl is not None and _cached_git_info is not None:
+            age = time.time() - _cached_git_info_time
+            if age < ttl:
+                return _cached_git_info
+
+        if not FURU_CONFIG.require_git:
+            try:
+                head = cls.run_git_command(["rev-parse", "HEAD"])
+                branch = cls.run_git_command(["rev-parse", "--abbrev-ref", "HEAD"])
+            except subprocess.CalledProcessError:
+                return GitInfo(
+                    git_commit="<no-git>",
+                    git_branch="<no-git>",
+                    git_remote=None,
+                    git_patch="<no-git>",
+                    git_submodules={},
+                )
+        else:
+            head = cls.run_git_command(["rev-parse", "HEAD"])
+            branch = cls.run_git_command(["rev-parse", "--abbrev-ref", "HEAD"])
+
+        if FURU_CONFIG.require_git_remote:
+            remote = cls.run_git_command(["remote", "get-url", "origin"])
+        else:
+            try:
+                remote = cls.run_git_command(["remote", "get-url", "origin"])
+            except subprocess.CalledProcessError:
+                remote = None
+
+        if ignore_diff:
+            patch = "<ignored-diff>"
+        else:
+            unstaged = cls.run_git_command(["diff"])
+            staged = cls.run_git_command(["diff", "--cached"])
+            untracked = cls.run_git_command(
+                ["ls-files", "--others", "--exclude-standard"]
+            ).splitlines()
+
+            untracked_patches = "\n".join(
+                cls.run_git_command(["diff", "--no-index", "/dev/null", f])
+                for f in untracked
+            )
+
+            patch = "\n".join(
+                filter(
+                    None,
+                    [
+                        "# === unstaged ==================================================",
+                        unstaged,
+                        "# === staged ====================================================",
+                        staged,
+                        "# === untracked ================================================",
+                        untracked_patches,
+                    ],
+                )
+            )
+
+            if len(patch) > 50_000:
+                raise ValueError(
+                    f"Git diff too large ({len(patch):,} bytes). "
+                    "Use ignore_diff=True or FURU_IGNORE_DIFF=1"
+                )
+
+        submodules: dict[str, str] = {}
+        for line in cls.run_git_command(["submodule", "status"]).splitlines():
+            parts = line.split()
+            if len(parts) >= 2:
+                submodules[parts[1]] = parts[0]
+
+        result = GitInfo(
+            git_commit=head,
+            git_branch=branch,
+            git_remote=remote,
+            git_patch=patch,
+            git_submodules=submodules,
+        )
+
+        # Cache result if caching is enabled
+        if ttl is not None:
+            _cached_git_info = result
+            _cached_git_info_time = time.time()
+
+        return result
+
+    @staticmethod
+    def collect_environment_info() -> EnvironmentInfo:
+        """Collect environment information."""
+        return EnvironmentInfo(
+            timestamp=datetime.datetime.now(datetime.timezone.utc).isoformat(
+                timespec="microseconds"
+            ),
+            command=" ".join(sys.argv) if sys.argv else "<unknown>",
+            python_version=sys.version,
+            executable=sys.executable,
+            platform=platform.platform(),
+            hostname=socket.gethostname(),
+            user=getpass.getuser(),
+            pid=os.getpid(),
+        )
+
+    @classmethod
+    def create_metadata(
+        cls, furu_obj: "Furu", directory: Path, ignore_diff: bool = False
+    ) -> FuruMetadata:
+        """Create complete metadata for a Furu object."""
+        git_info = cls.collect_git_info(ignore_diff)
+        env_info = cls.collect_environment_info()
+
+        serialized_obj = FuruSerializer.to_dict(furu_obj)
+        if not isinstance(serialized_obj, dict):
+            raise TypeError(
+                f"Expected FuruSerializer.to_dict to return dict, got {type(serialized_obj)}"
+            )
+
+        return FuruMetadata(
+            furu_python_def=FuruSerializer.to_python(furu_obj, multiline=False),
+            furu_obj=serialized_obj,
+            furu_hash=FuruSerializer.compute_hash(furu_obj),
+            furu_path=str(directory.resolve()),
+            git_commit=git_info.git_commit,
+            git_branch=git_info.git_branch,
+            git_remote=git_info.git_remote,
+            git_patch=git_info.git_patch,
+            git_submodules=git_info.git_submodules,
+            timestamp=env_info.timestamp,
+            command=env_info.command,
+            python_version=env_info.python_version,
+            executable=env_info.executable,
+            platform=env_info.platform,
+            hostname=env_info.hostname,
+            user=env_info.user,
+            pid=env_info.pid,
+        )
+
+    @classmethod
+    def write_metadata(cls, metadata: FuruMetadata, directory: Path) -> None:
+        """Write metadata to file."""
+        metadata_path = cls.get_metadata_path(directory)
+        metadata_path.parent.mkdir(parents=True, exist_ok=True)
+        metadata_path.write_text(
+            json.dumps(
+                metadata.model_dump(mode="json"),
+                indent=2,
+                default=lambda o: o.model_dump()
+                if PydanticBaseModel is not None and isinstance(o, PydanticBaseModel)
+                else str(o),
+            )
+        )
+
+    @classmethod
+    def read_metadata(cls, directory: Path) -> FuruMetadata:
+        """Read metadata from file."""
+        metadata_path = cls.get_metadata_path(directory)
+        if not metadata_path.is_file():
+            raise FileNotFoundError(f"Metadata not found: {metadata_path}")
+        data = json.loads(metadata_path.read_text())
+        return FuruMetadata.model_validate(data)
+
+    @classmethod
+    def read_metadata_raw(cls, directory: Path) -> dict[str, JsonValue] | None:
+        """Read raw metadata JSON from file, returning None if not found."""
+        metadata_path = cls.get_metadata_path(directory)
+        if not metadata_path.is_file():
+            return None
+        return json.loads(metadata_path.read_text())

furu/storage/migration.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict
+
+from ..config import FURU_CONFIG
+from ..serialization.serializer import JsonValue
+
+
+RootKind = Literal["data", "git"]
+MigrationPolicy = Literal["alias", "move", "copy"]
+MigrationKind = Literal["alias", "moved", "copied", "migrated"]
+
+
+class MigrationRecord(BaseModel):
+    model_config = ConfigDict(extra="ignore", strict=True)
+
+    kind: MigrationKind
+    policy: MigrationPolicy
+    from_namespace: str
+    from_hash: str
+    from_root: RootKind
+    to_namespace: str
+    to_hash: str
+    to_root: RootKind
+    migrated_at: str
+    overwritten_at: str | None = None
+    default_values: dict[str, JsonValue] | None = None
+    origin: str | None = None
+    note: str | None = None
+
+
+class MigrationManager:
+    INTERNAL_DIR = ".furu"
+    MIGRATION_FILE = "migration.json"
+
+    @classmethod
+    def get_migration_path(cls, directory: Path) -> Path:
+        return directory / cls.INTERNAL_DIR / cls.MIGRATION_FILE
+
+    @classmethod
+    def read_migration(cls, directory: Path) -> MigrationRecord | None:
+        path = cls.get_migration_path(directory)
+        if not path.is_file():
+            return None
+        data = json.loads(path.read_text())
+        return MigrationRecord.model_validate(data)
+
+    @classmethod
+    def write_migration(cls, record: MigrationRecord, directory: Path) -> None:
+        path = cls.get_migration_path(directory)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        tmp = path.with_suffix(".tmp")
+        tmp.write_text(json.dumps(record.model_dump(mode="json"), indent=2))
+        tmp.replace(path)
+
+    @classmethod
+    def resolve_dir(
+        cls, record: MigrationRecord, *, target: Literal["from", "to"]
+    ) -> Path:
+        if target == "from":
+            namespace = record.from_namespace
+            furu_hash = record.from_hash
+            root_kind = record.from_root
+        else:
+            namespace = record.to_namespace
+            furu_hash = record.to_hash
+            root_kind = record.to_root
+        root = FURU_CONFIG.get_root(version_controlled=root_kind == "git")
+        return root / Path(*namespace.split(".")) / furu_hash
+
+    @classmethod
+    def root_kind_for_dir(cls, directory: Path) -> RootKind:
+        for version_controlled in (False, True):
+            root = FURU_CONFIG.get_root(version_controlled=version_controlled)
+            if directory.is_relative_to(root):
+                return "git" if version_controlled else "data"
+        raise ValueError(f"Directory {directory} is not under a Furu root")