fractfs 0.1.0__py3-none-any.whl

fractfs/__init__.py

@@ -0,0 +1,172 @@
+"""fractfs — drop-in tiered file storage for apps on ephemeral nodes.
+
+The whole drop-in surface is three calls::
+
+    import fractfs
+    fractfs.init()        # load config, provision symlinks, restore, start syncing
+    fractfs.sync_now()    # optional: force a checkpoint (e.g. before shutdown)
+    fractfs.status()      # optional: inspect tiers and last sync time
+"""
+
+from __future__ import annotations
+
+import atexit
+import logging
+import os
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+from .backend import make_backend
+from .config import Config, load_config
+from .provisioner import ClobberError, provision, warnings_for
+from .resolver import Tier, resolve
+from .sync import SyncDaemon, SyncEngine
+
+__all__ = [
+    "init",
+    "sync_now",
+    "status",
+    "shutdown",
+    "resolve",
+    "Tier",
+    "Config",
+    "load_config",
+    "ClobberError",
+]
+
+__version__ = "0.1.0"
+
+log = logging.getLogger("fractfs")
+
+
+@dataclass
+class _Runtime:
+    cfg: Config
+    engine: SyncEngine
+    daemon: Optional[SyncDaemon] = None
+
+
+_RUNTIME: Optional[_Runtime] = None
+
+
+def init(
+    root: Optional[os.PathLike] = None,
+    *,
+    force: bool = False,
+    start_daemon: bool = True,
+    restore: bool = True,
+) -> Config:
+    """Initialise fractfs: load config, provision symlinks, restore, start syncing.
+
+    Blocks on restore before returning so the app never reads cold state. Safe to
+    call once at startup. ``force=True`` lets provisioning migrate a non-empty real
+    local dir into the Volume (see :class:`ClobberError`).
+    """
+    global _RUNTIME
+
+    cfg = load_config(root)
+    engine = SyncEngine(cfg, backend=make_backend(cfg)) if cfg.is_provisionable() else None
+
+    for warning in warnings_for(cfg):
+        log.warning("fractfs config: %s", warning)
+
+    if cfg.is_provisionable():
+        actions = provision(cfg, force=force)
+        for a in actions:
+            log.debug("provision: %s", a)
+        # Identify the deployed bundle before restore so it's excluded from the
+        # checkpoint (re-supplied from the image on every cold start anyway).
+        if cfg.auto_ignore_bundle and engine is not None:
+            bundle = engine.detect_bundle()
+            if bundle:
+                log.info("fractfs auto-ignoring %d deploy-bundle file(s)", len(bundle))
+        # Cold-start ordering: restore must finish before the app reads anything.
+        if restore and engine is not None:
+            restored = engine.restore()
+            if restored:
+                log.info("fractfs restored %d checkpointed file(s)", len(restored))
+    else:
+        log.warning(
+            "fractfs: no fractfs_VOLUME_ROOT set; running in passthrough mode "
+            "(no redirect, no checkpoint)."
+        )
+
+    daemon = None
+    if engine is not None and start_daemon and cfg.sync_interval > 0:
+        daemon = SyncDaemon(engine, cfg.sync_interval)
+        daemon.start()
+        atexit.register(_atexit_stop)
+
+    _RUNTIME = _Runtime(cfg=cfg, engine=engine, daemon=daemon) if engine is not None else _Runtime(
+        cfg=cfg, engine=_NullEngine(cfg)  # type: ignore[arg-type]
+    )
+    return cfg
+
+
+def sync_now() -> List[str]:
+    """Force a checkpoint immediately. Returns the rel paths that were copied."""
+    rt = _require_runtime()
+    return rt.engine.checkpoint()
+
+
+def status() -> Dict[str, Any]:
+    """Report current configuration, the tier of each tracked path, and last sync."""
+    rt = _require_runtime()
+    cfg = rt.cfg
+    tracked: Dict[str, str] = {}
+    for d in cfg.dir_paths:
+        tracked[d] = resolve(d, cfg).value
+    return {
+        "backend": cfg.backend,
+        "volume_root": str(cfg.volume_root) if cfg.volume_root else None,
+        "scratch": str(cfg.scratch),
+        "sync_interval": cfg.sync_interval,
+        "provisionable": cfg.is_provisionable(),
+        "daemon_running": rt.daemon is not None,
+        "last_sync_time": getattr(rt.engine, "last_sync_time", None),
+        "auto_ignore_bundle": cfg.auto_ignore_bundle,
+        "bundle_file_count": len(getattr(rt.engine, "bundle_paths", set())),
+        "dirs": tracked,
+        "ignore_patterns": list(cfg.ignore_patterns),
+        "local_patterns": list(cfg.local_patterns),
+        "warnings": warnings_for(cfg),
+    }
+
+
+def shutdown(*, final_sync: bool = True) -> None:
+    """Stop the sync daemon, optionally running one last checkpoint."""
+    global _RUNTIME
+    if _RUNTIME is not None and _RUNTIME.daemon is not None:
+        _RUNTIME.daemon.stop(final_sync=final_sync)
+        _RUNTIME.daemon = None
+
+
+def _require_runtime() -> _Runtime:
+    if _RUNTIME is None:
+        raise RuntimeError("fractfs.init() has not been called")
+    return _RUNTIME
+
+
+def _atexit_stop() -> None:
+    try:
+        shutdown(final_sync=True)
+    except Exception:  # best-effort on interpreter shutdown
+        pass
+
+
+class _NullEngine:
+    """Stand-in when there is no Volume: checkpoint/restore are no-ops."""
+
+    def __init__(self, cfg: Config):
+        self.cfg = cfg
+        self.last_sync_time = None
+        self.bundle_paths: set = set()
+
+    def detect_bundle(self) -> set:
+        return set()
+
+    def checkpoint(self) -> List[str]:
+        return []
+
+    def restore(self, **_: Any) -> List[str]:
+        return []

fractfs/backend.py

@@ -0,0 +1,112 @@
+"""Backend abstraction for the remote/durable store.
+
+The primary target (Databricks Volumes) is a POSIX FUSE mount, and ``local``
+backing is just a directory, so both are served by :class:`PosixBackend`. The
+``fsspec``-backed path (``s3`` and friends) is wired through :class:`FsspecBackend`
+which is imported lazily so the base install stays dependency-light.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+from pathlib import Path
+from typing import Iterable, Protocol, runtime_checkable
+
+
+@runtime_checkable
+class Backend(Protocol):
+    """Minimal surface the provisioner and sync daemon need from a store."""
+
+    def exists(self, path: str) -> bool: ...
+
+    def makedirs(self, path: str) -> None: ...
+
+    def put_file(self, local_path: os.PathLike, remote_path: str) -> None:
+        """Copy a local file to ``remote_path``, atomically where possible."""
+
+    def get_file(self, remote_path: str, local_path: os.PathLike) -> None:
+        """Copy ``remote_path`` down to a local file, atomically where possible."""
+
+    def list_files(self, path: str) -> Iterable[str]:
+        """Yield remote-relative paths of every file under ``path`` (recursive)."""
+
+    def remove(self, path: str) -> None: ...
+
+
+class PosixBackend:
+    """Backend over a POSIX-visible root (a Volume mount or a local directory).
+
+    ``root`` is the absolute remote root; ``path`` arguments to every method are
+    interpreted relative to it (or accepted as already-absolute paths under it).
+    """
+
+    def __init__(self, root: os.PathLike, *, atomic_rename: bool = True):
+        self.root = Path(root)
+        # FUSE mounts don't always honour atomic rename; callers can disable it
+        # to fall back to a plain copy (see plan: FUSE atomicity open question).
+        self.atomic_rename = atomic_rename
+
+    def _abs(self, path: str) -> Path:
+        p = Path(path)
+        return p if p.is_absolute() else self.root / p
+
+    def exists(self, path: str) -> bool:
+        return self._abs(path).exists()
+
+    def makedirs(self, path: str) -> None:
+        self._abs(path).mkdir(parents=True, exist_ok=True)
+
+    def put_file(self, local_path: os.PathLike, remote_path: str) -> None:
+        dst = self._abs(remote_path)
+        dst.parent.mkdir(parents=True, exist_ok=True)
+        self._write_atomic(Path(local_path), dst)
+
+    def get_file(self, remote_path: str, local_path: os.PathLike) -> None:
+        dst = Path(local_path)
+        dst.parent.mkdir(parents=True, exist_ok=True)
+        self._write_atomic(self._abs(remote_path), dst)
+
+    def list_files(self, path: str) -> Iterable[str]:
+        base = self._abs(path)
+        if not base.exists():
+            return
+        for dirpath, _dirnames, filenames in os.walk(base):
+            for name in filenames:
+                full = Path(dirpath) / name
+                yield str(full.relative_to(self.root))
+
+    def remove(self, path: str) -> None:
+        p = self._abs(path)
+        if p.is_dir() and not p.is_symlink():
+            shutil.rmtree(p)
+        elif p.exists() or p.is_symlink():
+            p.unlink()
+
+    def _write_atomic(self, src: Path, dst: Path) -> None:
+        """Copy ``src`` to ``dst`` via a temp file + rename when atomic_rename."""
+        if not self.atomic_rename:
+            shutil.copy2(src, dst)
+            return
+        tmp = dst.with_name(f".{dst.name}.fractfs.tmp.{os.getpid()}")
+        try:
+            shutil.copy2(src, tmp)
+            os.replace(tmp, dst)
+        finally:
+            if tmp.exists():
+                tmp.unlink()
+
+
+def make_backend(cfg) -> Backend:
+    """Construct the backend for a :class:`~fractfs.config.Config`."""
+    if cfg.backend in ("volumes", "local"):
+        if cfg.volume_root is None:
+            raise ValueError(
+                f"backend {cfg.backend!r} requires fractfs_VOLUME_ROOT to be set"
+            )
+        return PosixBackend(cfg.volume_root)
+    if cfg.backend == "s3":
+        from .fsspec_backend import FsspecBackend
+
+        return FsspecBackend(cfg)
+    raise ValueError(f"no backend implementation for {cfg.backend!r}")

fractfs/config.py

@@ -0,0 +1,160 @@
+"""Config loader: parse ``.fractfs.toml`` + environment into a ``Config``.
+
+Env vars override the TOML file for the scalar fields so deployments can tune
+behaviour (backend, volume root, cadence) without editing the repo.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List, Optional
+
+import pathspec
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:  # pragma: no cover - exercised only on <3.11
+    import tomli as tomllib
+
+CONFIG_FILENAME = ".fractfs.toml"
+
+# Backends we know how to provision against.
+BACKENDS = ("volumes", "s3", "local")
+
+# pathspec renamed the gitignore factory; prefer the current name, fall back for
+# older pathspec releases that only ship "gitwildmatch".
+try:
+    pathspec.PathSpec.from_lines("gitignore", [])
+    _PATHSPEC_FACTORY = "gitignore"
+except (ValueError, KeyError, LookupError):  # pragma: no cover - old pathspec
+    _PATHSPEC_FACTORY = "gitwildmatch"
+
+_DEFAULT_SYNC_INTERVAL = 300
+_DEFAULT_CHECKPOINT_SUBDIR = "_checkpoint"
+_DEFAULT_SCRATCH = "/tmp/fractfs"
+
+
+@dataclass
+class Config:
+    """Resolved configuration.
+
+    ``root`` is the application root (the dir holding ``.fractfs.toml``); all
+    ``dir_paths`` and resolver inputs are relative to it.
+    """
+
+    root: Path
+    backend: str = "local"
+    volume_root: Optional[Path] = None
+    scratch: Path = Path(_DEFAULT_SCRATCH)
+    sync_interval: int = _DEFAULT_SYNC_INTERVAL
+    checkpoint_subdir: str = _DEFAULT_CHECKPOINT_SUBDIR
+    dir_paths: List[str] = field(default_factory=list)
+    ignore_patterns: List[str] = field(default_factory=list)
+    local_patterns: List[str] = field(default_factory=list)
+    use_content_hash: bool = False
+    auto_ignore_bundle: bool = True
+
+    ignore_spec: pathspec.PathSpec = field(init=False)
+    local_spec: pathspec.PathSpec = field(init=False)
+
+    def __post_init__(self) -> None:
+        self.root = Path(self.root).resolve()
+        if self.volume_root is not None:
+            self.volume_root = Path(self.volume_root)
+        self.scratch = Path(self.scratch)
+        # Normalize dir paths to forward-slash relative strings (drop "./", trailing "/").
+        self.dir_paths = [d.strip("/").replace("\\", "/") for d in self.dir_paths if d.strip("/")]
+        self.ignore_spec = pathspec.PathSpec.from_lines(_PATHSPEC_FACTORY, self.ignore_patterns)
+        self.local_spec = pathspec.PathSpec.from_lines(_PATHSPEC_FACTORY, self.local_patterns)
+        if self.backend not in BACKENDS:
+            raise ValueError(
+                f"unknown fractfs backend {self.backend!r}; expected one of {BACKENDS}"
+            )
+
+    # -- derived paths -----------------------------------------------------
+
+    @property
+    def checkpoint_root(self) -> Optional[Path]:
+        """Absolute path under the Volume where LOCAL_SYNCED checkpoints land."""
+        if self.volume_root is None:
+            return None
+        return self.volume_root / self.checkpoint_subdir
+
+    def is_provisionable(self) -> bool:
+        """Whether dir-redirect / back-symlink provisioning can run.
+
+        Requires a Volume root; without one (pure ``local`` backend, no mount)
+        only checkpoint/restore against a local volume_root is meaningful.
+        """
+        return self.volume_root is not None
+
+
+def _env(name: str) -> Optional[str]:
+    return os.environ.get(f"fractfs_{name}") or os.environ.get(f"FRACTFS_{name}")
+
+
+def load_config(root: Optional[os.PathLike] = None) -> Config:
+    """Load config from ``<root>/.fractfs.toml`` with env-var overrides.
+
+    ``root`` defaults to ``$fractfs_ROOT`` then the current working directory.
+    """
+    if root is None:
+        root = _env("ROOT") or os.getcwd()
+    root = Path(root).resolve()
+
+    data = _read_toml(root / CONFIG_FILENAME)
+
+    dirs = data.get("dirs", {}).get("paths", []) or []
+    ignore = data.get("ignore", {}).get("patterns", []) or []
+    local = data.get("local", {}).get("patterns", []) or []
+
+    backend = _env("BACKEND") or data.get("backend") or "local"
+
+    volume_root = _env("VOLUME_ROOT") or data.get("volume_root")
+    scratch = _env("SCRATCH") or data.get("scratch") or _DEFAULT_SCRATCH
+    checkpoint_subdir = (
+        _env("CHECKPOINT_SUBDIR") or data.get("checkpoint_subdir") or _DEFAULT_CHECKPOINT_SUBDIR
+    )
+
+    sync_interval_raw = _env("SYNC_INTERVAL") or data.get("sync_interval")
+    sync_interval = int(sync_interval_raw) if sync_interval_raw is not None else _DEFAULT_SYNC_INTERVAL
+
+    hash_raw = _env("CONTENT_HASH") or data.get("content_hash")
+    use_content_hash = _as_bool(hash_raw, default=False)
+
+    bundle_raw = _env("AUTO_IGNORE_BUNDLE")
+    if bundle_raw is None and "auto_ignore_bundle" in data:
+        bundle_raw = data.get("auto_ignore_bundle")
+    auto_ignore_bundle = _as_bool(bundle_raw, default=True)
+
+    return Config(
+        root=root,
+        backend=backend,
+        volume_root=volume_root,
+        scratch=scratch,
+        sync_interval=sync_interval,
+        checkpoint_subdir=checkpoint_subdir,
+        dir_paths=list(dirs),
+        ignore_patterns=list(ignore),
+        local_patterns=list(local),
+        use_content_hash=use_content_hash,
+        auto_ignore_bundle=auto_ignore_bundle,
+    )
+
+
+def _read_toml(path: Path) -> dict:
+    if not path.exists():
+        return {}
+    with open(path, "rb") as fh:
+        return tomllib.load(fh)
+
+
+def _as_bool(value, default: bool) -> bool:
+    if value is None:
+        return default
+    if isinstance(value, bool):
+        return value
+    return str(value).strip().lower() in ("1", "true", "yes", "on")

fractfs/fsspec_backend.py

@@ -0,0 +1,61 @@
+"""fsspec-backed implementation (S3 / ADLS / GCS) — optional extra.
+
+Kept in its own module so importing fractfs never pulls in ``fsspec``/``s3fs``
+unless an fsspec backend is actually requested. Install with ``fractfs[s3]``.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Iterable
+
+
+class FsspecBackend:
+    """Backend over any fsspec filesystem rooted at ``cfg.volume_root`` URL.
+
+    ``volume_root`` here is an fsspec URL such as ``s3://bucket/prefix``.
+    """
+
+    def __init__(self, cfg):
+        try:
+            import fsspec
+        except ImportError as exc:  # pragma: no cover - depends on optional extra
+            raise ImportError(
+                "the s3/fsspec backend requires the 'fsspec' extra: pip install 'fractfs[s3]'"
+            ) from exc
+        if cfg.volume_root is None:
+            raise ValueError("fsspec backend requires fractfs_VOLUME_ROOT (an fsspec URL)")
+        self.url_root = str(cfg.volume_root).rstrip("/")
+        self.fs, self.path_root = fsspec.core.url_to_fs(self.url_root)
+
+    def _abs(self, path: str) -> str:
+        path = str(path)
+        if path.startswith(self.path_root):
+            return path
+        return f"{self.path_root}/{path.lstrip('/')}"
+
+    def exists(self, path: str) -> bool:
+        return self.fs.exists(self._abs(path))
+
+    def makedirs(self, path: str) -> None:
+        self.fs.makedirs(self._abs(path), exist_ok=True)
+
+    def put_file(self, local_path: os.PathLike, remote_path: str) -> None:
+        self.fs.put_file(str(local_path), self._abs(remote_path))
+
+    def get_file(self, remote_path: str, local_path: os.PathLike) -> None:
+        os.makedirs(os.path.dirname(local_path) or ".", exist_ok=True)
+        self.fs.get_file(self._abs(remote_path), str(local_path))
+
+    def list_files(self, path: str) -> Iterable[str]:
+        base = self._abs(path)
+        if not self.fs.exists(base):
+            return
+        for full in self.fs.find(base):
+            rel = full[len(self.path_root):].lstrip("/")
+            yield rel
+
+    def remove(self, path: str) -> None:
+        target = self._abs(path)
+        if self.fs.exists(target):
+            self.fs.rm(target, recursive=True)

fractfs/patterns.py

@@ -0,0 +1,63 @@
+"""Static analysis of gitignore patterns, used to decide what can be pinned.
+
+A ``local``/``ignore`` file inside a Volume-redirected ``[dirs]`` directory only
+stays node-local if a back-symlink exists *before* the write. We can pre-create
+that symlink for any pattern naming a concrete location; we cannot for a glob,
+because the filename isn't known until the app creates it.
+"""
+
+from __future__ import annotations
+
+from typing import List, Tuple
+
+from .resolver import _under_any_dir
+
+# gitignore wildcard metacharacters. A pattern containing any of these names a
+# set of paths, not one path, so its back-symlink location can't be predicted.
+_GLOB_META = "*?["
+
+
+def is_glob(pattern: str) -> bool:
+    return any(c in pattern for c in _GLOB_META)
+
+
+def is_negation(pattern: str) -> bool:
+    return pattern.lstrip().startswith("!")
+
+
+def is_dir_pattern(pattern: str) -> bool:
+    """True if the pattern targets a directory (trailing slash)."""
+    return pattern.rstrip().endswith("/")
+
+
+def is_anchored(pattern: str) -> bool:
+    """gitignore anchoring: a leading or internal slash pins the pattern to root."""
+    body = pattern.strip()
+    if body.startswith("/"):
+        return True
+    return "/" in body.rstrip("/")
+
+
+def is_concrete(pattern: str) -> bool:
+    """A pattern we can pre-create a back-symlink for (an exact path/name)."""
+    return bool(pattern.strip()) and not is_negation(pattern) and not is_glob(pattern)
+
+
+def pin_targets(pattern: str, dir_paths: List[str]) -> List[Tuple[str, bool]]:
+    """Back-symlink targets for ``pattern`` that fall inside ``dir_paths``.
+
+    Returns ``(rel_path, is_dir)`` tuples. Empty for globs/negations (can't be
+    predicted) and for concrete paths that don't land inside any Volume dir
+    (those already live in the plain local tree and need no symlink).
+    """
+    if not is_concrete(pattern):
+        return []
+    is_dir = is_dir_pattern(pattern)
+    clean = pattern.strip().strip("/")
+    if not clean:
+        return []
+    if is_anchored(pattern):
+        return [(clean, is_dir)] if _under_any_dir(clean, dir_paths) else []
+    # Unanchored bare name (e.g. "manifest.json", ".locks/"): the common case is
+    # one at the top of each Volume dir — reserve a slot there.
+    return [(f"{d}/{clean}", is_dir) for d in dir_paths]