featm 0.5.7__py3-none-any.whl

featm/__init__.py

@@ -0,0 +1,86 @@
+from . import features as _features  # register built-ins
+from . import state as _state  # register Step 2 stateful two-pass encoders
+from . import gpu  # Step 3: registers no CUDA import at module scope -- see featm.gpu.backend
+from .engine import (
+    AutoFeatures,
+    create_feature,
+    extract,
+    feature_info,
+    list_families,
+    list_features,
+    list_modalities,
+    search,
+)
+from .extractor import FeatureExtractor, FittedFeatureExtractor, fit, load_fitted
+from .distributed import distributed_extract
+from .matrix import FeatureMatrix, MaterializationGuardError
+from .registry import (
+    BatchSemantics,
+    Complexity,
+    Cost,
+    FeatureOutput,
+    FeatureSpec,
+    MemoryProfile,
+    REGISTRY,
+    RegistryValidationError,
+    register_feature,
+)
+from .report import ExecutionReport
+from .resources import ResourceBudget, parse_size
+from .schema import DatasetProfile, SemanticType, infer_schema
+from .sources import (
+    ArrowSource,
+    CSVSource,
+    IteratorSource,
+    JSONLSource,
+    ParquetSource,
+    PandasSource,
+    Source,
+    SourceNotReplayableError,
+    resolve_source,
+)
+
+__all__ = [
+    "ArrowSource",
+    "AutoFeatures",
+    "BatchSemantics",
+    "CSVSource",
+    "Complexity",
+    "Cost",
+    "DatasetProfile",
+    "ExecutionReport",
+    "FeatureExtractor",
+    "FeatureMatrix",
+    "FeatureOutput",
+    "FeatureSpec",
+    "FittedFeatureExtractor",
+    "IteratorSource",
+    "JSONLSource",
+    "MaterializationGuardError",
+    "MemoryProfile",
+    "PandasSource",
+    "ParquetSource",
+    "REGISTRY",
+    "RegistryValidationError",
+    "ResourceBudget",
+    "SemanticType",
+    "Source",
+    "SourceNotReplayableError",
+    "create_feature",
+    "distributed_extract",
+    "extract",
+    "feature_info",
+    "fit",
+    "gpu",
+    "infer_schema",
+    "list_families",
+    "list_features",
+    "list_modalities",
+    "load_fitted",
+    "parse_size",
+    "register_feature",
+    "resolve_source",
+    "search",
+]
+
+__version__ = "0.5.7"

featm/cache.py

@@ -0,0 +1,190 @@
+"""Shared intermediate computation cache.
+
+Selected features frequently depend on the same expensive intermediate (one
+FFT feeding several spectral descriptors, one Delaunay triangulation feeding
+several simplex statistics, one graph Laplacian eigendecomposition feeding
+several spectral graph features, and so on). This module gives feature
+implementations a way to memoize such intermediates *within one execution
+scope* (one batch of one :func:`featm.extract` / engine run) so that
+selecting several related features does not recompute shared work.
+
+Design:
+
+- A cache is explicitly scoped via :func:`scoped_cache`, a context manager
+  entered once per batch/call. Each scope gets its own cache instance, so
+  data from different batches or different datasets can never leak into one
+  another -- the cache simply does not outlive its scope.
+- Within a scope, callers look up the *current* cache with
+  :func:`current_cache` (backed by a ``contextvars.ContextVar`` so it is
+  correct across threads) and call :meth:`SharedIntermediateCache.get_or_compute`.
+- The cache obeys a byte budget with LRU eviction so a scope cannot grow
+  without bound; evicted entries are simply recomputed on next use.
+"""
+from __future__ import annotations
+
+import contextvars
+import hashlib
+import threading
+from collections import OrderedDict
+from contextlib import contextmanager
+from typing import Any, Callable, Iterator
+
+import numpy as np
+
+_DEFAULT_BUDGET_BYTES = 256 * 1024 * 1024  # 256MB per execution scope by default
+
+_CURRENT: contextvars.ContextVar["SharedIntermediateCache | None"] = contextvars.ContextVar(
+    "featm_current_cache", default=None
+)
+
+
+def _default_size_estimate(value: Any) -> int:
+    if isinstance(value, np.ndarray):
+        return int(value.nbytes)
+    if isinstance(value, (list, tuple)):
+        return sum(_default_size_estimate(v) for v in value) or 64
+    if isinstance(value, dict):
+        return sum(_default_size_estimate(v) for v in value.values()) or 64
+    if isinstance(value, (bytes, bytearray, str)):
+        return len(value)
+    return 64  # crude fallback for scalars/small objects
+
+
+class SharedIntermediateCache:
+    """A bounded, LRU-evicting cache for one execution scope."""
+
+    def __init__(self, max_bytes: int = _DEFAULT_BUDGET_BYTES) -> None:
+        self.max_bytes = max_bytes
+        self._store: "OrderedDict[tuple, tuple[Any, int]]" = OrderedDict()
+        self._bytes = 0
+        self.hits = 0
+        self.misses = 0
+        self.evictions = 0
+        self.oversized_skips = 0
+        self._lock = threading.Lock()
+
+    def get_or_compute(
+        self,
+        key: tuple,
+        compute_fn: Callable[[], Any],
+        *,
+        size_estimator: Callable[[Any], int] | None = None,
+    ) -> Any:
+        with self._lock:
+            hit = self._store.get(key)
+            if hit is not None:
+                self.hits += 1
+                self._store.move_to_end(key)
+                return hit[0]
+
+        # Compute outside the lock: recomputation for an already-cached key by a
+        # concurrent caller is wasted work, not a correctness problem, and this
+        # avoids holding the lock across a potentially expensive computation.
+        value = compute_fn()
+        estimator = size_estimator or _default_size_estimate
+        nbytes = max(int(estimator(value)), 1)
+
+        with self._lock:
+            self.misses += 1
+            if nbytes > self.max_bytes:
+                # A single value larger than the whole budget must never be
+                # retained -- caching it would permanently keep the cache over
+                # budget (nothing left to evict it in favor of). Serve it to
+                # the caller without storing it; it will simply be recomputed
+                # next time. This is the correct trade for a hard budget.
+                self.oversized_skips += 1
+                return value
+            self._store[key] = (value, nbytes)
+            self._store.move_to_end(key)
+            self._bytes += nbytes
+            self._evict_if_needed()
+        return value
+
+    def _evict_if_needed(self) -> None:
+        # No "leave at least one entry" exception: an entry only ever reaches
+        # this point after already passing the oversized-value check above, so
+        # evicting everything else is always sufficient to get back under
+        # budget without having to evict the entry that was just inserted.
+        while self._bytes > self.max_bytes and self._store:
+            _, (_, nbytes) = self._store.popitem(last=False)
+            self._bytes -= nbytes
+            self.evictions += 1
+
+    def clear(self) -> None:
+        with self._lock:
+            self._store.clear()
+            self._bytes = 0
+
+    @property
+    def size_bytes(self) -> int:
+        return self._bytes
+
+    @property
+    def entry_count(self) -> int:
+        return len(self._store)
+
+    def stats(self) -> dict[str, int]:
+        return {
+            "hits": self.hits,
+            "misses": self.misses,
+            "evictions": self.evictions,
+            "oversized_skips": self.oversized_skips,
+            "entries": self.entry_count,
+            "bytes": self._bytes,
+        }
+
+
+def current_cache() -> SharedIntermediateCache | None:
+    """Return the cache for the current execution scope, or ``None`` outside one."""
+    return _CURRENT.get()
+
+
+@contextmanager
+def scoped_cache(
+    cache: SharedIntermediateCache | None = None,
+    *,
+    max_bytes: int = _DEFAULT_BUDGET_BYTES,
+) -> Iterator[SharedIntermediateCache]:
+    """Open one fresh cache scope (one batch, or one whole-array ``extract`` call).
+
+    A new :class:`SharedIntermediateCache` is created per scope unless one is
+    passed explicitly, guaranteeing intermediates never leak across scopes.
+    """
+    c = cache if cache is not None else SharedIntermediateCache(max_bytes=max_bytes)
+    token = _CURRENT.set(c)
+    try:
+        yield c
+    finally:
+        _CURRENT.reset(token)
+
+
+def array_cache_key(namespace: str, arr: Any, **params: Any) -> tuple:
+    """Build a content-addressed cache key for a numpy-convertible array.
+
+    Content hashing (rather than object identity) is used here because arrays
+    are frequently re-wrapped/copied between the caller and the feature
+    implementation; hashing the actual bytes still gives a correct cache hit
+    without relying on the same Python object surviving the whole call.
+
+    ``np.ascontiguousarray(a).tobytes()`` always makes a full copy, which is
+    wasteful for a large array whose only purpose here is to be hashed. The
+    common case -- an already C-contiguous array -- can be hashed straight
+    from its existing buffer via a ``memoryview`` with no copy at all; a copy
+    is made only for the genuinely non-contiguous case, where one is
+    unavoidable to get a well-defined byte order to hash.
+    """
+    a = np.asarray(arr)
+    buf = a.data if a.flags["C_CONTIGUOUS"] else np.ascontiguousarray(a).tobytes()
+    digest = hashlib.blake2b(buf, digest_size=16).hexdigest()
+    return (namespace, digest, a.shape, str(a.dtype), tuple(sorted(params.items())))
+
+
+def identity_cache_key(namespace: str, obj: Any, **params: Any) -> tuple:
+    """Build an object-identity cache key for objects too expensive to hash by content.
+
+    Valid only *within one execution scope*: Python guarantees ``id()`` is
+    unique among objects alive at the same time, and a scope's cache never
+    outlives the batch whose objects it was built from, so this cannot
+    collide across batches or datasets.
+    """
+    return (namespace, id(obj), tuple(sorted(params.items())))

featm/checkpoint.py

@@ -0,0 +1,231 @@
+"""Checkpoint / resume for long out-of-core extraction runs.
+
+A checkpoint directory holds one small JSON manifest (written atomically —
+temp file + ``os.replace``, so a crash mid-write cannot leave a corrupt
+manifest that looks valid) plus any fitted-state artifacts and the partial
+disk-backed output written so far.
+
+Granularity: checkpointing is at the *batch* level for one execution plan
+(the whole set of selected features run together), not at the sub-batch
+level and not independently per feature. A run has up to two phases:
+
+- ``fit``   -- only present if at least one selected feature is stateful;
+              scans the source once, merging each stateful feature's state,
+              checkpointing the running accumulator after every batch so an
+              interrupted fit resumes mid-pass rather than from scratch.
+- ``transform`` -- scans the source (again, if there was a fit phase) computing
+              every selected feature per batch and writing one output shard
+              per batch, checkpointing after each shard.
+
+On resume, already-completed batches for the active phase are skipped by
+re-iterating the source and discarding them (cheap: no feature computation
+runs against a skipped batch) rather than recomputing anything. Compatibility
+is verified against a fingerprint of the source, schema, selected features
+and parameters, and the relevant execution settings before any work is
+reused; an incompatible checkpoint is rejected with a clear explanation
+rather than silently reused.
+"""
+from __future__ import annotations
+
+import json
+import os
+import pickle
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+CHECKPOINT_MANIFEST_VERSION = 3
+MANIFEST_FILENAME = "checkpoint.json"
+
+
+class CheckpointIncompatibleError(RuntimeError):
+    """Raised when an existing checkpoint does not match the requested run."""
+
+
+@dataclass
+class CheckpointIdentity:
+    """Everything that must match between a checkpoint and the run resuming it."""
+
+    featm_version: str
+    source_fingerprint: str
+    schema_fingerprint: str
+    features: list[str]
+    params: dict[str, dict[str, Any]]
+    settings: dict[str, Any]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "featm_version": self.featm_version,
+            "source_fingerprint": self.source_fingerprint,
+            "schema_fingerprint": self.schema_fingerprint,
+            "features": list(self.features),
+            "params": self.params,
+            "settings": self.settings,
+        }
+
+
+def _atomic_write_json(path: Path, payload: dict) -> None:
+    tmp = path.with_suffix(path.suffix + f".tmp{os.getpid()}")
+    tmp.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8")
+    os.replace(tmp, path)
+
+
+def _safe_filename(name: str) -> str:
+    return "".join(c if c.isalnum() or c in "._-" else "_" for c in name)
+
+
+class CheckpointManager:
+    def __init__(self, directory: str | Path) -> None:
+        self.directory = Path(directory)
+        self.directory.mkdir(parents=True, exist_ok=True)
+        self.output_dir = self.directory / "output"
+        self.state_dir = self.directory / "state"
+        self.manifest_path = self.directory / MANIFEST_FILENAME
+
+    # --- manifest lifecycle -------------------------------------------------
+    def load(self) -> dict[str, Any] | None:
+        if not self.manifest_path.exists():
+            return None
+        manifest = json.loads(self.manifest_path.read_text(encoding="utf-8"))
+        if not isinstance(manifest, dict):
+            raise CheckpointIncompatibleError(
+                f"Existing checkpoint manifest at {self.manifest_path} is invalid: "
+                "its JSON root must be an object. Use a fresh checkpoint directory."
+            )
+        return manifest
+
+    def _check_compatible(self, existing: dict[str, Any], identity: CheckpointIdentity) -> None:
+        # manifest_version is compared explicitly (local checkpoint
+        # replay-history closure pass, v0.4.3 -> v0.4.4): a checkpoint
+        # written under an older manifest schema (v1, before failure/
+        # degradation history was persisted at all) must be rejected as
+        # incompatible on resume, not silently reused -- an old manifest
+        # simply has no `fit_failed_features`/`transform_priming_failed_
+        # features`/`degraded_batches_by_feature` keys to restore, which
+        # would silently reintroduce the exact history-loss bug this pass
+        # fixes for any pre-existing checkpoint. A versioned package string
+        # (`featm_version`) is not a substitute for this -- the schema, not
+        # just the code, changed.
+        if existing.get("manifest_version") != CHECKPOINT_MANIFEST_VERSION:
+            raise CheckpointIncompatibleError(
+                f"Checkpoint at {self.directory} was written under manifest schema version "
+                f"{existing.get('manifest_version')!r}, but this version of featm requires "
+                f"{CHECKPOINT_MANIFEST_VERSION!r} (older checkpoints do not persist enough failure/degradation "
+                "history to resume with a replay-equivalent report). Use a fresh checkpoint directory."
+            )
+        wanted = identity.to_dict()
+        mismatches = []
+        for key, want in wanted.items():
+            got = existing.get(key)
+            if got != want:
+                mismatches.append(f"{key}: checkpoint has {got!r}, requested run has {want!r}")
+        if mismatches:
+            raise CheckpointIncompatibleError(
+                f"Checkpoint at {self.directory} is not compatible with this run:\n  "
+                + "\n  ".join(mismatches)
+                + "\nUse a fresh checkpoint directory, or match the source/features/params/settings exactly."
+            )
+
+    def initialize_or_resume(self, identity: CheckpointIdentity, *, has_fit_phase: bool) -> dict[str, Any]:
+        """Load a compatible existing manifest, or create a fresh one.
+
+        Raises :class:`CheckpointIncompatibleError` if a checkpoint exists at
+        this directory but does not match ``identity``.
+        """
+        existing = self.load()
+        if existing is not None:
+            self._check_compatible(existing, identity)
+            return existing
+        manifest = identity.to_dict()
+        manifest.update(
+            {
+                "manifest_version": CHECKPOINT_MANIFEST_VERSION,
+                "phase": "fit" if has_fit_phase else "transform",
+                "fit_batches_completed": 0,
+                "fit_accumulator_file": None,
+                "fit_state_files": {},
+                # Local checkpoint replay-history closure pass (v0.4.3 ->
+                # v0.4.4): durable failure/degradation metadata, so a resumed
+                # (partial or zero-read "done") run's report is
+                # replay-equivalent to what an uninterrupted run would have
+                # reported -- not silently clean merely because the feature
+                # that failed left no successfully-fitted-state file behind
+                # to key a restore off of. Each `..._failed_features` value
+                # is ``{"kind": "failed" | "optional_dependency", "message": str}``
+                # so a restore can call the matching ExecutionReport method.
+                "fit_failed_features": {},
+                "transform_priming_failed_features": {},
+                "degraded_batches_by_feature": {},
+                "transform_batches_completed": 0,
+                "transform_shards_written": 0,
+                "done": False,
+            }
+        )
+        self.save(manifest)
+        return manifest
+
+    def save(self, manifest: dict[str, Any]) -> None:
+        _atomic_write_json(self.manifest_path, manifest)
+
+    # --- fitted state artifacts ----------------------------------------------
+    def save_fit_state(self, feature_name: str, state: Any) -> str:
+        self.state_dir.mkdir(parents=True, exist_ok=True)
+        filename = f"{_safe_filename(feature_name)}.pkl"
+        path = self.state_dir / filename
+        tmp = path.with_suffix(path.suffix + f".tmp{os.getpid()}")
+        with open(tmp, "wb") as fh:
+            pickle.dump(state, fh)
+        os.replace(tmp, path)
+        return filename
+
+    def load_fit_state(self, filename: str) -> Any:
+        with open(self.state_dir / filename, "rb") as fh:
+            return pickle.load(fh)
+
+    def save_fit_accumulators(self, accumulators: dict[str, Any], batch_index: int) -> str:
+        """Persist in-progress fit accumulators as a *versioned* snapshot, named by batch index.
+
+        This does NOT update the manifest -- the caller must separately call
+        :meth:`save` with the manifest referencing this filename once it wants
+        to commit to this snapshot as the new resume point. Two files being
+        each individually atomic (this one, and the manifest) does not make
+        writing both together atomic: a crash between the two would otherwise
+        leave the manifest pointing at (and reprocessing) a batch whose
+        accumulator snapshot has already moved past it, double-counting that
+        batch's contribution to the fitted state. Versioning by batch index
+        means an orphaned snapshot from a crash before the manifest commit is
+        simply never referenced and is harmless (cleaned up opportunistically
+        by :meth:`cleanup_orphan_accumulator_snapshots`).
+        """
+        self.state_dir.mkdir(parents=True, exist_ok=True)
+        filename = f"_fit_accumulators-{batch_index:010d}.pkl"
+        path = self.state_dir / filename
+        tmp = path.with_suffix(path.suffix + f".tmp{os.getpid()}")
+        with open(tmp, "wb") as fh:
+            pickle.dump(accumulators, fh)
+        os.replace(tmp, path)
+        return filename
+
+    def load_fit_accumulators(self, filename: str) -> dict[str, Any] | None:
+        """Load exactly the snapshot the manifest committed to -- never "the latest on disk"."""
+        path = self.state_dir / filename
+        if not path.exists():
+            from .matrix import CheckpointCorruptionError
+            raise CheckpointCorruptionError(f"Committed fit snapshot is missing: {path}")
+        try:
+            with open(path, "rb") as fh:
+                return pickle.load(fh)
+        except (OSError, pickle.UnpicklingError, EOFError) as exc:
+            from .matrix import CheckpointCorruptionError
+            raise CheckpointCorruptionError(f"Committed fit snapshot is unreadable: {path}") from exc
+
+    def cleanup_orphan_accumulator_snapshots(self, keep_filename: str | None) -> None:
+        """Remove old/orphaned accumulator snapshots, keeping only the manifest-committed one."""
+        if not self.state_dir.exists():
+            return
+        for p in self.state_dir.glob("_fit_accumulators-*.pkl"):
+            if keep_filename is None or p.name != keep_filename:
+                try:
+                    p.unlink()
+                except OSError:
+                    pass

featm/cli.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+import argparse
+import json
+from dataclasses import asdict
+
+from .engine import feature_info, list_families, list_features, list_modalities, search
+
+
+def _spec_dict(spec):
+    data = asdict(spec)
+    data.pop("implementation", None)
+    cost = data.get("cost")
+    if hasattr(cost, "value"):
+        data["cost"] = cost.value
+    return data
+
+
+def _print_specs(specs) -> None:
+    for spec in specs:
+        suffix = f"  [{', '.join(spec.requires)}]" if spec.requires else ""
+        print(f"{spec.name}{suffix}")
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="featm",
+        description="Browse the featm feature registry.",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    p_list = sub.add_parser("list", help="List registered feature names")
+    p_list.add_argument("pattern", nargs="?", default=None)
+    p_list.add_argument("--modality", default=None)
+    p_list.add_argument("--family", default=None)
+
+    p_search = sub.add_parser("search", help="Search feature names/descriptions/tags")
+    p_search.add_argument("query")
+
+    p_info = sub.add_parser("info", help="Show metadata for one feature")
+    p_info.add_argument("name")
+    p_info.add_argument("--json", action="store_true")
+
+    sub.add_parser("modalities", help="List available modalities")
+
+    p_families = sub.add_parser("families", help="List feature families")
+    p_families.add_argument("--modality", default=None)
+
+    p_run = sub.add_parser("run", help="Run an out-of-core extraction against a file source")
+    p_run.add_argument("source", help="Path to a CSV/TSV/JSONL/Parquet/Arrow file")
+    p_run.add_argument("--features", nargs="+", required=True, help="Explicit feature names to extract")
+    p_run.add_argument(
+        "--column", action="append", default=[], metavar="FEATURE=COLUMN",
+        help="Explicit input column for one tabular feature, e.g. tabular.scaling.zscore=price. Repeatable.",
+    )
+    p_run.add_argument("--batch-rows", type=int, default=None)
+    p_run.add_argument("--memory-budget", default=None, help="e.g. 2GB")
+    p_run.add_argument("--disk-budget", default=None, help="e.g. 50GB")
+    p_run.add_argument("--output", choices=["auto", "memory", "disk"], default="auto")
+    p_run.add_argument("--output-path", default=None, help="Write the result here (.parquet or .csv)")
+    p_run.add_argument("--mode", choices=["exact", "approximate", "auto"], default="exact")
+    p_run.add_argument("--errors", choices=["raise", "warn", "skip"], default="skip")
+    p_run.add_argument("--checkpoint", default=None, help="Checkpoint directory for resumable runs")
+    p_run.add_argument("--json", action="store_true", help="Print the execution report as JSON")
+
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+
+    if args.command == "list":
+        for name in list_features(args.pattern, modality=args.modality, family=args.family):
+            print(name)
+        return 0
+
+    if args.command == "search":
+        _print_specs(search(args.query))
+        return 0
+
+    if args.command == "info":
+        spec = feature_info(args.name)
+        data = _spec_dict(spec)
+        if args.json:
+            print(json.dumps(data, indent=2, default=str))
+        else:
+            for key, value in data.items():
+                print(f"{key}: {value}")
+        return 0
+
+    if args.command == "modalities":
+        for modality in list_modalities():
+            print(modality)
+        return 0
+
+    if args.command == "families":
+        for family in list_families(modality=args.modality):
+            print(family)
+        return 0
+
+    if args.command == "run":
+        from .engine import extract as _extract
+
+        columns = {}
+        for item in args.column:
+            key, _, value = item.partition("=")
+            if not _:
+                print(f"--column expects FEATURE=COLUMN, got: {item!r}")
+                return 2
+            columns[key] = value
+
+        result = _extract(
+            args.source,
+            features=args.features,
+            columns=columns or None,
+            batch_rows=args.batch_rows,
+            memory_budget=args.memory_budget,
+            disk_budget=args.disk_budget,
+            output=args.output,
+            mode=args.mode,
+            errors=args.errors,
+            checkpoint=args.checkpoint,
+        )
+        if args.json:
+            print(json.dumps(result.report.to_dict(), indent=2, default=str))
+        else:
+            print(f"shape: {result.shape}  backend: {'disk' if result.is_disk_backed else 'memory'}")
+            print(result.report.summary())
+        if args.output_path:
+            if str(args.output_path).endswith(".csv"):
+                result.to_csv(args.output_path)
+            else:
+                result.to_parquet(args.output_path)
+            print(f"wrote {result.shape[0]} rows x {result.shape[1]} columns to {args.output_path}")
+        return 0 if not result.report.failed_features else 1
+
+    return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())