diffract-core 0.2.1__py3-none-any.whl

diffract/__init__.py

@@ -0,0 +1,52 @@
+"""Diffract: neural network weight analysis and evolution tracking.
+
+Quick start::
+
+    from diffract import Session
+
+    # In-memory session (no persistence)
+    session = Session(profile="ram")
+
+    # Persistent local session (SQLite in .diffract/)
+    session = Session(profile="local")
+
+    # Or with a custom config file
+    session = Session(config_path="my_config.ini")
+
+Available profiles: "ram", "local", "hybrid".
+Use ``diffract.list_profiles()`` to see all options.
+"""
+
+from __future__ import annotations
+
+import importlib
+from typing import TYPE_CHECKING, Any
+
+__version__ = "0.2.1"
+
+from .containers import list_profiles
+from .core.data.nn.params.schema import ParameterType
+from .session import ParameterOverrides, Session
+
+if TYPE_CHECKING:
+    from . import viz as viz
+
+__all__ = [
+    "ParameterOverrides",
+    "ParameterType",
+    "Session",
+    "__version__",
+    "list_profiles",
+    "viz",
+]
+
+
+def __getattr__(name: str) -> Any:
+    if name == "viz":
+        return importlib.import_module(".viz", __name__)
+    msg = f"module {__name__!r} has no attribute {name!r}"
+    raise AttributeError(msg)
+
+
+def __dir__() -> list[str]:
+    return sorted(__all__)

diffract/configs/fast_speed_without_disk.ini

@@ -0,0 +1,48 @@
+; Fast configuration: RAM storage, no cache
+
+[logging]
+version = 1
+disable_existing_loggers = false
+
+[logging.formatters.standard]
+format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+datefmt = "%Y-%m-%d %H:%M:%S"
+
+[logging.handlers.console]
+class = logging.StreamHandler
+level = WARNING
+formatter = standard
+stream = ext://sys.stdout
+
+[logging.loggers.diffract]
+level = INFO
+handlers = ["console"]
+propagate = false
+
+[logging.root]
+level = WARNING
+handlers = ["console"]
+
+[storage]
+backend = "ram"
+
+[metadata]
+backend = "sqlite"
+
+[metadata.sqlite]
+path = ":memory:"
+
+[cache]
+backend = "none"
+
+[parallel.thread_pool]
+max_workers = 8
+
+[parallel.process_pool]
+max_workers = 8
+
+[nn.extractor]
+skip_not_implemented_types = true
+
+[export]
+default_export_format = "dict"

diffract/configs/hybrid.ini

@@ -0,0 +1,74 @@
+[logging]
+version = 1
+disable_existing_loggers = false
+
+[logging.formatters.standard]
+format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+datefmt = "%Y-%m-%d %H:%M:%S"
+
+[logging.handlers.console]
+class = logging.StreamHandler
+level = INFO
+formatter = standard
+stream = ext://sys.stdout
+
+[logging.handlers.file]
+class = logging.handlers.RotatingFileHandler
+level = INFO
+formatter = standard
+filename = ".diffract/hybrid.log"
+maxBytes = 1048576
+backupCount = 1
+
+[logging.loggers.diffract]
+level = INFO
+handlers = ["console", "file"]
+propagate = false
+
+[logging.root]
+level = INFO
+handlers = ["console", "file"]
+
+[storage]
+backend = "hybrid"
+
+[storage.hybrid]
+light = "sqlite"
+heavy = "hdf5"
+array_threshold = 1048576
+
+[storage.sqlite]
+path = ".diffract/hybrid/sqlite_storage.db"
+blob_write_workers = 8
+array_threshold = 1048576
+read_mmap_size = 4294967296
+write_mmap_size = 4294967296
+batch_size_limit_bytes = 4294967296
+
+[storage.hdf5]
+path = ".diffract/hybrid/hdf5_storage.h5"
+batch_size_limit_bytes = 4294967296
+
+[metadata]
+backend = "sqlite"
+
+[metadata.sqlite]
+path = ".diffract/hybrid/metadata_index.db"
+
+[cache]
+backend = "simple"
+
+[cache.simple]
+max_memory_mb = 128_000
+
+[parallel.thread_pool]
+max_workers = 8
+
+[parallel.process_pool]
+max_workers = 8
+
+[nn.extractor]
+skip_not_implemented_types = true
+
+[export]
+default_export_format = "pandas"

diffract/configs/sqlite.ini

@@ -0,0 +1,62 @@
+; SQLite-backed storage configuration
+
+[logging]
+version = 1
+disable_existing_loggers = false
+
+[logging.formatters.standard]
+format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+datefmt = "%Y-%m-%d %H:%M:%S"
+
+[logging.handlers.console]
+class = logging.StreamHandler
+level = WARNING
+formatter = standard
+stream = ext://sys.stdout
+
+[logging.handlers.file]
+class = logging.handlers.RotatingFileHandler
+level = INFO
+formatter = standard
+filename = ".diffract/sqlite.log"
+maxBytes = 1048576
+backupCount = 1
+
+[logging.loggers.diffract]
+level = DEBUG
+handlers = ["console", "file"]
+propagate = false
+
+[logging.root]
+level = DEBUG
+handlers = ["console", "file"]
+
+[storage]
+backend = "sqlite"
+
+[storage.sqlite]
+path = ".diffract/sqlite/sqlite_storage.db"
+
+[metadata]
+backend = "sqlite"
+
+[metadata.sqlite]
+path = ".diffract/sqlite/metadata_index.db"
+
+[cache]
+backend = "simple"
+
+[cache.simple]
+max_memory_mb = 4096
+
+[parallel.thread_pool]
+max_workers = 8
+
+[parallel.process_pool]
+max_workers = 8
+
+[nn.extractor]
+skip_not_implemented_types = true
+
+[export]
+default_export_format = "pandas"

diffract/containers.py

@@ -0,0 +1,405 @@
+"""Main application dependency injection container.
+
+This module provides the MainContainer class that orchestrates all subsystem
+containers for the diffract application, including storage, cache, compute,
+model parameters, and export functionality.
+"""
+
+from __future__ import annotations
+
+import configparser
+import contextlib
+import json
+import logging
+import os
+import re
+from pathlib import Path
+from typing import Any
+
+from dependency_injector import containers, providers
+
+from .core.cache.containers import CacheContainer
+from .core.compute.containers import (
+    ComputeContainer,
+    ComputeSingletonContainer,
+    ComputeSingletonContainerWiringConfig,
+)
+from .core.data.metadata.containers import MetadataContainer
+from .core.data.nn.containers import ModelParametersContainer
+from .core.export.containers import ExportContainer
+from .core.parallel import ParallelSingletonContainer
+from .core.storage.containers import StorageContainer
+from .core.utils.exceptions import format_exception_message
+
+logger = logging.getLogger(__name__)
+
+# Built-in configuration profiles for common use cases.
+#
+# Values are relative paths to INI configs shipped with the repo/package.
+PROFILES: dict[str, str] = {
+    "ram": "configs/fast_speed_without_disk.ini",
+    "local": "configs/sqlite.ini",
+    "hybrid": "configs/hybrid.ini",
+}
+
+
+def list_profiles() -> list[str]:
+    """Return available profile names."""
+    return list(PROFILES.keys())
+
+
+def _resolve_packaged_or_repo_path(relative_path: str) -> Path:
+    """Resolve a config path within package or repository layout."""
+    candidates = [
+        # Packaged layout: <site-packages>/diffract/<relative_path>
+        Path(__file__).resolve().parent / relative_path,
+        # Repo layout: <root>/<relative_path>
+        Path(__file__).resolve().parents[2] / relative_path,
+    ]
+    for candidate in candidates:
+        if candidate.exists():
+            return candidate
+    msg = (
+        f"Config file not found for relative path {relative_path!r}. "
+        f"Tried: {', '.join(str(c) for c in candidates)}"
+    )
+    raise FileNotFoundError(msg)
+
+
+def get_profile_config(profile: str) -> dict[str, Any]:
+    """Get configuration dictionary for a profile."""
+    if profile not in PROFILES:
+        available = ", ".join(PROFILES.keys())
+        msg = f"Unknown profile '{profile}'. Available: {available}"
+        raise ValueError(msg)
+    config_path = _resolve_packaged_or_repo_path(PROFILES[profile])
+    return _parse_ini_config(config_path)
+
+
+_FALLBACK_LOGGING_CONFIG: dict[str, Any] = {
+    "version": 1,
+    "disable_existing_loggers": False,
+    "formatters": {
+        "standard": {"format": "%(asctime)s [%(levelname)s] %(name)s: %(message)s"}
+    },
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "level": "INFO",
+            "formatter": "standard",
+            "stream": "ext://sys.stdout",
+        }
+    },
+    "root": {"level": "INFO", "handlers": ["console"]},
+}
+
+
+def _expand_env_vars(value: str) -> str:
+    """Expand ${VAR} and $VAR patterns with environment variables."""
+
+    def replacer(match: re.Match[str]) -> str:
+        var_name = match.group(1) or match.group(2)
+        return os.environ.get(var_name, match.group(0))
+
+    return re.sub(r"\$\{([^}]+)\}|\$([A-Za-z_][A-Za-z0-9_]*)", replacer, value)
+
+
+def _coerce_ini_value(value: str) -> Any:
+    """Convert an INI string value into a reasonable Python type.
+
+    INI files store values as strings; for service initialization configs we
+    accept basic scalar types and JSON literals for lists/dicts.
+    Supports ${VAR} syntax for environment variable expansion.
+    """
+    raw = _expand_env_vars(value.strip())
+    lowered = raw.lower()
+
+    if lowered in {"none", "null"}:
+        return None
+    if lowered in {"true", "false"}:
+        return lowered == "true"
+
+    # Try JSON first for explicit literals like lists/dicts/quoted strings.
+    if raw.startswith(("{", "[", '"')) or raw in {"0", "1"}:
+        with contextlib.suppress(json.JSONDecodeError):
+            return json.loads(raw)
+
+    # Numeric scalars (allow underscores).
+    int_candidate = raw.replace("_", "")
+    with contextlib.suppress(ValueError):
+        return int(int_candidate)
+
+    try:
+        return float(int_candidate)
+    except ValueError:
+        return raw
+
+
+def _parse_ini_config(path: Path) -> dict[str, Any]:
+    """Parse an INI file into a nested dictionary.
+
+    Section names can use dot-notation to represent nesting, e.g.:
+      [storage.hdf5]
+      path=/tmp/main.h5
+    """
+    # Disable interpolation because logging formatters use "%(name)s" syntax.
+    parser = configparser.ConfigParser(interpolation=None)
+    # Preserve case for keys like "maxBytes" used by logging.config.dictConfig.
+    parser.optionxform = str
+    parser.read(path)
+
+    data: dict[str, Any] = {}
+    for section in parser.sections():
+        cursor: dict[str, Any] = data
+        for part in section.split("."):
+            cursor = cursor.setdefault(part, {})
+        for key, val in parser.items(section):
+            cursor[key] = _coerce_ini_value(val)
+    return data
+
+
+def _default_config_path() -> Path | None:
+    """Return the default launch config path if present."""
+    with contextlib.suppress(FileNotFoundError):
+        return _resolve_packaged_or_repo_path(PROFILES["ram"])
+    return None
+
+
+class MainContainer(containers.DeclarativeContainer):
+    """Main application container that orchestrates all subsystem containers.
+
+    This container manages the dependency injection for the entire application,
+    providing centralized configuration and wiring of all subsystem components
+    including storage, caching, compute kernels, model parameters, and export.
+    """
+
+    # Configuration
+    config = providers.Configuration()
+
+    # Logging configuration
+    logging_config = providers.Resource(
+        "logging.config.dictConfig",
+        config=config.logging,
+    )
+
+    # Subsystem containers
+    storage = providers.Container(
+        StorageContainer,
+        config=config.storage,
+    )
+
+    cache = providers.Container(
+        CacheContainer,
+        config=config.cache,
+    )
+
+    metadata = providers.Container(
+        MetadataContainer,
+        config=config.metadata,
+    )
+
+    compute_singleton = providers.Container(ComputeSingletonContainer)
+
+    parallel_singleton = providers.Container(
+        ParallelSingletonContainer,
+        config=config.parallel,
+    )
+
+    nn = providers.Container(
+        ModelParametersContainer,
+        storage_manager=storage.storage_manager_resource,
+        cache_manager=cache.cache_manager_resource,
+        metadata_index=metadata.metadata_index_resource,
+        parallel=parallel_singleton.thread_pool_context,
+        config=config.nn,
+    )
+
+    compute = providers.Container(
+        ComputeContainer,
+        config=config.compute,
+        compute_singleton=compute_singleton,
+        parallel=parallel_singleton.thread_pool_context,
+        process_pool=parallel_singleton.process_pool_context,
+        aggregate_repository=nn.aggregate_repository,
+    )
+
+    export = providers.Container(
+        ExportContainer,
+        config=config.export,
+    )
+
+
+class WiringConfiguration:
+    """Complete application wiring configuration.
+
+    Manages the dependency injection wiring for all containers in the application,
+    ensuring proper module and package registration for dependency resolution.
+    """
+
+    @classmethod
+    def get_wiring_configs(cls) -> list[dict[str, Any]]:
+        """Get all wiring configurations for the application.
+
+        Returns:
+            List of dictionaries containing container, modules, and packages
+            configuration for each subsystem that requires wiring.
+        """
+        return [
+            {
+                "container": ComputeSingletonContainer,
+                "modules": ComputeSingletonContainerWiringConfig.modules,
+                "packages": ComputeSingletonContainerWiringConfig.packages,
+            },
+        ]
+
+    @classmethod
+    def wire(cls, container: MainContainer) -> None:
+        """Wire the entire application with all subsystem containers.
+
+        Args:
+            container: The main application container to wire.
+
+        Raises:
+            Exception: If wiring fails for any container or module.
+        """
+        try:
+            # Wire main container
+            container.wire(
+                modules=[
+                    __name__,
+                    "diffract.session",
+                    "diffract.session.namespaces",
+                    "diffract.session.namespaces.models",
+                ],
+            )
+
+            # Wire subsystem containers
+            for config in cls.get_wiring_configs():
+                container_cls = config["container"]
+                parts = re.findall(r"[A-Z][^A-Z]*", container_cls.__name__)
+                parts = [p.lower() for p in parts]
+                parts.remove("container")
+                container_name = "_".join(parts)
+                container_instance = getattr(container, container_name)
+
+                if config.get("modules"):
+                    container_instance.wire(modules=config["modules"])
+
+                if config.get("packages"):
+                    container_instance.wire(packages=config["packages"])
+
+            logger.debug("Successfully wired all application containers")
+
+        except Exception:
+            logger.exception("Failed to wire application containers")
+            raise
+
+
+compute_singleton_container = ComputeSingletonContainer()
+
+
+def create_main_container(
+    config_path: str | Path | None = None,
+    profile: str | None = None,
+) -> MainContainer:
+    """Create and configure the main application container.
+
+    Args:
+        config_path: Path to configuration file (YAML/JSON/INI). Overrides
+            profile settings if both are provided.
+        profile: Built-in profile name ("ram", "local", "hybrid").
+
+    Returns:
+        Configured MainContainer instance with all subsystems wired
+        and default kernels registered.
+
+    Raises:
+        ValueError: If profile name is not recognized.
+        Exception: If container creation or wiring fails.
+    """
+    container = MainContainer(
+        compute_singleton=compute_singleton_container,
+    )
+
+    repo_root = Path(__file__).resolve().parents[2]
+    cwd = Path.cwd()
+
+    config_loaded = False
+
+    if config_path is not None:
+        config_path = Path(config_path)
+        if config_path.exists():
+            suffix = config_path.suffix.lower()
+            match suffix:
+                case ".yaml" | ".yml":
+                    container.config.from_yaml(str(config_path))
+                case ".json":
+                    container.config.from_json(str(config_path))
+                case ".ini":
+                    container.config.from_dict(_parse_ini_config(config_path))
+                case _:
+                    msg = f"Unsupported config file extension: '{suffix}'"
+                    raise ValueError(msg)
+            logger.info("Using config file: %s", config_path)
+            config_loaded = True
+        else:
+            logger.warning("Configuration file not found: %s", config_path)
+
+    if not config_loaded and profile is not None:
+        profile_config = get_profile_config(profile)
+        container.config.from_dict(profile_config)
+        logger.info("Using profile: %s", profile)
+        config_loaded = True
+
+    if not config_loaded:
+        default_path = _default_config_path()
+        if default_path is not None:
+            container.config.from_dict(_parse_ini_config(default_path))
+            logger.debug("Loaded default configuration from %s", default_path)
+
+    # Resolve relative file paths to absolute paths.
+    base_path = cwd if profile is not None else repo_root
+    resolved = container.config()
+    for key in ("path", "filename"):
+        stack: list[dict[str, Any]] = [resolved]
+        while stack:
+            cur = stack.pop()
+            for k, v in list(cur.items()):
+                if isinstance(v, dict):
+                    stack.append(v)
+                    continue
+                if k != key or not isinstance(v, str):
+                    continue
+                value = v.strip().strip('"')
+                if value.startswith(("ext://", "/", "~")):
+                    continue
+                cur[k] = str(base_path / value)
+    container.config.from_dict(resolved)
+
+    # Configs without a [logging] section get the fallback silently.
+    if not resolved.get("logging"):
+        container.config.logging.override(_FALLBACK_LOGGING_CONFIG)
+
+    # logging.config.dictConfig cannot create missing log directories itself.
+    for handler in resolved.get("logging", {}).get("handlers", {}).values():
+        filename = handler.get("filename") if isinstance(handler, dict) else None
+        if isinstance(filename, str):
+            Path(filename.strip().strip('"')).parent.mkdir(parents=True, exist_ok=True)
+
+    try:
+        container.logging_config()
+    except Exception as e:  # noqa: BLE001
+        logger.warning(
+            "Failed to configure logging; using fallback config: %s",
+            format_exception_message(e),
+        )
+        logger.debug("Logging configuration failure details", exc_info=True)
+        container.config.logging.override(_FALLBACK_LOGGING_CONFIG)
+        container.logging_config()
+
+    # Wire all containers
+    WiringConfiguration.wire(container)
+
+    container.compute_singleton.register_default_kernels()
+
+    return container

diffract/core/__init__.py

@@ -0,0 +1,23 @@
+"""Core module for diffract."""
+
+from diffract.core.constants import (
+    CONTEXT_SEPARATOR,
+    HDF5_INDEX_DATASET,
+    HDF5_INDEX_GROUP,
+    HDF5_INDEX_TOMBSTONE,
+    MODELS_CONTEXT_PREFIX,
+    PARAMS_CONTEXT_PREFIX,
+    REGEX_PREFIX,
+    WEIGHTS_FIELD,
+)
+
+__all__ = [
+    "CONTEXT_SEPARATOR",
+    "HDF5_INDEX_DATASET",
+    "HDF5_INDEX_GROUP",
+    "HDF5_INDEX_TOMBSTONE",
+    "MODELS_CONTEXT_PREFIX",
+    "PARAMS_CONTEXT_PREFIX",
+    "REGEX_PREFIX",
+    "WEIGHTS_FIELD",
+]

diffract/core/cache/__init__.py

@@ -0,0 +1,24 @@
+"""Cache management module providing flexible caching backends.
+
+This module offers a unified interface for caching with support for multiple
+backends including Redis and in-memory implementations. It uses dependency
+injection for flexible configuration and backend selection.
+
+Example:
+    >>> from diffract.core.cache import CacheContainer
+    >>> container = CacheContainer()
+    >>> cache_manager = container.cache_manager()
+"""
+
+from .containers import CacheContainer
+from .interface import UID, ICacheManager
+from .redis_manager import RedisLRUCacheManager
+from .simple_manager import SimpleLRUCacheManager
+
+__all__ = [
+    "UID",
+    "CacheContainer",
+    "ICacheManager",
+    "RedisLRUCacheManager",
+    "SimpleLRUCacheManager",
+]