dsgrid-toolkit 0.3.3__cp313-cp313-win_amd64.whl

dsgrid/registry/registry_manager_base.py

@@ -0,0 +1,367 @@
+"""Base class for all registry managers."""
+
+import abc
+import copy
+import logging
+from pathlib import Path
+from typing import Any, Self, Type
+
+from semver import VersionInfo
+from sqlalchemy import Connection
+
+from dsgrid.config.config_base import ConfigBase
+from dsgrid.exceptions import (
+    DSGInvalidParameter,
+    DSGValueNotRegistered,
+    DSGDuplicateValueRegistered,
+)
+from dsgrid.registry.registration_context import RegistrationContext
+from dsgrid.registry.registry_interface import RegistryInterfaceBase
+from dsgrid.registry.common import RegistryManagerParams, VersionUpdateType
+
+
+logger = logging.getLogger(__name__)
+
+
+class RegistryManagerBase(abc.ABC):
+    """Base class for all registry managers."""
+
+    def __init__(self, path, params: RegistryManagerParams):
+        self._path = path
+        self._params = params
+        self._db = None
+
+        if not path.exists():
+            logger.warning(
+                "The registry data path=%s does not exist. You will able to inspect the registry "
+                "contents, but you will not be able to perform any data-related activities.",
+                path,
+            )
+
+    @property
+    @abc.abstractmethod
+    def db(self) -> RegistryInterfaceBase:
+        """Return the database interface."""
+
+    @db.setter
+    @abc.abstractmethod
+    def db(self, db: RegistryInterfaceBase) -> None:
+        """Return the database interface."""
+
+    @classmethod
+    def load(cls, path, params, db, *args: Any, **kwargs: Any) -> Self:
+        """Load the registry manager.
+
+        path : str
+        params : RegistryManagerParams
+
+        Returns
+        -------
+        RegistryManagerBase
+
+        """
+        mgr = cls(path, params)
+        mgr.db = db
+        return mgr
+
+    @classmethod
+    def _load(cls, path, params: RegistryManagerParams, *args):
+        mgr = cls(path, params, *args)
+        return mgr
+
+    @staticmethod
+    @abc.abstractmethod
+    def config_class() -> Type:
+        """Return the class used for storing the config."""
+
+    @abc.abstractmethod
+    def get_by_id(
+        self, config_id: str, version: str | None = None, conn: Connection | None = None
+    ) -> ConfigBase:
+        """Get the item matching matching ID. Returns from cache if already loaded.
+
+        Parameters
+        ----------
+        config_id : str
+        version : str
+            If None, return the latest version.
+
+        Returns
+        -------
+        DSGBaseModel
+
+        Raises
+        ------
+        DSGValueNotRegistered
+            Raised if the ID is not stored.
+
+        """
+
+    @staticmethod
+    @abc.abstractmethod
+    def name() -> str:
+        """Return the name of the registry, used for reporting.
+
+        Returns
+        -------
+        str
+
+        """
+
+    @abc.abstractmethod
+    def register(self, *args: Any, **kwargs: Any) -> Any:
+        """Registers a config file in the registry.
+
+        Raises
+        ------
+        ValueError
+            Raised if the config_file is invalid.
+        DSGDuplicateValueRegistered
+            Raised if the config ID is already registered.
+
+        """
+
+    @abc.abstractmethod
+    def register_from_config(self, config: ConfigBase, *args: Any, **kwargs) -> Any:
+        """Registers a config file in the registry.
+
+        Raises
+        ------
+        ValueError
+            Raised if the config_file is invalid.
+        DSGDuplicateValueRegistered
+            Raised if the config ID is already registered.
+
+        """
+
+    @abc.abstractmethod
+    def update_from_file(self, *args: Any, **kwargs: Any) -> ConfigBase:
+        """Updates the current registry with new parameters or data from a config file.
+
+        Raises
+        ------
+        ValueError
+            Raised if the config_file is invalid.
+        DSGInvalidParameter
+            Raised if config_id does not match config_file.
+            Raised if the version is not the current version.
+
+        """
+
+    @abc.abstractmethod
+    def update(
+        self,
+        config: ConfigBase,
+        *args: Any,
+        **kwargs: Any,
+    ) -> ConfigBase:
+        """Updates the current registry with new parameters or data.
+
+        Raises
+        ------
+        ValueError
+            Raised if the config_file is invalid.
+        DSGInvalidParameter
+            Raised if config_id does not match config_file.
+            Raised if the version is not the current version.
+
+        """
+
+    def _check_update(
+        self, conn: Connection, config: ConfigBase, config_id: str, version: str
+    ) -> None:
+        if config.config_id != config_id:
+            msg = f"ID={config_id} does not match ID in file: {config.config_id}"
+            raise DSGInvalidParameter(msg)
+
+        cur_version = self.get_latest_version(config_id, conn=conn)
+        if version != cur_version:
+            msg = f"version={version} is not current. Current={cur_version}"
+            raise DSGInvalidParameter(msg)
+
+    @staticmethod
+    def get_next_version(version: str, update_type: VersionUpdateType):
+        ver = VersionInfo.parse(version)
+        if update_type == VersionUpdateType.MAJOR:
+            next_version = ver.bump_major()
+        elif update_type == VersionUpdateType.MINOR:
+            next_version = ver.bump_minor()
+        elif update_type == VersionUpdateType.PATCH:
+            next_version = ver.bump_patch()
+        else:
+            msg = f"invalid version {update_type=}"
+            raise NotImplementedError(msg)
+
+        return str(next_version)
+
+    def _update_config(self, config, context: RegistrationContext):
+        config_id = config.config_id
+        cur_version = config.model.version
+        new_model = copy.deepcopy(config.model)
+        new_model.version = self.get_next_version(cur_version, context.registration.update_type)
+        updated_model = self.db.update(context.connection, new_model, context.registration)
+        logger.info(
+            "Updated registry and config information for %s ID=%s version=%s",
+            self.name(),
+            config_id,
+            updated_model.version,
+        )
+        return updated_model
+
+    def _check_if_already_registered(self, conn: Connection, config_id):
+        if self.db.has(conn, config_id):
+            msg = f"{self.name()}={config_id}"
+            raise DSGDuplicateValueRegistered(msg)
+
+    def _check_if_not_registered(self, conn: Connection, config_id):
+        if not self.db.has(conn, config_id):
+            msg = f"{self.name()}={config_id}"
+            raise DSGValueNotRegistered(msg)
+
+    def _log_offline_mode_prefix(self):
+        return "* OFFLINE MODE * |" if self.offline_mode else ""
+
+    @property
+    def cloud_interface(self):
+        """Return the CloudStorageInterface to sync remote data."""
+        return self._params.cloud_interface
+
+    @cloud_interface.setter
+    def cloud_interface(self, cloud_interface):
+        """Set the CloudStorageInterface (used in testing)"""
+        self._params = self._params._replace(cloud_interface=cloud_interface)
+
+    def dump(
+        self,
+        config_id,
+        directory,
+        version=None,
+        conn: Connection | None = None,
+        force: bool = False,
+    ):
+        """Dump the config file to directory.
+
+        Parameters
+        ----------
+        config_id : str
+        directory : str
+        version : VersionInfo | None
+            Defaults to current version.
+        force : bool
+            If True, overwrite files if they exist.
+
+        """
+        path = Path(directory)
+        path.mkdir(exist_ok=True, parents=True)
+        config = self.get_by_id(config_id, version, conn=conn)
+        filename = config.serialize(path, force=force)
+        logger.info(
+            "Dumped config for type=%s ID=%s version=%s to %s",
+            self.name(),
+            config_id,
+            config.model.version,
+            filename,
+        )
+
+    def finalize_registration(self, conn: Connection, config_ids: set[str], error_occurred: bool):
+        """Peform final actions after a registration process.
+
+        Parameters
+        ----------
+        config_ids : set[str]
+            Config IDs that were registered
+        error_occurred : bool
+            Set to True if an error occurred and all intermediately-registered IDs should be
+            removed.
+        """
+
+    @property
+    def fs_interface(self):
+        """Return the FilesystemInterface to list directories and read/write files."""
+        return self._params.fs_interface
+
+    @property
+    def offline_mode(self):
+        """Return True if there is to be no syncing with the remote registry."""
+        return self._params.offline
+
+    def get_latest_version(self, config_id, conn: Connection | None = None):
+        """Return the current version in the registry.
+
+        Returns
+        -------
+        str
+
+        """
+        return self.db.get_latest_version(conn, config_id)
+
+    def get_registry_data_directory(self, config_id):
+        """Return the directory containing data for config_id (parquet files).
+
+        Parameters
+        ----------
+        config_id : str
+
+        Returns
+        -------
+        str
+
+        """
+        return Path(self._params.base_path) / "data" / config_id
+
+    def has_id(self, config_id, version=None, conn: Connection | None = None):
+        """Return True if an item matching the parameters is stored.
+
+        Parameters
+        ----------
+        config_id : str
+        version : str
+            If None, use latest.
+
+        Returns
+        -------
+        bool
+
+        """
+        return self.db.has(conn, config_id, version=version)
+
+    def iter_configs(self, conn: Connection | None = None):
+        """Return an iterator over the registered configs."""
+        for config_id in self.iter_ids(conn):
+            yield self.get_by_id(config_id, conn=conn)
+
+    def iter_ids(self, conn: Connection | None = None):
+        """Return an iterator over the registered dsgrid IDs."""
+        yield from self.db.list_model_ids(conn)
+
+    def list_ids(self, conn: Connection | None = None, **kwargs: Any):
+        """Return the IDs.
+
+        Returns
+        -------
+        list
+
+        """
+        return sorted(self.iter_ids(conn))
+
+    def relative_remote_path(self, path):
+        """Return relative remote registry path."""
+        relative_path = Path(path).relative_to(self._params.base_path)
+        remote_path = f"{self._params.remote_path}/{relative_path}"
+        return remote_path
+
+    @abc.abstractmethod
+    def remove(self, config_id: str, conn: Connection | None = None) -> None:
+        """Remove an item from the registry.
+
+        Parameters
+        ----------
+        config_id : str
+
+        Raises
+        ------
+        DSGValueNotRegistered
+            Raised if the project_id is not registered.
+
+        """
+        # TODO: Do we want to handle specific versions? This removes all configs.

dsgrid/registry/versioning.py

@@ -0,0 +1,92 @@
+import os
+import json5
+
+DATASET_REGISTRY_PATH = "registry/datasets/"
+PROJECT_REGISTRY_PATH = "registry/projects/"
+DIMENSION_REGISTRY_PATH = "registry/dimensions/"
+
+
+def versioning(registry_type, id_handle, update):
+    """Determine registration or project version for registration.
+
+    TODO: Current solution is a quick hack. This needs to be better/formalized.
+        - Need smarter version updating / checks; use semvar packages
+        - Set to work with some central version (like S3)
+        - Currently only updating major version
+        - NOTE: not currently utilitzing the update_type in
+                ConfigRegistrationDetails. Could use this to set
+                major/minor/patch update decisiosns
+
+    Args:
+        registry_type (RegistryType): type of registry (e.g., Project, Dataset)
+        id_handle (str): ID handle is either the project_id or dataset_id
+        update (bool): config registration update setting
+    """
+
+    # get registry path
+    if registry_type == "dataset":
+        registry_path = DATASET_REGISTRY_PATH
+    if registry_type == "project":
+        registry_path = PROJECT_REGISTRY_PATH
+    if registry_type == "dimension":
+        registry_path = DIMENSION_REGISTRY_PATH
+
+    # TODO: remove when done. project path should be set somewhere else
+    if not os.path.exists(registry_path):
+        msg = f"Path does not exist: {registry_path}"
+        raise ValueError(msg)
+
+    # if config.update is False, then assume major=1, minor=0, patch=0
+    if not update:
+        version = f"{id_handle}-v1.0.0"
+        registry_file = f"{registry_path}/{version}.json5"
+        # Raise error if v1.0.0 registry exists for project_id
+        if os.path.exists(registry_file):
+            msg = (
+                f'{registry_type} registry for "{registry_file}" already '
+                f"exists. If you want to update the project registration"
+                f" with a new {registry_type} version, then you will need to"
+                f" set update=True in {registry_type} config. Alternatively, "
+                f"if you want to initiate a new dsgrid {registry_type}, you "
+                "will need to specify a new version handle in the "
+                f"{registry_type} config."
+            )
+            raise ValueError(msg)
+    # if update is true...
+    else:
+        # list existing project registries
+        existing_versions = []
+        for f in os.listdir(registry_path):
+            if f.startswith(id_handle):
+                existing_versions.append(int(f.split("-v")[1].split(".")[0]))
+        # check for existing project registries
+        if len(existing_versions) == 0:
+            msg = (
+                "Registration.update=True, however, no updates can be made "
+                f"because there are no existing registries for {registry_type}"
+                f" ID = {id_handle}. Check project_id or set "
+                f"Registration.update=True in the {registry_type} Config."
+            )
+            raise ValueError(msg)
+        # find the latest registry version
+        # NOTE: this is currently based on major verison only
+        last_vmajor_nbr = sorted(existing_versions)[-1]
+        old_project_version = f"{id_handle}-v{last_vmajor_nbr}.0.0"
+        old_registry_file = f"{registry_path}/{old_project_version}.json5"
+
+        # depricate old project registry
+        t = json5.load(old_registry_file)
+        t["status"] = "Deprecated"
+        with open(old_registry_file.format(**locals()), "w") as f:
+            json5.dump(t, f)
+
+        # update version
+        # TODO NEED REAL LOGIC FOR THIS!
+        #   - Currently assuming only major version is being updated
+        major = int(last_vmajor_nbr) + 1
+        minor = 0  # TODO: assume 0 for now
+        patch = 0  # TODO: assume 0 for now
+
+        version = f"{id_handle}-v{major}.{minor}.{patch}"
+
+    return version

dsgrid/rust_ext/__init__.py

@@ -0,0 +1,14 @@
+"""Python wrapper for Rust-based pattern finding functionality."""
+
+from dsgrid.rust_ext.find_minimal_patterns import find_minimal_patterns_from_file
+
+try:
+    from dsgrid.minimal_patterns import Pattern, PatternConfig
+except ImportError as e:
+    msg = (
+        "Failed to import minimal_patterns Rust extension. "
+        "Make sure the package was built with maturin: `pip install -e .` or `maturin develop`"
+    )
+    raise ImportError() from e
+
+__all__ = ["Pattern", "PatternConfig", "find_minimal_patterns_from_file"]

dsgrid/rust_ext/find_minimal_patterns.py

@@ -0,0 +1,129 @@
+"""Python wrapper for Rust-based pattern finding functionality."""
+
+import csv
+import logging
+import shutil
+from collections import defaultdict
+from pathlib import Path
+
+try:
+    from dsgrid.minimal_patterns import Pattern, PatternConfig, find_minimal_patterns
+except ImportError as e:
+    msg = (
+        "Failed to import minimal_patterns Rust extension. "
+        "Make sure the package was built with maturin: `pip install -e .` or `maturin develop`"
+    )
+    raise ImportError() from e
+
+logger = logging.getLogger(__name__)
+
+
+def find_minimal_patterns_from_file(
+    file_path: str | Path,
+    max_depth: int = 0,
+    prune_miss_empty: bool = True,
+    ratio_threshold: float = 50.0,
+    threads: int = 0,
+    verbose: bool = False,
+    output_dir: str | Path | None = "missing_associations",
+) -> list[Pattern]:
+    """Find minimal closed patterns in a Parquet file containing categorical data.
+
+    This function analyzes a Parquet file to discover minimal closed patterns -
+    the simplest column combinations that characterize complete subsets of your data.
+    Patterns are grouped by their column combinations and written to CSV files.
+
+    Parameters
+    ----------
+    file_path : str | Path
+        Path to the input Parquet file
+    max_depth : int, optional
+        Maximum pattern size (number of columns). 0 = unlimited. Default: 0.
+    prune_miss_empty : bool, optional
+        Prune patterns with no matching rows (recommended: True). Default: True.
+    ratio_threshold : float, optional
+        Ratio threshold for pruning. Default: 50.0.
+    threads : int, optional
+        Number of threads to use (0 = use all available cores). Default: 0.
+    verbose : bool, optional
+        Enable verbose progress output. Default: False.
+    output_dir : str | Path | None, optional
+        Directory to write CSV files grouping patterns by column combinations.
+        Each unique combination of columns produces a separate CSV file named
+        ``<col1>__<col2>__...__<colN>.csv``. If None, no files are written.
+        Default: "missing_associations".
+
+    Returns
+    -------
+    list[Pattern]
+        List of Pattern objects, each containing:
+
+        - pattern_id : Unique identifier
+        - columns : List of column names in the pattern
+        - values : List of values for each column
+        - num_rows : Number of rows matching this pattern
+
+    Raises
+    ------
+    RuntimeError
+        If there's an error reading the Parquet file or finding patterns.
+
+    Examples
+    --------
+    >>> from dsgrid.rust_ext import find_patterns_from_parquet
+    >>> patterns = find_patterns_from_parquet("missing_records.parquet", max_depth=3, verbose=True)
+    >>> for p in patterns:
+    ...     print(f"Pattern {p.pattern_id}: {p.columns} = {p.values} ({p.num_rows} rows)")
+    """
+    config = PatternConfig(
+        max_depth=max_depth,
+        prune_miss_empty=prune_miss_empty,
+        ratio_threshold=ratio_threshold,
+        threads=threads,
+        verbose=verbose,
+    )
+
+    parquet_path_str = str(file_path)
+    logger.info("Finding minimal closed patterns in %s", parquet_path_str)
+
+    patterns = find_minimal_patterns(parquet_path_str, config)
+
+    logger.info("Found %d minimal closed patterns", len(patterns))
+
+    if output_dir is not None:
+        _write_patterns_to_csv(patterns, output_dir)
+
+    return patterns
+
+
+def _write_patterns_to_csv(patterns: list[Pattern], output_dir: str | Path) -> None:
+    """Write patterns to CSV files grouped by column combinations.
+
+    Parameters
+    ----------
+    patterns : list[Pattern]
+        List of Pattern objects to write.
+    output_dir : str | Path
+        Directory to write CSV files to.
+    """
+    output_path = Path(output_dir)
+    if output_path.exists():
+        shutil.rmtree(output_path)
+    output_path.mkdir(parents=True)
+
+    grouped: dict[tuple[str, ...], list[Pattern]] = defaultdict(list)
+    for pattern in patterns:
+        key = tuple(pattern.columns)
+        grouped[key].append(pattern)
+
+    for columns, group_patterns in grouped.items():
+        filename = "__".join(columns) + ".csv"
+        filepath = output_path / filename
+
+        with open(filepath, "w", newline="") as f:
+            writer = csv.writer(f)
+            writer.writerow(list(columns))
+            for pattern in group_patterns:
+                writer.writerow(pattern.values)
+
+        logger.info("Wrote %d patterns to %s", len(group_patterns), filepath)