dsgrid-toolkit 0.3.3__cp313-cp313-win_amd64.whl

dsgrid/filesystem/factory.py

@@ -0,0 +1,32 @@
+from pathlib import Path
+
+from .local_filesystem import LocalFilesystem
+
+# from .s3_filesystem import S3Filesystem
+
+# from dsgrid.common import AWS_PROFILE_NAME
+
+
+def make_filesystem_interface(path):
+    """Make an instance of FilesystemInterface appropriate for the path.
+
+    Parameters
+    ----------
+    path : str
+
+    Returns
+    -------
+    FilesystemInterface
+
+    """
+    if isinstance(path, Path):
+        path = str(path)
+    if path.lower().startswith("s3"):
+        msg = f"Support for S3 is currently disabled: {path=}"
+        raise NotImplementedError(msg)
+        #    path = "s3" + path[2:]
+        # fs_intf = S3Filesystem(path, AWS_PROFILE_NAME)
+    else:
+        fs_intf = LocalFilesystem()
+
+    return fs_intf

dsgrid/filesystem/filesystem_interface.py

@@ -0,0 +1,136 @@
+"""Defines interface to access all filesystems"""
+
+import abc
+from pathlib import Path
+
+
+class FilesystemInterface(abc.ABC):
+    """Interface to access and edit directories and files on a local or remote filesystem"""
+
+    @abc.abstractmethod
+    def copy_file(self, src, dst):
+        """Copy a file to a destination.
+
+        Parameters
+        ----------
+        src : str
+            Path to source file
+        dst : str
+            Path to destination file
+
+        """
+
+    @abc.abstractmethod
+    def copy_tree(self, src, dst):
+        """Copy src to dst recursively.
+
+        Parameters
+        ----------
+        src : str
+            Source directory
+        dst : str
+            Destination directory
+
+        """
+
+    @abc.abstractmethod
+    def exists(self, path):
+        """Return True if path exists.
+
+        Parameters
+        ----------
+        path : str
+
+        Returns
+        -------
+        bool
+
+        """
+
+    @abc.abstractmethod
+    def listdir(self, directory, files_only=False, directories_only=False, exclude_hidden=False):
+        """List the contents of a directory.
+
+        Parameters
+        ----------
+        directory : str
+        files_only : bool
+            only return files
+        directories_only : bool
+            only return directories
+        exclude_hidden : bool
+            exclude names starting with "."
+
+        Returns
+        -------
+        list
+            list of str
+
+        """
+
+    @abc.abstractmethod
+    def mkdir(self, directory):
+        """Make a directory. Do nothing if the directory exists.
+
+        Parameters
+        ----------
+        directory : str
+
+        """
+
+    @abc.abstractmethod
+    def path(self, path) -> Path:
+        """Return an object that meets the interface of pathlib.Path.
+
+        Parameters
+        ----------
+        path : str
+
+        Returns
+        -------
+        Path
+        """
+
+    @abc.abstractmethod
+    def rglob(
+        self,
+        directory,
+        files_only=False,
+        directories_only=False,
+        exclude_hidden=False,
+        pattern="*",
+    ):
+        """Recursively search a path and return a list of relative paths that match criteria.
+
+        Parameters
+        ----------
+        directory : str
+        files_only : bool, optional
+            Return files only, by default False
+        directories_only : bool, optional
+            Return directories only, by default False
+        exclude_hidden : bool, optional
+            Exclude hidden files, by default False
+        pattern : str, optional
+            Search for files with a specific pattern, by default "*"
+        """
+
+    @abc.abstractmethod
+    def rm_tree(self, directory):
+        """Remove all files and directories, recursively.
+
+        Parameters
+        ----------
+        directory : str
+
+        """
+
+    @abc.abstractmethod
+    def touch(self, path):
+        """Touch
+
+        Parameters
+        ----------
+        directory : str
+            filepath
+        """

dsgrid/filesystem/local_filesystem.py

@@ -0,0 +1,74 @@
+"""Implementation for local filesytem"""
+
+import logging
+import os
+import shutil
+from pathlib import Path
+
+from dsgrid.filesystem.filesystem_interface import FilesystemInterface
+
+logger = logging.getLogger(__name__)
+
+
+class LocalFilesystem(FilesystemInterface):
+    """Provides access to the local filesystem."""
+
+    def copy_file(self, src, dst):
+        return shutil.copyfile(src, dst)
+
+    def copy_tree(self, src, dst):
+        return shutil.copytree(src, dst)
+
+    def exists(self, path):
+        return os.path.exists(path)
+
+    def listdir(self, directory, files_only=False, directories_only=False, exclude_hidden=False):
+        contents = os.listdir(directory)
+        if exclude_hidden:
+            contents = [x for x in contents if not x.startswith(".")]
+        if files_only:
+            return [x for x in contents if os.path.isfile(os.path.join(directory, x))]
+        if directories_only:
+            return [x for x in contents if os.path.isdir(os.path.join(directory, x))]
+        return contents
+
+    def path(self, path) -> Path:
+        return Path(path)
+
+    def rglob(
+        self,
+        directory,
+        files_only=False,
+        directories_only=False,
+        exclude_hidden=False,
+        pattern="*",
+    ):
+        contents = [c for c in Path(directory).rglob(pattern)]
+        if exclude_hidden:
+            # NOTE: this does not currently ignore hidden directories in the path.
+            contents = [x for x in contents if not x.name.startswith(".")]
+        if files_only:
+            return [x for x in contents if os.path.isfile(x)]
+        if directories_only:
+            return [x for x in contents if os.path.isdir(x)]
+        return contents
+
+    def mkdir(self, directory):
+        os.makedirs(directory, exist_ok=True)
+
+    def rm_tree(self, directory):
+        return shutil.rmtree(directory)
+
+    def rm(self, path):
+        if os.path.exists(path):
+            if os.path.isdir(path):
+                if os.listdir(path):
+                    self.rm_tree(path)
+                else:
+                    os.removedirs(path)
+            elif os.path.isfile(path):
+                os.remove(path)
+        logger.warning("path %s does not exist", path)
+
+    def touch(self, path):
+        Path(path).touch()

dsgrid/filesystem/s3_filesystem.py

@@ -0,0 +1,118 @@
+"""Implementation for AWS S3 bucket filesystem"""
+
+import logging
+import re
+
+import boto3
+from s3path import S3Path, register_configuration_parameter
+
+from .cloud_filesystem import CloudFilesystemInterface
+
+
+logger = logging.getLogger(__name__)
+
+
+class S3Filesystem(CloudFilesystemInterface):
+    """Provides access to an AWS S3 bucket."""
+
+    REGEX_S3_PATH = re.compile(r"s3:\/\/(?P<bucket>[\w-]+)\/?(?P<prefix>.*)?")
+
+    def __init__(self, path, profile):
+        match = self.REGEX_S3_PATH.search(str(path))
+        assert match, f"Failed to parse AWS S3 bucket: {path}"
+        self._bucket = match.groupdict()["bucket"]
+        self._relpath = match.groupdict()["prefix"]
+        self._uri = str(path)
+        self._profile = profile
+        self._session = boto3.session.Session(profile_name=self._profile)
+        self._client = self._session.client("s3")
+
+        register_configuration_parameter(S3Path("/"), resource=self._session.resource("s3"))
+
+    @property
+    def profile(self):
+        """Return the AWS profile."""
+        return self._profile
+
+    @property
+    def bucket(self):
+        """Return the AWS bucket."""
+        return self._bucket
+
+    def _Key(self, path):
+        """Get formatted S3 key from provided path for S3Path module"""
+        if not path:
+            path = ""
+        path = str(path)
+        if path.startswith(f"/{self._bucket}"):
+            path = path[len(f"/{self._bucket}") + 1 :]
+        elif path.startswith(self._bucket):
+            path = path[len(self._bucket) + 1 :]
+        elif path.startswith(self._uri):
+            path = path.replace(self._uri + "/", "")
+        elif path.startswith("/"):
+            path = path[1:]
+        return path
+
+    def check_versions(self, directory):
+        assert False, "not supported yet"
+
+    def copy_file(self, src, dst):
+        assert False, "not supported yet"
+
+    def copy_tree(self, src, dst):
+        assert False, "not supported yet"
+
+    def exists(self, path):
+        return self.path(path).exists()
+
+    def listdir(
+        self, directory="", files_only=False, directories_only=False, exclude_hidden=False
+    ):
+        contents = [x for x in self.path(directory).glob("*") if x.name != ""]
+        if exclude_hidden:
+            # NOTE: this does not currently ignore hidden directories in the path.
+            contents = [x for x in contents if not x.name.startswith(".")]
+        if files_only:
+            return [x.name for x in contents if x.is_file()]
+        if directories_only:
+            return [x.name for x in contents if x.is_dir()]
+        return [x.name for x in contents]
+
+    def list_versions(self, path):
+        assert False, "not supported yet"
+        # self._s3.list_object_versions(Bucket=self._bucket, Prefix=prefix)
+
+    def mkdir(self, directory):
+        key = self._Key(directory)
+        self._client.put_object(Bucket=self._bucket, Body="", Key=f"{key}/")
+        return None
+
+    def rglob(
+        self,
+        directory=None,
+        files_only=False,
+        directories_only=False,
+        exclude_hidden=True,
+        pattern="*",
+    ):
+        directory = str(self.path(directory))
+        contents = list(self.path(directory).rglob(pattern))
+        if exclude_hidden:
+            # NOTE: this does not currently ignore hidden directories in the path.
+            contents = [str(x) for x in contents if not x.name.startswith(".")]
+        if files_only:
+            return [str(x) for x in contents if x.is_file()]
+        if directories_only:
+            return [str(x) for x in contents if x.is_dir()]
+        return [str(x) for x in contents]
+
+    def rm_tree(self, directory):
+        assert False, "not supported yet"
+
+    def path(self, path):
+        """Returns S3Path"""
+        return S3Path(f"/{self._bucket}/{self._Key(path)}")
+
+    def touch(self, path):
+        self.path(path).touch()

dsgrid/loggers.py

@@ -0,0 +1,132 @@
+"""Contains logging configuration data."""
+
+import logging
+import logging.config
+import os
+from contextlib import contextmanager
+
+import chronify.loggers
+
+
+# ETH@20210325 - What if you want to set up logging for all loggers, or for all
+# dsgrid loggers? What name should be provided? Should that be the default?
+# Should filename default to None?
+
+
+# ETH@20210325 - name and packages seems like two different functions? That is,
+# you're either setting up logger name, or you want to set up a bunch of loggers
+# for the different packages?
+def setup_logging(
+    name, filename, console_level=logging.INFO, file_level=logging.INFO, packages=None, mode="w"
+):
+    """Configures logging to file and console.
+
+    Parameters
+    ----------
+    name : str
+        logger name
+    filename : str | None
+        log filename
+    console_level : int, optional
+        console log level. defaults to logging.INFO
+    file_level : int, optional
+        file log level. defaults to logging.INFO
+    packages : list, optional
+        enable logging for these package names. Always adds dsgrid.
+    """
+    log_config = {
+        "version": 1,
+        "disable_existing_loggers": False,
+        "formatters": {
+            "basic": {"format": "%(message)s"},
+            "short": {
+                "format": "%(asctime)s - %(levelname)s [%(name)s "
+                "%(filename)s:%(lineno)d] : %(message)s",
+            },
+            "detailed": {
+                "format": "%(asctime)s - %(levelname)s [%(name)s "
+                "%(filename)s:%(lineno)d] : %(message)s",
+            },
+        },
+        "handlers": {
+            "console": {
+                "level": console_level,
+                "formatter": "short",
+                "class": "logging.StreamHandler",
+            },
+            "file": {
+                "class": "logging.FileHandler",
+                "level": file_level,
+                "filename": filename,
+                "mode": mode,
+                "formatter": "detailed",
+            },
+        },
+        "loggers": {
+            name: {"handlers": ["console", "file"], "level": "DEBUG", "propagate": True},
+        },
+    }
+
+    packages = packages or []
+    packages = set(packages)
+    packages.add("dsgrid")
+    for package in packages:
+        log_config["loggers"][package] = {
+            "handlers": ["console"],
+            "level": "DEBUG",
+            "propagate": True,
+        }
+        if filename is not None:
+            log_config["loggers"][package]["handlers"].append("file")
+
+    # ETH@20210325 - This logic should be applied to packages as well? This makes
+    # me think that this should really be two functions--one for setting up a
+    # logger by name and the other for setting up loggers for a list of packages.
+    # DT: I think the issue is fixed, but we can still consider your point.
+    if filename is None:
+        log_config["handlers"].pop("file")
+
+    logging.config.dictConfig(log_config)
+    logger = logging.getLogger(name)
+
+    # TODO: more consideration is warranted, but this is usually what we want.
+    # If we migrate dsgrid to use loguru, it will be easier. We could use the TRACE level
+    # in dsgrid.
+    chronify.loggers.setup_logging(
+        console_level="WARNING",
+        file_level="DEBUG",
+        filename=filename,
+        mode=mode,
+    )
+    return logger
+
+
+def check_log_file_size(filename, limit_mb=10, no_prompts=False):
+    if not filename.exists():
+        return
+
+    size_mb = filename.stat().st_size / (1024 * 1024)
+    if size_mb > limit_mb and not no_prompts:
+        msg = f"The log file {filename} has exceeded {limit_mb} MiB. Delete it? [Y] >>> "
+        val = input(msg)
+        if val == "" or val.lower() == "y":
+            os.remove(filename)
+
+
+@contextmanager
+def disable_console_logging(name="dsgrid"):
+    logger = logging.getLogger(name)
+    console_level = None
+    try:
+        for handler in logger.handlers:
+            if handler.name == "console":
+                console_level = handler.level
+                handler.setLevel(logging.FATAL)
+                break
+        yield
+    finally:
+        for handler in logger.handlers:
+            if handler.name == "console":
+                assert console_level is not None
+                handler.setLevel(console_level)
+                break