swak 0.0.3__py3-none-any.whl

swak/cli/__init__.py

@@ -0,0 +1,14 @@
+"""Tools to assist in consolidating sources of project configurations."""
+
+from .importer import Importer
+from .envparser import EnvParser
+from .argparser import ArgParser, USAGE, DESCRIPTION, EPILOG
+
+__all__ = [
+    'Importer',
+    'EnvParser',
+    'ArgParser',
+    'USAGE',
+    'DESCRIPTION',
+    'EPILOG'
+]

swak/cli/argparser.py

@@ -0,0 +1,159 @@
+import sys
+import json
+from json import JSONDecodeError
+from ast import literal_eval
+from argparse import ArgumentParser, RawTextHelpFormatter
+from typing import Any
+from itertools import takewhile, dropwhile, chain
+from .exceptions import ArgParseError
+
+type Parsed = tuple[list[str], dict[str, Any]]
+
+USAGE = '%(prog)s [action(s)] [-h]'
+
+DESCRIPTION = 'Refer to the README.md for the available actions!'
+
+EPILOG = """
+Additionally, all fields of the program's config can be set via
+long-format options. Nested fields can be set by dot-separating
+levels, e.g., "--root.level1.level2 value"
+
+{!r}
+"""
+
+
+class ArgParser:
+    """Parse the command line for actions and any long-format options.
+
+    Using this command-line argument parser alleviates the need for
+    defining any groups or options beforehand. Arguments immediately
+    following the program call are interpreted as actions to perform as long
+    as they do not start with a hyphen. Starting with the first argument
+    that starts with a hyphen, command-line arguments will be interpreted as
+    ``--key value`` pairs and this long format is the only one allowed.
+    Abbreviated options (``-k value``) right after actions (and before any
+    long-format options) are ignored.
+
+    Parameters
+    ----------
+    default_action: str, optional
+        Default action to return if none is found in the command-line
+        arguments. Defaults to no action.
+    usage: str, optional
+        Program usage message.
+    description: str, optional
+        Program description.
+    epilog: str, optional
+        Text displayed after the help on command-line options.
+    fmt_cls: type, optional
+        Option passed on to the underlying ``argparse.ArgumentParser``.
+        Defaults to ``argparse.RawTextHelpFormatter``
+
+    """
+
+    def __init__(
+            self,
+            default_action: str | None = None,
+            usage: str = USAGE,
+            description: str = DESCRIPTION,
+            epilog: str = EPILOG,
+            fmt_cls: type = RawTextHelpFormatter
+    ) -> None:
+        self.default_action = default_action
+        self.usage = usage
+        self.description = description
+        self.epilog = epilog
+        self.fmt_cls = fmt_cls
+        self.__parse = ArgumentParser(
+            usage=usage,
+            description=description,
+            epilog=epilog,
+            formatter_class=fmt_cls
+        ).parse_known_args
+
+    def __repr__(self) -> str:
+        cls = self.__class__.__name__
+        return f'{cls}({self.default_action}, ...)'
+
+    def __call__(self, args: list[str] | None = None) -> Parsed:
+        """Parse the command-line arguments into actions and options.
+
+        Parameters
+        ----------
+        args: list of str, optional
+            The command-line arguments to parse. Mainly a debugging feature.
+            If none is given, ``sys.argv[1:]`` will be parsed.
+
+        Returns
+        -------
+        actions: list of str
+            A list with the actions (as strings) to perform. If none are
+            found on the command line and no `default_action` is specified,
+            that list will be empty.
+        options: dict
+            Dictionary with keys and values parsed from long-format
+            command line arguments.
+
+        """
+        args = sys.argv[1:] if args is None else args
+        _, unknowns = self.__parse(args)
+        actions, args = self.__split(unknowns)
+        actions = self.__valid(actions)
+        args = self.__compatible(args)
+        options = self.__mapped(args)
+        return actions, options
+
+    def __split(self, args: list[str]) -> tuple[list[str], list[str]]:
+        """Split command-line options into actions and config settings."""
+        # Everything before the first option (starting with "-") are actions.
+        actions = tuple(takewhile(lambda arg: not arg.startswith('-'), args))
+        # If there are none, fall back onto the default action (if specified).
+        if not actions and self.default_action is not None:
+            actions = self.default_action,
+        # Either way, replace dashes with underscores
+        actions = (action.lower().replace('-', '_') for action in actions)
+
+        # Everything after the action(s) are options (starting with "--").
+        args = dropwhile(lambda arg: not arg.startswith('--'), args)
+        # In case options are given as "--key=value", we split.
+        args = chain.from_iterable(arg.split('=') for arg in args)
+        return list(actions), list(args)
+
+    @staticmethod
+    def __valid(actions: list[str]) -> list[str]:
+        """Raise if any action string is not a valid python identifier."""
+        for action in actions:
+            if not action.isidentifier():
+                msg = 'Actions must be valid identifiers, unlike "{}"!'
+                raise ArgParseError(msg.format(action))
+        return actions
+
+    @staticmethod
+    def __compatible(args: list[str]) -> list[str]:
+        """Raise if a command-line option is not in long format with value."""
+        long_form = all(arg.startswith('--') for arg in args[::2])
+        alternating = all(not arg.startswith('-') for arg in args[1::2])
+        even_number = len(args) % 2 == 0
+        if not (long_form and alternating and even_number):
+            msg = ('Command-line arguments must be passed in long format (i.e.'
+                   ', as "--key value"), and a value must always be present!')
+            raise ArgParseError(msg)
+        return args
+
+    def __mapped(self, args: list[str]) -> dict[str, Any]:
+        """Create dictionary wih config keys and (string) values from args."""
+        zipped = list(zip(args[::2], args[1::2]))
+        # Drop the first two characters (always "--") and replace "-" with "_".
+        return {k[2:].replace('-', '_'): self.__parsed(v) for k, v in zipped}
+
+    @staticmethod
+    def __parsed(value: str) -> Any:
+        """Try to parse (string) command-line options into python objects."""
+        try:
+            parsed = json.loads(value)
+        except (TypeError, JSONDecodeError):
+            try:
+                parsed = literal_eval(value)
+            except (TypeError, ValueError, SyntaxError):
+                parsed = value
+        return parsed

swak/cli/envparser.py

@@ -0,0 +1,67 @@
+import os
+import json
+from json import JSONDecodeError
+from ast import literal_eval
+from typing import Any
+from ..misc import ArgRepr
+
+
+class EnvParser(ArgRepr):
+    """Parse OS environment variables, preferring prefixed over pure versions.
+
+    Sometimes, environment variables desired for individual use are already
+    taken by the operating system or some other system component. In these
+    cases, one can resort to prefixing these to avoid conflicts. The present
+    class is instantiated with that prefix and will resolve conflicts when
+    objects are called, returning the OS environment as a dictionary with
+    the values of all variables parsed into python literals.
+
+    Parameters
+    ----------
+    prefix: str, optional
+        Prefix of environment variables that would otherwise be shadowed
+        by existing ones. Defaults to empty string.
+
+    """
+
+    def __init__(self, prefix: str = '') -> None:
+        super().__init__(prefix)
+        self.prefix = prefix
+
+    def __call__(self, env: dict[str, str] | None = None) -> dict[str, Any]:
+        """Parse the OS environment, resolving potentially prefixed variables.
+
+        Parameters
+        ----------
+        env: dict, optional
+            Dictionary to be parsed and resolved. Defaults to ``os.environ``.
+
+        Returns
+        -------
+        dict
+            Environment with prefixed keys removed and the values of their
+            non-prefixed counterparts updated accordingly.
+
+        """
+        env = os.environ if env is None else env
+        prefixed = {}
+        original = {}
+        for key in env:
+            if key.startswith(self.prefix):
+                prefixed[key.removeprefix(self.prefix)] = env[key]
+            else:
+                original[key] = env[key]
+        merged = original | prefixed
+        return {key: self.__parsed(value) for key, value in merged.items()}
+
+    @staticmethod
+    def __parsed(value: str) -> Any:
+        """Try to parse (string) environment variables into python objects."""
+        try:
+            parsed = json.loads(value)
+        except (TypeError, JSONDecodeError):
+            try:
+                parsed = literal_eval(value)
+            except (TypeError, ValueError, SyntaxError):
+                parsed = value
+        return parsed

swak/cli/exceptions.py

@@ -0,0 +1,6 @@
+class ArgParseError(Exception):
+    pass
+
+
+class ImporterError(Exception):
+    pass

swak/cli/importer.py

@@ -0,0 +1,68 @@
+from importlib import import_module
+from ..misc import ArgRepr
+from .exceptions import ImporterError
+
+
+class Importer(ArgRepr):
+    """Programmatically import objects from a module under a top-level package.
+
+    For ease of use and clarity in API, relative imports are not supported.
+    Objects are instantiated with where to import from and called with what
+    to import.
+
+    Parameters
+    ----------
+    package: str
+        Name of the top-level package to import from. Must not start with dots
+        but can contain any number of dots to indicate sub-packages.
+    module: str, optional
+        The specific module to import objects from. May contain dots to
+        indicate that it is located further down within some sub-package.
+        Defaults to "steps".
+
+    """
+
+    def __init__(self, package: str, module: str = 'steps') -> None:
+        self.package = package.strip(' ./')
+        self.module = module.strip(' ./')
+        super().__init__(self.package, self.module)
+
+    @property
+    def path(self) -> str:
+        """Full path specification of (sub-)package and module concatenated."""
+        return '.'.join([self.package, self.module])
+
+    def __call__(self, *names: str) -> list:
+        """Import any number of objects from the specified package.module.
+
+        Parameters
+        ----------
+        *names: str
+            Name(s) of object(s) to import from ``package.module``.
+
+        Returns
+        -------
+        list
+            Imported objects.
+
+        Raises
+        ------
+        ImporterError
+            When the ``package.module`` is mis-specified, can't be found, or
+            when the specified object(s) can't be found in it.
+
+        """
+        try:
+            location = import_module(self.path)
+        except (TypeError, ModuleNotFoundError) as error:
+            msg = 'Could not import module "{}"!'
+            raise ImporterError(msg.format(self.path)) from error
+        imports = []
+        for name in names:
+            try:
+                imported = getattr(location, name)
+            except AttributeError as err:
+                msg = 'Could not import "{}" from module "{}"!'
+                raise ImporterError(msg.format(name, self.path)) from err
+            imports.append(imported)
+        return imports

swak/cloud/__init__.py

@@ -0,0 +1,14 @@
+"""Tools to interact with hosted cloud services.
+
+Current only supports elements of the Google Cloud Project (GCP) but, in
+the future, Amazon Web Services (AWS) might get implemented as well.
+
+"""
+
+from importlib.util import find_spec
+
+required = 'google_cloud_bigquery', 'google_cloud_storage', 'pandas_gbq'
+
+if any(find_spec(package) for package in required) is None:
+    msg = 'Install {} with the [cloud] extra to unlock this subpackage!'
+    raise ImportError(msg.format(__package__.split('.')[0]))

swak/cloud/gcp/__init__.py

@@ -0,0 +1,31 @@
+"""Tools to interact with elements of the Google Cloud Project (GCP).
+
+Specifically, data scientists tend to interact mostly with Google's BigQuery
+(BQ) data-warehouse solution and the Google Cloud Storage (GCS).
+
+"""
+
+from .dataset import Collation, Rounding, Billing, GbqDataset
+from .bucket import Storage, GcsBucket
+from .query import GbqQuery
+from .query2gcs import GbqQuery2GcsParquet
+from .gcs2local import GcsDir2LocalDir
+from .gcs2df import GcsParquet2DataFrame
+from .query2df import GbqQuery2DataFrame
+from .df2gbq import IfExists, DataFrame2Gbq
+
+__all__ = [
+    'Collation',
+    'Rounding',
+    'Billing',
+    'GbqDataset',
+    'Storage',
+    'GcsBucket',
+    'GbqQuery',
+    'GbqQuery2GcsParquet',
+    'GcsDir2LocalDir',
+    'GcsParquet2DataFrame',
+    'GbqQuery2DataFrame',
+    'IfExists',
+    'DataFrame2Gbq'
+]

swak/cloud/gcp/bucket.py

@@ -0,0 +1,161 @@
+from enum import StrEnum
+from typing import Any, Literal
+from google.cloud.storage import Client, Bucket
+from google.api_core.retry import Retry
+from ...misc import ArgRepr
+
+type StorageType = Literal['STANDARD', 'NEARLINE', 'COLDLINE', 'ARCHIVE']
+
+
+class Storage(StrEnum):
+    """Specify storage class for blobs in a Google Cloud Storage bucket."""
+    STANDARD = 'STANDARD'
+    NEARLINE = 'NEARLINE'
+    COLDLINE = 'COLDLINE'
+    ARCHIVE = 'ARCHIVE'
+
+
+class GcsBucket(ArgRepr):
+    """Create a new bucket on Google Cloud Storage.
+
+    Parameters
+    ----------
+    project: str
+        The project to create the bucket in.
+    bucket: str
+        The name of the bucket to create.
+    location: str
+        The physical datacenter location to create the bucket in. See the
+        Google Cloud Platform `documentation <https://cloud.google.com/storage/
+        docs/locations>`__ for options.
+    blob_expire_days: int, optional
+        Defaults to ``None``. If sets, blobs older than the specified number of
+        days will be automatically deleted.
+    labels: dict, optional
+        Any number of string-valued labels of the bucket. Defaults to none.
+    user_project: str, optional
+        The project billed for interacting with the bucket. Defaults to the
+        `project`
+    storage_class: str, optional
+        Defaults storage class for blobs in this bucket. Defaults to ``None``,
+        which results in "STANDARD".  Use the ``Storage`` enum to specify
+        explicitly.
+    requester_pays: bool, optional
+        Whether the requester will be billed for interacting with the bucket.
+        Defaults to ``False``, which means that the (`user_`)`project` will be
+        billed.
+    **kwargs
+        Additional keyword arguments are passed to the constructor of the
+        Google Storage ``Client`` (see `documentation <https://cloud.google.
+        com/python/docs/reference/storage/latest/google.cloud.storage.
+        client.Client#parameters>`__ for options).
+
+    Notes
+    -----
+    There are a lot more options to set, which have been deliberately omitted
+    because of the complexity involved.
+
+    See Also
+    --------
+    Storage
+
+    """
+
+    def __init__(
+            self,
+            project: str,
+            bucket: str,
+            location: str,
+            blob_expire_days: int | None = None,
+            labels: dict[str, str] | None = None,
+            user_project: str | None = None,
+            storage_class: StorageType | None = None,
+            requester_pays: bool = False,
+            **kwargs: Any
+    ) -> None:
+        self.project = project.strip(' /.')
+        self.bucket = bucket.strip(' /.')
+        self.location = location.strip().upper()
+        self.blob_expire_days = blob_expire_days
+        self.labels = {} if labels is None else labels
+        if user_project is None:
+            self.user_project = self.project
+        else:
+            self.user_project = user_project.strip(' /.')
+        self.storage_class = storage_class
+        self.requester_pays = requester_pays
+        self.kwargs = kwargs
+        super().__init__(
+            self.project,
+            self.bucket,
+            self.location,
+            blob_expire_days=self.blob_expire_days,
+            labels=self.labels,
+            user_project=self.user_project,
+            storage_class=self.storage_class,
+            requester_pays=self.requester_pays,
+            **kwargs
+        )
+
+    def __call__(
+            self,
+            exists_ok: bool = True,
+            retry: Retry | None = None,
+            timeout: float | tuple[float, float] | None = None,
+    ) -> tuple[Bucket, bool]:
+        """Create a new bucket on Google Cloud Storage.
+
+        Parameters
+        ----------
+        exists_ok: bool, optional
+            Whether to raise a ``Conflict`` exception if the targeted bucket
+            already exists or not. Defaults to ``True``.
+        retry: Retry, optional
+            Retry policy for the request. Defaults to ``None``, which disables
+            retries. See the Google Cloud Platform `guide
+            <https://cloud.google.com/python/docs/reference/storage/1.39.0/
+            retry_timeout#configuring-retries>`__ and `reference
+            <https://googleapis.dev/python/google-api-core/latest/retry.html>`__
+            for options.
+        timeout: float, optional
+            The number of seconds to wait for the HTTP response to the API call
+            before using `retry` or a tuple with separate values for connection
+            and request timeouts. Defaults to ``None``, meaning wait forever.
+
+        Raises
+        ------
+        Conflict
+            If `exists_ok` is set to ``False`` and the dataset already exists.
+
+        Returns
+        -------
+        Bucket
+            The existing or newly created bucket. If existing, then the bucket
+            is returned unchanged, that is, none of the specified options are
+            applied.
+        bool
+            ``True`` if the requested bucket is newly created and ``False``
+            if an existing bucket is returned.
+
+        """
+        client = Client(self.project, **self.kwargs)
+        bucket = Bucket(client, self.bucket, self.user_project)
+
+        bucket.requester_pays = self.requester_pays
+        bucket.storage_class = self.storage_class
+        bucket.labels = self.labels
+        if self.blob_expire_days:
+            bucket.add_lifecycle_delete_rule(age=self.blob_expire_days)
+
+        if bucket.exists() and exists_ok:
+            existing = client.get_bucket(bucket, retry=retry, timeout=timeout)
+            return existing, False
+
+        bucket.create(
+            client,
+            self.project,
+            self.location,
+            retry=retry,
+            timeout=timeout
+        )
+        return bucket, True