bluer-objects 6.3.1__py3-none-any.whl

bluer_objects/env.py

@@ -0,0 +1,72 @@
+from typing import Union
+import os
+
+from blue_options.env import load_config, load_env, get_env
+
+load_env(__name__)
+load_config(__name__)
+
+HOME = get_env("HOME")
+
+ABCLI_AWS_REGION = get_env("ABCLI_AWS_REGION")
+
+ABCLI_AWS_S3_BUCKET_NAME = get_env(
+    "ABCLI_AWS_S3_BUCKET_NAME",
+    "kamangir",
+)
+
+ABCLI_AWS_S3_PREFIX = get_env(
+    "ABCLI_AWS_S3_PREFIX",
+    "bolt",
+)
+
+ABCLI_AWS_S3_PUBLIC_BUCKET_NAME = get_env("ABCLI_AWS_S3_PUBLIC_BUCKET_NAME")
+
+
+abcli_object_path = get_env("abcli_object_path")
+
+ABCLI_PATH_STORAGE = get_env(
+    "ABCLI_PATH_STORAGE",
+    os.path.join(HOME, "storage"),
+)
+
+abcli_object_name = get_env("abcli_object_name")
+
+ABCLI_S3_OBJECT_PREFIX = get_env(
+    "ABCLI_S3_OBJECT_PREFIX",
+    f"s3://{ABCLI_AWS_S3_BUCKET_NAME}/{ABCLI_AWS_S3_PREFIX}",
+)
+
+
+ABCLI_OBJECT_ROOT = get_env(
+    "ABCLI_OBJECT_ROOT",
+    os.path.join(ABCLI_PATH_STORAGE, "abcli"),
+)
+
+abcli_path_git = get_env(
+    "abcli_path_git",
+    os.path.join(HOME, "git"),
+)
+
+ABCLI_PATH_STATIC = get_env("ABCLI_PATH_STATIC")
+
+ABCLI_PUBLIC_PREFIX = get_env("ABCLI_PUBLIC_PREFIX")
+
+VANWATCH_TEST_OBJECT = get_env("VANWATCH_TEST_OBJECT")
+
+# https://www.randomtextgenerator.com/
+DUMMY_TEXT = "This is some dummy text. This is some dummy text. This is some dummy text. This is some dummy text. This is some dummy text. This is some dummy text. This is some dummy text. This is some dummy text. This is some dummy text. This is some dummy text."
+
+ABCLI_AWS_RDS_DB = get_env("ABCLI_AWS_RDS_DB")
+ABCLI_AWS_RDS_PORT = get_env("ABCLI_AWS_RDS_PORT")
+ABCLI_AWS_RDS_USER = get_env("ABCLI_AWS_RDS_USER")
+
+ABCLI_AWS_RDS_HOST = get_env("ABCLI_AWS_RDS_HOST")
+ABCLI_AWS_RDS_PASSWORD = get_env("ABCLI_AWS_RDS_PASSWORD")
+
+DATABRICKS_WORKSPACE = get_env("DATABRICKS_WORKSPACE")
+
+DATABRICKS_HOST = get_env("DATABRICKS_HOST")
+DATABRICKS_TOKEN = get_env("DATABRICKS_TOKEN")
+
+ABCLI_MLFLOW_EXPERIMENT_PREFIX = get_env("ABCLI_MLFLOW_EXPERIMENT_PREFIX")

bluer_objects/file/__init__.py

@@ -0,0 +1,41 @@
+from bluer_objects.file.classes import JsonEncoder, as_json
+from bluer_objects.file.functions import (
+    absolute,
+    add_extension,
+    add_prefix,
+    add_suffix,
+    auxiliary,
+    copy,
+    delete,
+    download,
+    exists,
+    extension,
+    list_of,
+    move,
+    name_and_extension,
+    name,
+    path,
+    relative,
+    size,
+)
+from bluer_objects.file.load import (
+    load_dataframe,
+    load_image,
+    load_json,
+    load_matrix,
+    load_text,
+    load_xml,
+    load_yaml,
+    load,
+)
+from bluer_objects.file.save import (
+    prepare_for_saving,
+    save_csv,
+    save_fig,
+    save_image,
+    save_json,
+    save_matrix,
+    save_text,
+    save_yaml,
+    save,
+)

bluer_objects/file/__main__.py

@@ -0,0 +1,51 @@
+import argparse
+from tqdm import tqdm
+
+from blueness import module
+from blueness.argparse.generic import sys_exit
+from blue_options import string
+
+from bluer_objects import file, NAME
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+parser = argparse.ArgumentParser(NAME)
+parser.add_argument(
+    "task",
+    type=str,
+    help="replace|size",
+)
+parser.add_argument(
+    "--filename",
+    type=str,
+)
+parser.add_argument(
+    "--this",
+    type=str,
+    help="<this-1+this-2+this-3>",
+)
+parser.add_argument(
+    "--that",
+    type=str,
+    help="<that-1+that-2+that-3>",
+)
+args = parser.parse_args()
+
+success = False
+if args.task == "replace":
+    logger.info(f"{NAME}.{args.task}: {args.this} -> {args.that} in {args.filename}")
+
+    success, content = file.load_text(args.filename)
+    if success:
+        for this, that in tqdm(zip(args.this.split("+"), args.that.split("+"))):
+            content = [line.replace(this, that) for line in content]
+
+        success = file.save_text(args.filename, content)
+elif args.task == "size":
+    print(string.pretty_bytes(file.size(args.filename)))
+    success = True
+else:
+    success = None
+
+sys_exit(logger, NAME, args.task, success)

bluer_objects/file/classes.py

@@ -0,0 +1,38 @@
+from typing import Any
+import datetime
+import json
+
+
+# https://gist.github.com/jsbueno/9b2ea63fb16b84658281ec29b375283e
+class JsonEncoder(json.JSONEncoder):
+    def default(self, obj):
+        try:
+            return super().default(obj)
+        except TypeError:
+            pass
+
+        if obj.__class__.__name__ == "ndarray":
+            return obj.tolist()
+
+        if isinstance(obj, datetime.datetime):
+            return "{}".format(obj)
+
+        return (
+            obj.__dict__
+            if not hasattr(type(obj), "__json_encode__")
+            else obj.__json_encode__
+        )
+
+
+def as_json(
+    thing: Any,
+) -> str:
+    # https://docs.python.org/2/library/json.html
+    return json.dumps(
+        thing,
+        sort_keys=True,
+        cls=JsonEncoder,
+        indent=0,
+        ensure_ascii=False,
+        separators=(",", ":"),
+    ).replace("\n", "")

bluer_objects/file/functions.py

@@ -0,0 +1,290 @@
+import urllib3
+from functools import reduce
+import fnmatch
+from typing import Any, List, Union
+import os
+import shutil
+
+from blueness import module
+from blue_options import string
+from blue_options.logger import crash_report
+
+from bluer_objects import NAME
+from bluer_objects.env import abcli_object_path
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+def absolute(
+    filename: str,
+    reference_path: Any = None,
+) -> str:
+    from bluer_objects.path import absolute as path_absolute
+
+    return os.path.join(
+        path_absolute(
+            path(filename),
+            os.getcwd() if reference_path is None else reference_path,
+        ),
+        name_and_extension(filename),
+    )
+
+
+def add_extension(
+    filename: str,
+    extension_: Any,
+    force: bool = True,
+):
+    if not isinstance(extension_, str):
+        extension_ = extension(extension_)
+
+    filename, extension_as_is = os.path.splitext(filename)
+    if extension_as_is != "":
+        extension_as_is = extension_as_is[1:]
+
+    if not force and extension_as_is == "":
+        extension_ = extension_as_is
+
+    return f"{filename}.{extension_}"
+
+
+def add_prefix(
+    filename: str,
+    prefix: str,
+) -> str:
+    pathname, filename = os.path.split(filename)
+    return os.path.join(pathname, f"{prefix}-{filename}")
+
+
+def add_suffix(
+    filename: str,
+    suffix: str,
+) -> str:
+    filename, extension = os.path.splitext(filename)
+    return f"{filename}-{suffix}{extension}"
+
+
+def auxiliary(
+    nickname: str,
+    extension: str,
+    add_timestamp: bool = True,
+) -> str:
+    filename = os.path.join(
+        abcli_object_path,
+        "auxiliary",
+        "-".join(
+            [nickname]
+            + (
+                [
+                    string.pretty_date(
+                        as_filename=True,
+                        squeeze=True,
+                        unique=True,
+                    )
+                ]
+                if add_timestamp
+                else []
+            )
+        )
+        + f".{extension}",
+    )
+
+    os.makedirs(path(filename), exist_ok=True)
+
+    return filename
+
+
+def copy(
+    source: str,
+    destination: str,
+    log: bool = True,
+    overwrite: bool = True,
+) -> bool:
+    if not overwrite and exists(destination):
+        if log:
+            logger.info(f"✅ {destination}")
+        return True
+
+    try:
+        os.makedirs(path(destination), exist_ok=True)
+
+        # https://stackoverflow.com/a/8858026
+        # better choice: copy2
+        shutil.copyfile(source, destination)
+    except:
+        crash_report(f"{NAME}: copy({source},{destination}): failed.")
+        return False
+
+    if log:
+        logger.info(f"{NAME}: {source} -> {destination}")
+
+    return True
+
+
+def delete(
+    filename: str,
+) -> bool:
+    if not os.path.isfile(filename):
+        return True
+
+    try:
+        os.remove(filename)
+
+        return True
+    except:
+        crash_report(f"{NAME}: delete({filename}): failed.")
+        return False
+
+
+def download(
+    url: str,
+    filename: str,
+    log: bool = True,
+    overwrite: bool = True,
+) -> bool:
+    if not overwrite and exists(filename):
+        if log:
+            logger.info(f"✅ {filename}")
+
+        return True
+
+    try:
+        # https://stackoverflow.com/a/27406501
+        with urllib3.PoolManager().request(
+            "GET", url, preload_content=False
+        ) as response, open(filename, "wb") as fp:
+            shutil.copyfileobj(response, fp)
+
+        response.release_conn()  # not 100% sure this is required though
+
+    except:
+        crash_report(f"{NAME}: download({url},{filename}): failed.")
+        return False
+
+    if log:
+        logger.info(f"{NAME}: {url} -> {filename}")
+
+    return True
+
+
+def exists(
+    filename: str,
+) -> bool:
+    return os.path.isfile(filename)
+
+
+def extension(
+    filename: Any,
+) -> str:
+    if isinstance(filename, str):
+        _, extension = os.path.splitext(filename)
+        if extension != "":
+            if extension[0] == ".":
+                extension = extension[1:]
+        return extension
+
+    if isinstance(filename, type):
+        return "py" + filename.__name__.lower()
+
+    return "py" + filename.__class__.__name__.lower()
+
+
+def list_of(
+    template: str,
+    recursive: bool = False,
+) -> List[str]:
+    from bluer_objects import path as path_module
+
+    if isinstance(template, list):
+        return reduce(
+            lambda x, y: x + y,
+            [list_of(template_, recursive) for template_ in template],
+            [],
+        )
+
+    if recursive:
+        return reduce(
+            lambda x, y: x + y,
+            [
+                list_of(
+                    os.path.join(pathname, name_and_extension(template)),
+                    recursive,
+                )
+                for pathname in path_module.list_of(path(template))
+            ],
+            list_of(template),
+        )
+
+    # https://stackoverflow.com/a/40566802
+    template_path = path(template)
+    if template_path == "":
+        template_path = path_module.current()
+
+    try:
+        return [
+            os.path.join(template_path, filename)
+            for filename in fnmatch.filter(
+                os.listdir(template_path),
+                name_and_extension(template),
+            )
+        ]
+    except:
+        return []
+
+
+def move(
+    source: str,
+    destination: str,
+) -> bool:
+    try:
+        os.makedirs(path(destination), exist_ok=True)
+
+        # https://stackoverflow.com/a/8858026
+        shutil.move(source, destination)
+    except:
+        crash_report(f"{NAME}: move({source},{destination}): failed.")
+        return False
+
+    return True
+
+
+def name(
+    filename: str,
+) -> str:
+    _, filename = os.path.split(filename)
+
+    return filename if "." not in filename else ".".join(filename.split(".")[:-1])
+
+
+def name_and_extension(
+    filename: str,
+) -> str:
+    return os.path.basename(filename)
+
+
+def path(
+    filename: str,
+) -> str:
+    return os.path.split(filename)[0]
+
+
+def relative(
+    filename: str,
+    reference_path: Union[None, str] = None,
+):
+    from bluer_objects.path import relative as path_relative
+
+    return path_relative(
+        path(filename),
+        os.getcwd() if reference_path is None else reference_path,
+    ) + name_and_extension(filename)
+
+
+def size(
+    filename: str,
+) -> int:
+    try:
+        return os.path.getsize(filename)
+    except:
+        return 0

bluer_objects/file/load.py

@@ -0,0 +1,219 @@
+from typing import Tuple, Any, List, Any
+from copy import deepcopy
+import json
+import numpy as np
+
+from blueness import module
+from blue_options import string
+from blue_options.logger import crash_report
+
+from bluer_objects import NAME
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+def load(
+    filename,
+    ignore_error=False,
+    default={},
+) -> Tuple[bool, Any]:
+    # https://wiki.python.org/moin/UsingPickle
+    data = deepcopy(default)
+
+    try:
+        import dill
+
+        with open(filename, "rb") as fp:
+            data = dill.load(fp)
+
+        return True, data
+    except:
+        if not ignore_error:
+            crash_report(f"{NAME}: load({filename}): failed.")
+
+        return False, data
+
+
+def load_dataframe(
+    filename,
+    ignore_error=False,
+    log=False,
+) -> Tuple[bool, Any]:
+    success = False
+    df = None
+
+    try:
+        import pandas
+
+        df = pandas.read_csv(filename)
+
+        success = True
+    except:
+        if not ignore_error:
+            crash_report(f"{NAME}: load_dataframe({filename}): failed.")
+
+    if success and log:
+        logger.info(
+            "loaded {} row(s) of {} from {}".format(
+                len(df),
+                ", ".join(df.columns),
+                filename,
+            )
+        )
+
+    return success, df
+
+
+def load_image(
+    filename,
+    ignore_error=False,
+    log=False,
+) -> Tuple[bool, np.ndarray]:
+    import cv2
+
+    success = True
+    image = np.empty((0,))
+
+    try:
+        image = cv2.imread(filename)
+
+        if len(image.shape) == 3:
+            if image.shape[2] == 4:
+                image = image[:, :, :3]
+
+            image = np.flip(image, axis=2)
+
+    except:
+        if not ignore_error:
+            crash_report(f"{NAME}: load_image({filename}): failed.")
+        success = False
+
+    if success and log:
+        logger.info(
+            "loaded {} from {}".format(
+                string.pretty_shape_of_matrix(image),
+                filename,
+            )
+        )
+
+    return success, image
+
+
+def load_json(
+    filename,
+    ignore_error=False,
+    default={},
+) -> Tuple[bool, Any]:
+    success = False
+    data = default
+
+    try:
+        with open(filename, "r") as fh:
+            data = json.load(fh)
+
+        success = True
+    except:
+        if not ignore_error:
+            crash_report(f"{NAME}: load_json({filename}): failed.")
+
+    return success, data
+
+
+def load_matrix(
+    filename: str,
+    ignore_error=False,
+    log: bool = False,
+) -> Tuple[bool, np.ndarray]:
+    success = True
+    matrix: np.ndarray = np.empty((0,))
+
+    try:
+        matrix = np.load(filename)
+    except:
+        if not ignore_error:
+            crash_report(f"{NAME}: load_matrix({filename}) failed.")
+        success = False
+
+    if success and log:
+        logger.info(
+            "loaded {} from {}".format(
+                string.pretty_shape_of_matrix(matrix),
+                filename,
+            )
+        )
+
+    return success, matrix
+
+
+def load_text(
+    filename,
+    ignore_error=False,
+    count=-1,
+    log=False,
+) -> Tuple[bool, List[str]]:
+    success = True
+    text = []
+
+    try:
+        if count == -1:
+            with open(filename, "r") as fp:
+                text = fp.read()
+            text = text.split("\n")
+        else:
+            # https://stackoverflow.com/a/1767589/10917551
+            with open(filename) as fp:
+                text = [next(fp) for _ in range(count)]
+    except:
+        success = False
+        if not ignore_error:
+            crash_report(f"{NAME}: load_text({filename}): failed.")
+
+    if success and log:
+        logger.info("loaded {} line(s) from {}.".format(len(text), filename))
+
+    return success, text
+
+
+def load_xml(
+    filename,
+    ignore_error=False,
+    default={},
+) -> Tuple[bool, Any]:
+    success = False
+    data = default
+
+    try:
+        import xml.etree.ElementTree as ET
+
+        tree = ET.parse(filename)
+        data = tree.getroot()
+
+        success = True
+    except:
+        if not ignore_error:
+            crash_report(f"{NAME}: load_xml({filename}): failed.")
+
+    return success, data
+
+
+def load_yaml(
+    filename,
+    ignore_error=False,
+    default={},
+) -> Tuple[bool, Any]:
+    success = False
+    data = default
+
+    try:
+        import yaml
+
+        with open(filename, "r") as fh:
+            data = yaml.safe_load(fh)
+
+        success = True
+    except:
+        if not ignore_error:
+            crash_report(f"{NAME}: load_yaml({filename}): failed.")
+
+    return success, data