bluer-objects 6.3.1__py3-none-any.whl

bluer_objects/storage/__main__.py

@@ -0,0 +1,114 @@
+import argparse
+
+from blueness import module
+from blueness.argparse.generic import sys_exit
+
+from bluer_objects import NAME
+from bluer_objects.env import ABCLI_AWS_S3_BUCKET_NAME
+from bluer_objects.storage.classes import Storage
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+parser = argparse.ArgumentParser(NAME)
+parser.add_argument(
+    "task",
+    type=str,
+    default="",
+    help="create_bucket|download_file|exists|list_of_objects|upload_file",
+)
+parser.add_argument(
+    "--filename",
+    type=str,
+    default="",
+)
+parser.add_argument(
+    "--bucket_name",
+    type=str,
+    default=ABCLI_AWS_S3_BUCKET_NAME,
+)
+parser.add_argument(
+    "--count",
+    type=int,
+    default=9999,
+)
+parser.add_argument(
+    "--delim",
+    type=str,
+    default=", ",
+)
+parser.add_argument(
+    "--item_name",
+    default="object",
+    type=str,
+)
+parser.add_argument(
+    "--log",
+    default=1,
+    type=int,
+    help="0|1",
+)
+parser.add_argument(
+    "--object_name",
+    type=str,
+    default="",
+)
+parser.add_argument(
+    "--prefix",
+    type=str,
+    default="",
+)
+parser.add_argument(
+    "--recursive",
+    type=int,
+    default=1,
+    help="0|1",
+)
+parser.add_argument(
+    "--suffix",
+    type=str,
+    default="",
+)
+
+args = parser.parse_args()
+
+delim = " " if args.delim == "space" else args.delim
+
+success = True
+output = None
+if args.task == "create_bucket":
+    success = Storage(args.bucket_name).s3 is not None
+elif args.task == "download_file":
+    success = Storage(args.bucket_name).download_file(
+        bucket_name=args.bucket_name,
+        object_name=args.object_name,
+        filename=args.filename,
+    )
+elif args.task == "exists":
+    print(Storage(args.bucket_name).exists(args.object_name))
+    success = True
+elif args.task == "list_of_objects":
+    output = Storage(args.bucket_name).list_of_objects(
+        count=args.count,
+        suffix=args.suffix,
+        prefix=args.prefix,
+        recursive=args.recursive,
+    )
+    success = True
+elif args.task == "upload_file":
+    success = Storage(args.bucket_name).upload_file(
+        filename=args.filename,
+        object_name=args.object_name,
+        bucket_name=args.bucket_name,
+    )
+else:
+    print(f"{NAME}: {args.task}: command not found.")
+
+if success and output is not None:
+    if args.log:
+        print(f"{len(output):,} {args.item_name}(s): {delim.join(output)}")
+    else:
+        print(delim.join(output))
+
+sys_exit(logger, NAME, args.task, success, log=args.log)

bluer_objects/storage/classes.py

@@ -0,0 +1,237 @@
+from typing import Union, Tuple
+import boto3
+import botocore
+import os
+import os.path
+import time
+
+from blueness import module
+from blue_options import string
+from blue_options.logger import crash_report
+
+from bluer_objects import file, path, NAME
+from bluer_objects.env import (
+    ABCLI_OBJECT_ROOT,
+    ABCLI_AWS_S3_BUCKET_NAME,
+    ABCLI_AWS_REGION,
+    ABCLI_PATH_STATIC,
+    ABCLI_AWS_S3_PREFIX,
+    abcli_object_name,
+)
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+class Storage:
+    def __init__(self, bucket_name=ABCLI_AWS_S3_BUCKET_NAME):
+        self.region = ABCLI_AWS_REGION
+
+        try:
+            self.s3 = boto3.client("s3", region_name=self.region)
+        except:
+            assert False, f"{NAME}.Storage: failed."
+
+        self.bucket_name = bucket_name
+
+        assert self.create_bucket()
+
+    def create_bucket(
+        self,
+        bucket_name: str = "",
+    ) -> bool:
+        if not bucket_name:
+            bucket_name = self.bucket_name
+
+        try:
+            if boto3.resource("s3").Bucket(bucket_name).creation_date is not None:
+                logger.debug(f"-storage: create_bucket: {bucket_name}: already exists.")
+                return True
+
+            self.s3.create_bucket(
+                Bucket=bucket_name,
+                CreateBucketConfiguration={"LocationConstraint": self.region},
+            )
+        except:
+            crash_report(f"-storage: create_bucket: {bucket_name}: failed.")
+            return False
+
+        return True
+
+    def download_file(
+        self,
+        object_name: str,
+        filename: str = "",
+        bucket_name: Union[None, str] = None,
+        ignore_error: bool = False,
+        log: bool = True,
+        overwrite: bool = False,
+    ) -> bool:
+        if filename == "static":
+            filename = os.path.join(
+                ABCLI_PATH_STATIC,
+                object_name.replace("/", "-"),
+            )
+
+        if filename == "object":
+            filename = os.path.join(
+                ABCLI_OBJECT_ROOT,
+                "/".join(object_name.split("/")[1:]),
+            )
+
+        if not overwrite and file.exists(filename):
+            if log:
+                logger.info(f"✅  {filename}")
+            return True
+
+        if not path.create(file.path(filename)):
+            return False
+
+        if bucket_name is None:
+            bucket_name = self.bucket_name
+
+        success = True
+        try:
+            self.s3.download_file(bucket_name, object_name, filename)
+        except:
+            success = False
+
+        message = "{}.Storage.downloaded_file: {}/{} -> {}".format(
+            NAME,
+            bucket_name,
+            object_name,
+            filename,
+        )
+
+        if not success:
+            crash_report(f"{message}: failed.")
+        elif log:
+            logger.info(message)
+
+        return success
+
+    def list_of_objects(
+        self,
+        prefix: str,
+        bucket_name: Union[None, str] = None,
+        count: int = -1,
+        suffix: str = "",
+        recursive: bool = True,
+        include_folders: bool = False,
+    ):
+        prefix = f"{ABCLI_AWS_S3_PREFIX}/{prefix}"
+
+        if bucket_name is None:
+            bucket_name = self.bucket_name
+
+        output = []
+        try:
+            output = [
+                string.after(object_summary.key, prefix)
+                for object_summary in boto3.resource("s3")
+                .Bucket(bucket_name)
+                .objects.filter(Prefix=prefix)
+                # .limit(count)
+            ]
+        except:
+            crash_report("-storage: list_of_objects: failed.")
+
+        output = [thing[1:] if thing.startswith("/") else thing for thing in output]
+
+        if include_folders:
+            output = sorted(list({thing.split("/")[0] for thing in output}))
+        elif not recursive:
+            output = [thing for thing in output if "/" not in thing]
+
+        if suffix:
+            output = [thing for thing in output if thing.endswith(suffix)]
+
+        if count != -1:
+            output = output[:count]
+
+        return output
+
+    def exists(
+        self,
+        object_name: str,
+        bucket_name: Union[None, str] = None,
+    ) -> bool:
+        if bucket_name is None:
+            bucket_name = self.bucket_name
+
+        try:
+            boto3.resource("s3").Object(
+                bucket_name, "/".join([ABCLI_AWS_S3_PREFIX, object_name])
+            ).load()
+        except botocore.exceptions.ClientError as e:
+            if e.response["Error"]["Code"] != "404":
+                crash_report("-storage: exists: failed.")
+            return False
+
+        return True
+
+    def upload_file(
+        self,
+        filename: str,
+        object_name: Union[None, str] = None,
+        bucket_name: Union[None, str] = None,
+        overwrite: bool = False,
+    ) -> Tuple[bool, str, str]:
+        if bucket_name is None:
+            bucket_name = self.bucket_name
+
+        if not filename:
+            logger.warning(f"{NAME}: Storage: upload_file(): upload_file: no file.")
+            return False, bucket_name, ""
+
+        if object_name is None:
+            object_name = "{}/{}{}".format(
+                ABCLI_AWS_S3_PREFIX,
+                abcli_object_name,
+                (
+                    string.after(filename, abcli_object_name)
+                    if abcli_object_name in filename
+                    else filename
+                ),
+            )
+
+        if not overwrite and self.exists(object_name):
+            logger.info(f"✅ {object_name}.")
+            return True, bucket_name, object_name
+
+        success = True
+        time_ = time.time()
+        try:
+            self.s3.upload_file(filename, bucket_name, object_name)
+        except:
+            success = False
+        duration = time.time() - time_
+
+        message = "{}.Storage.download_file: {}:{} -> {}/{}: {}.".format(
+            NAME,
+            filename,
+            string.pretty_bytes(file.size(filename)),
+            bucket_name,
+            object_name,
+            string.pretty_duration(
+                duration,
+                include_ms=True,
+                short=True,
+            ),
+        )
+
+        if not success:
+            crash_report(f"{message}: failed.")
+        else:
+            logger.info(message)
+
+        return success, bucket_name, object_name
+
+    def url(self, object_name: str, filename: str) -> str:
+        return "https://{}.s3.{}.amazonaws.com/{}/{}/{}".format(
+            self.bucket_name,
+            self.region,
+            ABCLI_AWS_S3_PREFIX,
+            object_name,
+            filename,
+        )

bluer_objects/tests/test_README.py

@@ -0,0 +1,5 @@
+from bluer_objects import README
+
+
+def test_README_build():
+    assert README.build_me()

bluer_objects/tests/test_env.py

@@ -0,0 +1,27 @@
+from abcli.tests.test_env import test_abcli_env
+
+from bluer_objects import env
+
+
+def test_required_env():
+    test_abcli_env()
+
+
+def test_bluer_objects_env():
+    assert env.ABCLI_PUBLIC_PREFIX
+    assert env.VANWATCH_TEST_OBJECT
+
+    assert env.DATABRICKS_WORKSPACE
+    assert env.DATABRICKS_HOST
+    assert env.DATABRICKS_TOKEN
+
+    assert env.ABCLI_MLFLOW_EXPERIMENT_PREFIX
+
+
+def test_bluer_objects_env_RDS():
+    assert env.ABCLI_AWS_RDS_DB
+    assert env.ABCLI_AWS_RDS_PORT
+    assert env.ABCLI_AWS_RDS_USER
+
+    assert env.ABCLI_AWS_RDS_HOST
+    assert env.ABCLI_AWS_RDS_PASSWORD

bluer_objects/tests/test_file_load_save.py

@@ -0,0 +1,105 @@
+import pytest
+from typing import Callable, Union, Tuple, List
+import numpy as np
+
+from blue_options import string
+
+from bluer_objects import file, objects, env
+from bluer_objects.file.load import (
+    load_image,
+    load_json,
+    load_matrix,
+    load_text,
+)
+from bluer_objects.file.save import (
+    save_image,
+    save_json,
+    save_matrix,
+    save_text,
+)
+from bluer_objects.tests.test_objects import test_object
+
+
+@pytest.mark.parametrize(
+    [
+        "load_func",
+        "filename",
+        "save_func",
+    ],
+    [
+        [
+            load_image,
+            "Victoria41East.jpg",
+            save_image,
+        ],
+        [
+            load_json,
+            "vancouver.json",
+            save_json,
+        ],
+        [
+            load_text,
+            "vancouver.json",
+            save_text,
+        ],
+    ],
+)
+def test_file_load_save(
+    test_object,
+    load_func: Callable,
+    filename: str,
+    save_func: Union[Callable, None],
+):
+    success, thing = load_func(
+        objects.path_of(
+            object_name=test_object,
+            filename=filename,
+        )
+    )
+    assert success
+
+    if not save_func is None:
+        assert save_func(
+            file.add_suffix(
+                objects.path_of(
+                    object_name=test_object,
+                    filename=filename,
+                ),
+                string.random(),
+            ),
+            thing,
+        )
+
+
+@pytest.mark.unit
+@pytest.mark.parametrize(
+    ["size", "dtype"],
+    [
+        [(10, 3), np.uint8],
+        [(10, 3), np.float16],
+        [(10, 20, 30), np.uint8],
+        [(10, 30, 20), np.uint8],
+        [(10, 30, 20), np.float32],
+        [(10, 10, 10, 5), np.uint8],
+    ],
+)
+def test_file_load_save_matrix(
+    size: Tuple[int, ...],
+    dtype: Union[np.dtype, type],
+) -> None:
+    object_name = objects.unique_object(test_file_load_save_matrix)
+
+    test_matrix = (
+        np.random.randint(0, 256, size=size, dtype=dtype)
+        if dtype == np.uint8
+        else np.array(np.random.random(size), dtype=dtype)
+    )
+
+    filename = objects.path_of("test.npy", object_name)
+
+    assert save_matrix(filename, test_matrix)
+
+    success, matrix_read = load_matrix(filename)
+    assert success
+    assert (matrix_read == test_matrix).all()
+    assert matrix_read.dtype == dtype

bluer_objects/tests/test_fullname.py

@@ -0,0 +1,5 @@
+from bluer_objects import fullname
+
+
+def test_fullname():
+    assert fullname()

bluer_objects/tests/test_graphics.py

@@ -0,0 +1,28 @@
+import pytest
+
+from blueness import module
+
+from bluer_objects import file, objects, NAME
+from bluer_objects.env import VANWATCH_TEST_OBJECT
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+@pytest.fixture
+def test_image():
+    object_name = VANWATCH_TEST_OBJECT
+
+    assert objects.download(object_name)
+
+    success, matrix = file.load_image(
+        objects.path_of(
+            "Victoria41East.jpg",
+            object_name,
+        )
+    )
+    assert success
+
+    yield matrix
+
+    logger.info(f"deleting {NAME}.test_object ...")

bluer_objects/tests/test_graphics_frame.py

@@ -0,0 +1,11 @@
+import numpy as np
+
+from bluer_objects.graphics.frame import add_frame
+from bluer_objects.tests.test_graphics import test_image
+
+
+def test_graphics_frame_add_frame(test_image):
+    assert isinstance(
+        add_frame(test_image, width=12),
+        np.ndarray,
+    )

bluer_objects/tests/test_graphics_gif.py

@@ -0,0 +1,29 @@
+import pytest
+import glob
+
+from bluer_objects import objects
+from bluer_objects.graphics.gif import generate_animated_gif
+
+
+@pytest.mark.parametrize(
+    ["object_name", "scale"],
+    [
+        ["void", 1],
+        ["2024-05-07-18-45-13-31678", 2],
+    ],
+)
+def test_graphics_gif_generate_animated_gif(
+    object_name: str,
+    scale: int,
+):
+    assert objects.download(object_name)
+
+    list_of_images = list(glob.glob(objects.path_of("*.png", object_name)))
+    if object_name != "void":
+        assert list_of_images
+
+    assert generate_animated_gif(
+        list_of_images,
+        objects.path_of("test.gif", object_name),
+        scale=scale,
+    )

bluer_objects/tests/test_graphics_screen.py

@@ -0,0 +1,8 @@
+from bluer_objects.graphics import screen
+
+
+def test_graphics_screen_get_size():
+
+    size = screen.get_size()
+
+    assert len(size) == 2

bluer_objects/tests/test_graphics_signature.py

@@ -0,0 +1,80 @@
+import pytest
+from typing import Union, List
+import numpy as np
+
+from blue_options import string
+
+from bluer_objects.env import DUMMY_TEXT
+from bluer_objects.graphics.signature import add_signature, justify_text
+from bluer_objects.tests.test_graphics import test_image
+
+
+@pytest.mark.parametrize(
+    ["line_width"],
+    [
+        [10],
+        [80],
+    ],
+)
+@pytest.mark.parametrize(
+    ["word_wrap"],
+    [
+        [True],
+        [False],
+    ],
+)
+def test_graphics_signature_add_signature(
+    line_width: int,
+    word_wrap: bool,
+    test_image,
+):
+    assert isinstance(
+        add_signature(
+            test_image,
+            header=[DUMMY_TEXT],
+            footer=[DUMMY_TEXT, DUMMY_TEXT],
+            word_wrap=word_wrap,
+            line_width=line_width,
+        ),
+        np.ndarray,
+    )
+
+
+@pytest.mark.parametrize(
+    ["text"],
+    [
+        [string.random(length=800)],
+        [[string.random(length=120) for _ in range(3)]],
+    ],
+)
+@pytest.mark.parametrize(
+    ["line_width"],
+    [
+        [80],
+    ],
+)
+@pytest.mark.parametrize(
+    ["return_str"],
+    [
+        [True],
+        [False],
+    ],
+)
+def test_justify_text(
+    text: Union[str, List[str]],
+    line_width: int,
+    return_str: bool,
+):
+    justified_text = justify_text(
+        text,
+        line_width=line_width,
+        return_str=return_str,
+    )
+
+    if return_str:
+        assert isinstance(justified_text, str)
+    else:
+        assert isinstance(justified_text, list)
+
+        for line in justified_text:
+            assert len(line) <= line_width, f'len("{line}") > {line_width}'

bluer_objects/tests/test_graphics_text.py

@@ -0,0 +1,14 @@
+import numpy as np
+
+from bluer_objects.graphics.text import render_text
+from bluer_objects.env import DUMMY_TEXT
+
+
+def test_graphics_text_render_text():
+    assert isinstance(
+        render_text(
+            text=[DUMMY_TEXT for _ in range(10)],
+            image_width=1024,
+        ),
+        np.ndarray,
+    )

bluer_objects/tests/test_logger.py

@@ -0,0 +1,5 @@
+from bluer_objects.logger import logger
+
+
+def test_logger():
+    logger.info("testing")