bluer-objects 6.3.1__py3-none-any.whl

bluer_objects/.abcli/tests/metadata.sh

@@ -0,0 +1,53 @@
+#! /usr/bin/env bash
+
+function test_bluer_objects_metadata() {
+    local returned_value
+    for post_func in {1..3}; do
+        local object_name=$(abcli_string_timestamp)
+        local object_path=$ABCLI_OBJECT_ROOT/$object_name
+        local filename=$object_path/metadata.yaml
+
+        local key=$(abcli_string_random)
+        local value=$(abcli_string_random)
+
+        [[ "$post_func" == 1 ]] &&
+            abcli metadata post \
+                $key $value \
+                filename \
+                $filename \
+                --verbose 1
+
+        [[ "$post_func" == 2 ]] &&
+            abcli metadata post \
+                $key $value \
+                object,filename=metadata.yaml \
+                $object_name \
+                --verbose 1
+
+        [[ "$post_func" == 3 ]] &&
+            abcli metadata post \
+                $key $value \
+                path,filename=metadata.yaml \
+                $object_path \
+                --verbose 1
+
+        for get_func in {1..3}; do
+            [[ "$get_func" == 1 ]] &&
+                returned_value=$(abcli metadata get \
+                    key=$key,filename \
+                    $filename)
+
+            [[ "$get_func" == 2 ]] &&
+                returned_value=$(abcli metadata get \
+                    key=$key,filename=metadata.yaml,object \
+                    $object_name)
+
+            [[ "$get_func" == 3 ]] &&
+                returned_value=$(abcli metadata get \
+                    key=$key,filename=metadata.yaml,path \
+                    $object_path)
+
+            abcli_assert "$value" "$returned_value"
+        done
+    done
+}

bluer_objects/.abcli/tests/mlflow_cache.sh

@@ -0,0 +1,14 @@
+#! /usr/bin/env bash
+
+function test_bluer_objects_mlflow_cache() {
+    local keyword="test-keyword-$(abcli_string_timestamp_short)"
+    local value="test-value-$(abcli_string_timestamp_short)"
+
+    abcli mlflow cache write \
+        $keyword $value
+    [[ $? -ne 0 ]] && return 1
+
+    abcli_assert \
+        $(abcli mlflow cache read $keyword) \
+        $value
+}

bluer_objects/.abcli/tests/mlflow_logging.sh

@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+function test_bluer_objects_mlflow_logging() {
+    local object_name="test-object-$(abcli_string_timestamp_short)"
+
+    abcli clone \
+        upload \
+        vanwatch-mlflow-validation-2024-09-23-10673 \
+        "$object_name"
+
+    abcli_mlflow rm $object_name
+}

bluer_objects/.abcli/tests/mlflow_tags.sh

@@ -0,0 +1,29 @@
+#! /usr/bin/env bash
+
+function test_bluer_objects_mlflow_tags_validation() {
+    local object_name="test-object-$(abcli_string_timestamp_short)"
+    local tag="test-tag-$(abcli_string_timestamp_short)"
+    local value="test-value-$(abcli_string_timestamp_short)"
+
+    abcli mlflow tags set \
+        $object_name \
+        $tag=$value
+    [[ $? -ne 0 ]] && return 1
+
+    abcli_assert \
+        "$(abcli mlflow tags get $object_name --tag $tag)" \
+        $value
+    [[ $? -ne 0 ]] && return 1
+
+    abcli_assert \
+        "$(abcli mlflow tags get $object_name --tag some-tag)" \
+        - empty
+}
+
+function test_bluer_objects_mlflow_tags_search() {
+    local options=$1
+
+    abcli_mlflow_tags search \
+        cloned.firms_area-template-v1=True \
+        --log 0
+}

bluer_objects/.abcli/tests/mlflow_test.sh

@@ -0,0 +1,7 @@
+#! /usr/bin/env bash
+
+function test_bluer_objects_mlflow_test() {
+    local options=$1
+
+    abcli_mlflow_test "$@"
+}

bluer_objects/.abcli/tests/mysql_cache.sh

@@ -0,0 +1,15 @@
+#! /usr/bin/env bash
+
+function test_bluer_objects_mysql_cache() {
+    local keyword="test-keyword-$(abcli_string_timestamp_short)"
+    local value="test-value-$(abcli_string_timestamp_short)"
+
+    abcli mysql cache write \
+        $keyword $value \
+        validate
+    [[ $? -ne 0 ]] && return 1
+
+    abcli_assert \
+        $(abcli mysql cache read $keyword) \
+        $value
+}

bluer_objects/.abcli/tests/mysql_relations.sh

@@ -0,0 +1,20 @@
+#! /usr/bin/env bash
+
+function test_bluer_objects_mysql_relations() {
+    local object_name_1="test-object-$(abcli_string_timestamp_short)"
+    local object_name_2="test-object-$(abcli_string_timestamp_short)"
+
+    local relation=$(abcli mysql relations list --return_list 1 --count 1 --log 0)
+    [[ -z "$relation" ]] && return 1
+
+    abcli mysql relations set \
+        $object_name_1 \
+        $object_name_2 \
+        $relation \
+        validate
+    [[ $? -ne 0 ]] && return 1
+
+    abcli_assert \
+        $(abcli mysql relations get $object_name_1 $object_name_2 --log 0) \
+        $relation
+}

bluer_objects/.abcli/tests/mysql_tags.sh

@@ -0,0 +1,16 @@
+#! /usr/bin/env bash
+
+function test_bluer_objects_mysql_tags() {
+    local object_name="test-object-$(abcli_string_timestamp_short)"
+    local tag="test-tag-$(abcli_string_timestamp_short)"
+
+    abcli mysql tags set \
+        $object_name \
+        $tag \
+        validate
+    [[ $? -ne 0 ]] && return 1
+
+    abcli_assert \
+        $(abcli mysql tags get $object_name --log 0) \
+        $tag
+}

bluer_objects/.abcli/tests/test_gif.sh

@@ -0,0 +1,13 @@
+#! /usr/bin/env bash
+
+function test_bluer_objects_gif() {
+    local options=$1
+
+    abcli_gif \
+        ~upload,$options \
+        $VANWATCH_TEST_OBJECT \
+        --frame_duration 200 \
+        --output_filename test.gif \
+        --scale 2 \
+        --suffix .jpg
+}

bluer_objects/.abcli/tests/version.sh

@@ -0,0 +1,10 @@
+#! /usr/bin/env bash
+
+function test_bluer_objects_version() {
+    local options=$1
+
+    abcli_eval ,$options \
+        "bluer_objects version ${@:2}"
+
+    return 0
+}

bluer_objects/.abcli/upload.sh

@@ -0,0 +1,73 @@
+#! /usr/bin/env bash
+
+function abcli_upload() {
+    local options=$1
+    local filename=$(abcli_option "$options" filename)
+    local do_open=$(abcli_option_int "$options" open 1)
+    local do_solid=$(abcli_option_int "$options" solid 0)
+    local warn_if_exists=$(abcli_option_int "$options" warn_if_exists 1)
+    local log_to_mlflow=$(abcli_option_int "$options" mlflow 0)
+    local no_mlflow=$(abcli_option_int "$options" no_mlflow 0)
+
+    local object_name=$(abcli_clarify_object $2 .)
+    local object_path=$ABCLI_OBJECT_ROOT/$object_name
+
+    # https://stackoverflow.com/a/45200066
+    local exists=$(aws s3 ls $ABCLI_AWS_S3_BUCKET_NAME/$ABCLI_AWS_S3_PREFIX/$object_name.tar.gz)
+    if [ ! -z "$exists" ]; then
+        if [[ "$warn_if_exists" == 1 ]]; then
+            abcli_log_warning "@abcli: upload: $object_name.tar.gz already exists on the cloud, use \"abcli object open\" to open the object."
+        else
+            abcli_log "✅ ☁️  $object_name.tar.gz."
+        fi
+        return
+    fi
+
+    rm -rf $object_path/auxiliary
+
+    if [ ! -z "$filename" ]; then
+        local file_size=$(bluer_objects_file size $filename)
+        abcli_log "uploading $object_name/$filename ($file_size) ..."
+
+        aws s3 cp \
+            $object_path/$filename \
+            $ABCLI_S3_OBJECT_PREFIX/$object_name/
+
+        return
+    fi
+
+    if [ "$do_open" == 1 ]; then
+        abcli_log "uploading $object_name ..."
+
+        aws s3 sync \
+            $object_path/ \
+            $ABCLI_S3_OBJECT_PREFIX/$object_name/
+
+        abcli_tags set $object_name open
+    fi
+
+    if [ "$do_solid" == 1 ]; then
+        pushd $ABCLI_OBJECT_ROOT >/dev/null
+
+        tar -czvf \
+            $object_name.tar.gz \
+            ./$object_name
+
+        local object_size=$(bluer_objects_file size $object_path.tar.gz)
+        abcli_log "uploading $object_name.tar.gz ($object_size) ..."
+
+        aws s3 cp \
+            $object_name.tar.gz \
+            $ABCLI_S3_OBJECT_PREFIX/
+
+        abcli_tags set $object_name solid
+
+        popd >/dev/null
+    fi
+
+    if [[ "$log_to_mlflow" == 1 ]]; then
+        abcli_mlflow log_artifacts $object_name
+    elif [[ "$no_mlflow" == 0 ]]; then
+        abcli_mlflow log_run $object_name
+    fi
+}

bluer_objects/README/__init__.py

@@ -0,0 +1,29 @@
+import os
+
+from blueness import module
+
+from bluer_objects import NAME as MY_NAME
+from bluer_objects import file
+from bluer_objects.README.functions import build
+from bluer_objects.README.items import Items
+from bluer_objects.logger import logger
+
+MY_NAME = module.name(__file__, MY_NAME)
+
+
+def build_me() -> bool:
+    from bluer_objects import NAME, VERSION, REPO_NAME, ICON
+
+    return all(
+        build(
+            path=os.path.join(file.path(__file__), readme["path"]),
+            ICON=ICON,
+            NAME=NAME,
+            VERSION=VERSION,
+            REPO_NAME=REPO_NAME,
+        )
+        for readme in [
+            {"path": "../.."},
+            {"path": "."},
+        ]
+    )

bluer_objects/README/functions.py

@@ -0,0 +1,285 @@
+from typing import List, Dict, Union, Callable
+import os
+import yaml
+
+from blueness import module
+from blue_options import fullname
+
+from bluer_objects import NAME as MY_NAME, ICON as MY_ICON
+from bluer_objects.env import ABCLI_PUBLIC_PREFIX
+from bluer_objects.metadata import get_from_object
+from bluer_objects import file
+from bluer_objects import markdown
+from bluer_objects.logger import logger
+
+MY_NAME = module.name(__file__, MY_NAME)
+
+
+def build(
+    NAME: str,
+    VERSION: str,
+    REPO_NAME: str,
+    items: List[str] = [],
+    template_filename: str = "",
+    filename: str = "",
+    path: str = "",
+    cols: int = 3,
+    ICON: str = "",
+    MODULE_NAME: str = "",
+    macros: Dict[str, str] = {},
+    help_function: Union[Callable[[List[str]], str], None] = None,
+    legacy_mode: bool = True,
+    assets_repo: str = "kamangir/assets",
+) -> bool:
+    if path:
+        if path.endswith(".md"):
+            filename = path
+            template_filename = file.add_suffix(path, "template")
+        else:
+            filename = os.path.join(path, "README.md")
+            template_filename = os.path.join(path, "template.md")
+
+    if not MODULE_NAME:
+        MODULE_NAME = REPO_NAME
+
+    logger.info(
+        "{}.build: {}-{}: {}[{}]: {} -{}> {}".format(
+            MY_NAME,
+            NAME,
+            VERSION,
+            REPO_NAME,
+            MODULE_NAME,
+            template_filename,
+            "+legacy-" if legacy_mode else "",
+            filename,
+        )
+    )
+
+    table_of_items = markdown.generate_table(items, cols=cols) if cols > 0 else items
+
+    signature = [
+        # f'to use on [AWS SageMaker](https://aws.amazon.com/sagemaker/) replace `<plugin-name>` with "{NAME}" and follow [these instructions](https://github.com/kamangir/notebooks-and-scripts/blob/main/SageMaker.md).',
+        "",
+        " ".join(
+            [
+                f"[![pylint](https://github.com/kamangir/{REPO_NAME}/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/{REPO_NAME}/actions/workflows/pylint.yml)",
+                f"[![pytest](https://github.com/kamangir/{REPO_NAME}/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/{REPO_NAME}/actions/workflows/pytest.yml)",
+                f"[![bashtest](https://github.com/kamangir/{REPO_NAME}/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/{REPO_NAME}/actions/workflows/bashtest.yml)",
+                f"[![PyPI version](https://img.shields.io/pypi/v/{MODULE_NAME}.svg)](https://pypi.org/project/{MODULE_NAME}/)",
+                f"[![PyPI - Downloads](https://img.shields.io/pypi/dd/{MODULE_NAME})](https://pypistats.org/packages/{MODULE_NAME})",
+            ]
+        ),
+        "",
+        "built by {} [`{}`]({}), based on {}[`{}-{}`]({}).".format(
+            MY_ICON,
+            fullname(),
+            "https://github.com/kamangir/awesome-bash-cli",
+            f"{ICON} " if ICON else "",
+            NAME,
+            VERSION,
+            f"https://github.com/kamangir/{REPO_NAME}",
+        ),
+    ]
+
+    success, template = file.load_text(template_filename)
+    if not success:
+        return success
+
+    def apply_legacy(line: str) -> str:
+        for before, after in {
+            "yaml:::": "metadata:::",
+            "--help--": "help:::",
+            "--include": "include:::",
+            "--table--": "items:::",
+            "--signature--": "signature:::",
+        }.items():
+            line = line.replace(before, after)
+        return line
+
+    if legacy_mode:
+        logger.info("applying legacy conversions...")
+        template = [apply_legacy(line) for line in template]
+
+    content: List[str] = []
+    mermaid_started: bool = False
+    variables: Dict[str, str] = {}
+    for template_line in template:
+        if template_line.startswith("ignore:::"):
+            content_section = [template_line.split(":::", 1)[1].strip()]
+        else:
+            if template_line.startswith("set:::"):
+                key, value = template_line.split("set:::", 1)[1].split(" ", 1)
+                variables[key] = value
+                logger.info(f":::{key} = {value}")
+                continue
+
+            for key, value in variables.items():
+                template_line = template_line.replace(
+                    f"get:::{key}",
+                    value,
+                )
+
+            if "assets:::" in template_line:
+                template_line = " ".join(
+                    [
+                        (
+                            (
+                                "![image](https://github.com/{}/blob/main/{}?raw=true)".format(
+                                    assets_repo,
+                                    token.split(":::")[1].strip(),
+                                )
+                                if any(
+                                    token.endswith(extension)
+                                    for extension in ["png", "jpg", "jpeg", "gif"]
+                                )
+                                else "[{}](https://github.com/{}/blob/main/{})".format(
+                                    file.name_and_extension(
+                                        token.split(":::")[1].strip()
+                                    ),
+                                    assets_repo,
+                                    token.split(":::")[1].strip(),
+                                )
+                            )
+                            if token.startswith("assets:::")
+                            else token
+                        )
+                        for token in template_line.split(" ")
+                    ]
+                )
+
+            if "object:::" in template_line:
+                template_line = " ".join(
+                    [
+                        (
+                            "[{}]({}/{}.tar.gz)".format(
+                                token.split(":::")[1].strip(),
+                                ABCLI_PUBLIC_PREFIX,
+                                token.split(":::")[1].strip(),
+                            )
+                            if token.startswith("object:::")
+                            else token
+                        )
+                        for token in template_line.split(" ")
+                    ]
+                )
+
+            content_section: List[str] = [template_line]
+
+            if template_line.startswith("details:::"):
+                suffix = template_line.split(":::", 1)[1]
+                if suffix:
+                    content_section = [
+                        "",
+                        "<details>",
+                        f"<summary>{suffix}</summary>",
+                        "",
+                    ]
+                else:
+                    content_section = [
+                        "",
+                        "</details>",
+                        "",
+                    ]
+            elif template_line.startswith("metadata:::"):
+                object_name_and_key = template_line.split(":::", 1)[1]
+                if ":::" not in object_name_and_key:
+                    object_name_and_key += ":::"
+                object_name, key = object_name_and_key.split(":::", 1)
+
+                value = get_from_object(
+                    object_name,
+                    key,
+                    {},
+                    download=True,
+                )
+
+                content_section = (
+                    ["```yaml"]
+                    + yaml.dump(
+                        value,
+                        default_flow_style=False,
+                    ).split("\n")
+                    + ["```"]
+                )
+            elif template_line.startswith("```mermaid"):
+                mermaid_started = True
+                logger.info("🧜🏽‍♀️  detected ...")
+            elif mermaid_started and template_line.startswith("```"):
+                mermaid_started = False
+            elif mermaid_started:
+                if '"' in template_line and ":::folder" not in template_line:
+                    template_line_pieces = template_line.split('"')
+                    if len(template_line_pieces) != 3:
+                        logger.error(
+                            f"🧜🏽‍♀️  mermaid line not in expected format: {template_line}."
+                        )
+                        return False
+
+                    template_line_pieces[1] = (
+                        template_line_pieces[1]
+                        .replace("<", "&lt;")
+                        .replace(">", "&gt;")
+                        .replace(" ", "<br>")
+                        .replace("~~", " ")
+                    )
+
+                    content_section = ['"'.join(template_line_pieces)]
+            elif "items:::" in template_line:
+                content_section = table_of_items
+            elif "signature:::" in template_line:
+                content_section = signature
+            elif "include:::" in template_line:
+                include_filename_relative = template_line.split(" ")[1].strip()
+                include_filename = file.absolute(
+                    include_filename_relative,
+                    file.path(template_filename),
+                )
+
+                success, content_section = file.load_text(include_filename)
+                if not success:
+                    return success
+
+                content_section = [
+                    line for line in content_section if not line.startswith("used by:")
+                ]
+
+                include_title = (template_line.split(" ", 2) + ["", "", ""])[2]
+                if include_title:
+                    content_section = [f"## {include_title}"] + content_section[1:]
+
+                if "include:::noref" not in template_line:
+                    content_section += [
+                        "using [{}]({}).".format(
+                            file.name(include_filename),
+                            include_filename_relative,
+                        )
+                    ]
+
+                logger.info(f"{MY_NAME}.build: including {include_filename} ...")
+            elif "help:::" in template_line:
+                if help_function is not None:
+                    help_command = template_line.split("help:::")[1].strip()
+
+                    tokens = help_command.strip().split(" ")[1:]
+
+                    help_content = help_function(tokens)
+                    if not help_content:
+                        logger.warning(f"help not found: {help_command}: {tokens}")
+                        return False
+
+                    logger.info(f"+= help: {help_command}")
+                    print(help_content)
+                    content_section = [
+                        "```bash",
+                        help_content,
+                        "```",
+                    ]
+            else:
+                for macro, macro_value in macros.items():
+                    if macro in template_line:
+                        content_section = macro_value
+                        break
+
+        content += content_section
+
+    return file.save_text(filename, content)

bluer_objects/README/items.py

@@ -0,0 +1,30 @@
+from typing import List, Dict
+
+
+# name, url, marquee, description
+def Items(
+    items: List[Dict[str, str]],
+) -> List[str]:
+    return [
+        (
+            "[`{}`]({}) [![image]({})]({}) {}".format(
+                item["name"],
+                item.get(
+                    "url",
+                    "#",
+                ),
+                item.get(
+                    "marquee",
+                    "https://github.com/kamangir/assets/raw/main/blue-plugin/marquee.png?raw=true",
+                ),
+                item.get(
+                    "url",
+                    "#",
+                ),
+                item.get("description", ""),
+            )
+            if "name" in item
+            else ""
+        )
+        for item in items
+    ]

bluer_objects/__init__.py

@@ -0,0 +1,19 @@
+NAME = "bluer_objects"
+
+ICON = "🌀"
+
+DESCRIPTION = f"{ICON} data objects for Bash."
+
+VERSION = "6.3.1"
+
+REPO_NAME = "bluer-objects"
+
+MARQUEE = (
+    "https://github.com/kamangir/assets/raw/main/blue-objects/marquee.png?raw=true"
+)
+
+ALIAS = "@objects"
+
+
+def fullname() -> str:
+    return f"{NAME}-{VERSION}"

bluer_objects/__main__.py

@@ -0,0 +1,16 @@
+from blueness.argparse.generic import main
+
+from bluer_objects import NAME, VERSION, DESCRIPTION, ICON, README
+from bluer_objects.logger import logger
+
+main(
+    ICON=ICON,
+    NAME=NAME,
+    DESCRIPTION=DESCRIPTION,
+    VERSION=VERSION,
+    main_filename=__file__,
+    tasks={
+        "build_README": lambda _: README.build_me(),
+    },
+    logger=logger,
+)

bluer_objects/config.env

@@ -0,0 +1,22 @@
+ABCLI_AWS_REGION=ca-central-1
+
+ABCLI_AWS_S3_BUCKET_NAME=kamangir
+
+ABCLI_AWS_S3_PREFIX=bolt
+
+ABCLI_AWS_S3_PUBLIC_BUCKET_NAME=kamangir-public
+
+ABCLI_PUBLIC_PREFIX=https://kamangir-public.s3.ca-central-1.amazonaws.com
+
+VANWATCH_TEST_OBJECT=vanwatch-test-object-v2
+
+# refer to ABCLI_AWS_* in sample.env for details
+ABCLI_AWS_RDS_DB=abcli
+ABCLI_AWS_RDS_PORT=3306
+ABCLI_AWS_RDS_USER=admin
+
+DATABRICKS_WORKSPACE=abcli-v4
+
+ABCLI_MLFLOW_URL_SUBDOMAIN=ml
+
+ABCLI_MLFLOW_EXPERIMENT_PREFIX=/Users/kamangirblog@gmail.com/