bluer-objects 6.104.1__py3-none-any.whl → 6.377.1__py3-none-any.whl

bluer_objects/README/utils.py

@@ -0,0 +1,275 @@
+import os
+from typing import List, Dict, Union, Callable, Tuple
+
+from bluer_options.env import get_env
+from bluer_objects import file
+from bluer_objects import env
+from bluer_objects import NAME as MY_NAME, ICON as MY_ICON
+
+from bluer_objects.logger import logger
+
+
+variables: Dict[str, str] = {}
+
+
+def apply_legacy_on_line(line: str) -> str:
+    for before, after in {
+        "yaml:::": "metadata:::",
+        "--help--": "help:::",
+        "--include": "include:::",
+        "--table--": "items:::",
+        "--signature--": "signature:::",
+    }.items():
+        line = line.replace(before, after)
+    return line
+
+
+def apply_legacy(template: List[str]) -> List[str]:
+    logger.info("applying legacy conversions...")
+    template = [apply_legacy_on_line(line) for line in template]
+    return template
+
+
+def process_assets(
+    template_line: str,
+    assets_repo: str,
+) -> str:
+    if "assets:::" in template_line:
+        template_line = " ".join(
+            [
+                (
+                    (
+                        "![image](https://github.com/{}/blob/main/{}?raw=true)".format(
+                            assets_repo,
+                            token.split(":::")[1].strip(),
+                        )
+                        if any(
+                            token.endswith(extension)
+                            for extension in ["png", "jpg", "jpeg", "gif"]
+                        )
+                        else "[{}](https://github.com/{}/blob/main/{})".format(
+                            file.name_and_extension(token.split(":::")[1].strip()),
+                            assets_repo,
+                            token.split(":::")[1].strip(),
+                        )
+                    )
+                    if token.startswith("assets:::")
+                    else token
+                )
+                for token in template_line.split(" ")
+            ]
+        )
+
+    return template_line
+
+
+def process_details(template_line: str) -> List[str]:
+    suffix = template_line.split(":::", 1)[1]
+    if suffix:
+        content_section = [
+            "",
+            "<details>",
+            f"<summary>{suffix}</summary>",
+            "",
+        ]
+    else:
+        content_section = [
+            "",
+            "</details>",
+            "",
+        ]
+
+    return content_section
+
+
+def process_envs(template_line: str) -> str:
+    while "env:::" in template_line:
+        env_name = template_line.split("env:::", 1)[1]
+        if " " in env_name:
+            env_name = env_name.split(" ", 1)[0]
+        else:
+            if ":::" in env_name:
+                env_name = env_name.split(":::", 1)[0]
+
+        env_value = get_env(env_name)
+
+        template_line = template_line.replace(
+            f"env:::{env_name}",
+            env_value,
+        )
+        logger.info(f"{env_name} -> {env_value}")
+
+    return template_line
+
+
+def process_help(
+    template_line: str,
+    help_function: Union[Callable[[List[str]], str], None] = None,
+) -> Tuple[bool, List[str]]:
+    help_command = template_line.split("help:::")[1].strip()
+
+    tokens = help_command.strip().split(" ")[1:]
+
+    help_content = help_function(tokens)
+    if not help_content:
+        logger.error(f"help not found: {help_command}: {tokens}")
+        return False, []
+
+    logger.info(f"+= help: {help_command}")
+    print(help_content)
+    content_section = [
+        "```bash",
+        help_content,
+        "```",
+    ]
+
+    return True, content_section
+
+
+def process_include(
+    template_line: str,
+    template_path: str,
+) -> List[str]:
+    include_filename_relative = template_line.split(" ")[1].strip()
+    include_filename = file.absolute(
+        include_filename_relative,
+        template_path,
+    )
+
+    success, content_section = file.load_text(include_filename)
+    if not success:
+        return success
+
+    content_section = [
+        line for line in content_section if not line.startswith("used by:")
+    ]
+
+    include_title = (template_line.split(" ", 2) + ["", "", ""])[2]
+    if include_title:
+        content_section = [f"## {include_title}"] + content_section[1:]
+
+    if "include:::noref" not in template_line:
+        content_section += [
+            "using [{}]({}).".format(
+                file.name(include_filename),
+                include_filename_relative,
+            )
+        ]
+
+    logger.info(f"{MY_NAME}.build: including {include_filename} ...")
+
+    return content_section
+
+
+def process_mermaid(template_line: str) -> List[str]:
+    template_line_pieces = template_line.split('"')
+    if len(template_line_pieces) != 3:
+        logger.error(f"🧜🏽‍♀️  mermaid line not in expected format: {template_line}.")
+        return False
+
+    template_line_pieces[1] = (
+        template_line_pieces[1]
+        .replace("<", "&lt;")
+        .replace(">", "&gt;")
+        .replace(" ", "<br>")
+        .replace("~~", " ")
+    )
+
+    return ['"'.join(template_line_pieces)]
+
+
+def process_objects(template_line: str) -> str:
+    if "object:::" in template_line:
+        template_line = " ".join(
+            [
+                (
+                    "[{}](https://{}.{}/{}.tar.gz)".format(
+                        token.split(":::")[1].strip(),
+                        env.S3_PUBLIC_STORAGE_BUCKET,
+                        env.S3_STORAGE_ENDPOINT_URL.split("https://", 1)[1],
+                        token.split(":::")[1].strip(),
+                    )
+                    if token.startswith("object:::")
+                    else token
+                )
+                for token in template_line.split(" ")
+            ]
+        )
+
+    return template_line
+
+
+def process_title(
+    template_line: str,
+    filename: str,
+) -> Tuple[bool, List[str]]:
+    template_line_pieces = [
+        piece for piece in template_line.strip().split(":::") if piece
+    ]
+    reference = template_line_pieces[1] if len(template_line_pieces) >= 2 else "docs"
+
+    filename_path_pieces = file.path(filename).split(os.sep)
+    if reference not in filename_path_pieces:
+        logger.error(
+            "reference: {} not found in {}.".format(
+                reference,
+                template_line,
+            )
+        )
+        return False, []
+
+    title_pieces = filename_path_pieces[filename_path_pieces.index(reference) + 1 :]
+    filename_name = file.name(filename)
+    if filename_name != "README":
+        title_pieces.append(filename_name)
+
+    return True, [
+        "# {}".format(
+            ": ".join(
+                [
+                    piece.replace(
+                        "_",
+                        "-",
+                    )
+                    for piece in title_pieces
+                ]
+            )
+        )
+    ]
+
+
+def process_variable(template_line: str):
+    key, value = template_line.split("set:::", 1)[1].split(" ", 1)
+    variables[key] = value
+    logger.info(f"{key} = {value}")
+
+
+def signature(
+    REPO_NAME: str,
+    NAME: str,
+    ICON: str,
+    MODULE_NAME: str,
+    VERSION: str,
+) -> List[str]:
+    return [
+        "",
+        " ".join(
+            [
+                f"[![pylint](https://github.com/kamangir/{REPO_NAME}/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/{REPO_NAME}/actions/workflows/pylint.yml)",
+                f"[![pytest](https://github.com/kamangir/{REPO_NAME}/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/{REPO_NAME}/actions/workflows/pytest.yml)",
+                f"[![bashtest](https://github.com/kamangir/{REPO_NAME}/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/{REPO_NAME}/actions/workflows/bashtest.yml)",
+                f"[![PyPI version](https://img.shields.io/pypi/v/{MODULE_NAME}.svg)](https://pypi.org/project/{MODULE_NAME}/)",
+                f"[![PyPI - Downloads](https://img.shields.io/pypi/dd/{MODULE_NAME})](https://pypistats.org/packages/{MODULE_NAME})",
+            ]
+        ),
+        "",
+        "built by {} [`{}`]({}), based on {}[`{}-{}`]({}).".format(
+            MY_ICON,
+            "bluer README",
+            "https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README",
+            f"{ICON} " if ICON else "",
+            NAME,
+            VERSION,
+            f"https://github.com/kamangir/{REPO_NAME}",
+        ),
+    ]

bluer_objects/__init__.py

@@ -4,7 +4,7 @@ ICON = "🌀"
 
 DESCRIPTION = f"{ICON} Object management in Bash."
 
-VERSION = "6.104.1"
+VERSION = "6.377.1"
 
 REPO_NAME = "bluer-objects"
 

bluer_objects/assets/__main__.py

@@ -0,0 +1,57 @@
+import argparse
+
+from blueness import module
+from blueness.argparse.generic import sys_exit
+
+from bluer_objects import NAME
+from bluer_objects.assets.functions import publish
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+parser = argparse.ArgumentParser(NAME)
+parser.add_argument(
+    "task",
+    type=str,
+    help="publish",
+)
+parser.add_argument(
+    "--arg",
+    type=bool,
+    default=0,
+    help="0|1",
+)
+parser.add_argument(
+    "--object_name",
+    type=str,
+)
+parser.add_argument(
+    "--extensions",
+    type=str,
+    default="png",
+    help="png+txt",
+)
+parser.add_argument(
+    "--prefix",
+    type=str,
+    default="",
+)
+parser.add_argument(
+    "--asset_name",
+    type=str,
+    default="",
+)
+args = parser.parse_args()
+
+success = False
+if args.task == "publish":
+    success = publish(
+        object_name=args.object_name,
+        list_of_extensions=args.extensions.split("+"),
+        prefix=args.prefix,
+        asset_name=args.asset_name,
+    )
+else:
+    success = None
+
+sys_exit(logger, NAME, args.task, success)

bluer_objects/assets/functions.py

@@ -0,0 +1,62 @@
+import os
+from typing import List
+import glob
+from tqdm import tqdm
+
+from blueness import module
+
+from bluer_objects import objects, file
+from bluer_objects.env import abcli_path_git
+from bluer_objects import NAME
+from bluer_objects.logger import logger
+
+
+NAME = module.name(__file__, NAME)
+
+
+def publish(
+    object_name: str,
+    list_of_extensions: List[str],
+    prefix: str = "",
+    asset_name: str = "",
+    log: bool = True,
+) -> bool:
+    if not asset_name:
+        asset_name = object_name
+    logger.info(
+        "{}.publish: {}/{}.* for {} -> {}".format(
+            NAME,
+            object_name,
+            prefix,
+            ", ".join(list_of_extensions),
+            asset_name,
+        )
+    )
+
+    for extension in tqdm(list_of_extensions):
+        for filename in glob.glob(
+            objects.path_of(
+                filename=f"{prefix}*.{extension}",
+                object_name=object_name,
+            )
+        ):
+            published_filename = os.path.join(
+                abcli_path_git,
+                "assets",
+                asset_name,
+                file.name_and_extension(filename).replace(
+                    object_name,
+                    asset_name,
+                ),
+            )
+
+            if not file.copy(
+                filename,
+                published_filename,
+                log=log,
+            ):
+                return False
+
+    logger.info(f"🔗  https://github.com/kamangir/assets/tree/main/{asset_name}")
+
+    return True

bluer_objects/config.env

@@ -1,3 +1,11 @@
 ABCLI_MLFLOW_EXPERIMENT_PREFIX=experiment/
 
-BLUER_OBJECTS_STORAGE_INTERFACE=webdavzip
+MLFLOW_LOCK_WAIT_FOR_CLEARANCE=3
+MLFLOW_LOCK_WAIT_FOR_EXCLUSIVITY=1
+
+S3_STORAGE_BUCKET=kamangir
+S3_PUBLIC_STORAGE_BUCKET=kamangir-public
+
+BLUER_OBJECTS_STORAGE_INTERFACE=s3
+
+BLUER_OBJECTS_TEST_OBJECT=2025-09-22-13-11-08-cjbb3q

bluer_objects/env.py

@@ -35,8 +35,31 @@ DUMMY_TEXT = "This is some dummy text. This is some dummy text. This is some dum
 
 ABCLI_MLFLOW_EXPERIMENT_PREFIX = get_env("ABCLI_MLFLOW_EXPERIMENT_PREFIX")
 
+S3_STORAGE_BUCKET = get_env("S3_STORAGE_BUCKET")
+S3_PUBLIC_STORAGE_BUCKET = get_env("S3_PUBLIC_STORAGE_BUCKET")
+
+S3_STORAGE_ENDPOINT_URL = get_env("S3_STORAGE_ENDPOINT_URL")
+S3_STORAGE_AWS_ACCESS_KEY_ID = get_env("S3_STORAGE_AWS_ACCESS_KEY_ID")
+S3_STORAGE_AWS_SECRET_ACCESS_KEY = get_env("S3_STORAGE_AWS_SECRET_ACCESS_KEY")
+
 BLUER_OBJECTS_STORAGE_INTERFACE = get_env("BLUER_OBJECTS_STORAGE_INTERFACE")
 
+MLFLOW_DEPLOYMENT = get_env("MLFLOW_DEPLOYMENT", "local")
+if MLFLOW_DEPLOYMENT == "local":
+    MLFLOW_TRACKING_URI = os.path.join(
+        os.environ.get("HOME"),
+        "mlflow",
+    )
+else:
+    MLFLOW_TRACKING_URI = MLFLOW_DEPLOYMENT
+os.environ["MLFLOW_TRACKING_URI"] = MLFLOW_TRACKING_URI
+MLFLOW_LOG_ARTIFACTS = "arvan" not in MLFLOW_DEPLOYMENT
+
+MLFLOW_LOCK_WAIT_FOR_CLEARANCE = get_env("MLFLOW_LOCK_WAIT_FOR_CLEARANCE", 3)
+MLFLOW_LOCK_WAIT_FOR_EXCLUSIVITY = get_env("MLFLOW_LOCK_WAIT_FOR_EXCLUSIVITY", 1)
+
 WEBDAV_HOSTNAME = get_env("WEBDAV_HOSTNAME")
 WEBDAV_LOGIN = get_env("WEBDAV_LOGIN")
 WEBDAV_PASSWORD = get_env("WEBDAV_PASSWORD")
+
+BLUER_OBJECTS_TEST_OBJECT = get_env("BLUER_OBJECTS_TEST_OBJECT")

bluer_objects/file/__main__.py

@@ -31,24 +31,69 @@ parser.add_argument(
     help="<that-1+that-2+that-3>",
 )
 parser.add_argument(
-    "--size",
+    "--pretty",
     type=int,
-    default=16,
+    default=1,
+    help="0 | 1",
+)
+parser.add_argument(
+    "--save",
+    type=int,
+    default=1,
+    help="0 | 1",
+)
+parser.add_argument(
+    "--whole_line",
+    type=int,
+    default=0,
+    help="0 | 1",
+)
+parser.add_argument(
+    "--log",
+    type=int,
+    default=0,
+    help="0 | 1",
+)
+parser.add_argument(
+    "--cat",
+    type=int,
+    default=0,
+    help="0 | 1",
 )
 args = parser.parse_args()
 
 success = False
 if args.task == "replace":
-    logger.info(f"{NAME}.{args.task}: {args.this} -> {args.that} in {args.filename}")
+    logger.info(
+        f'{NAME}.{args.task}: "{args.this}" -> "{args.that}" in {args.filename}'
+    )
 
-    success, content = file.load_text(args.filename)
+    success, content = file.load_text(
+        args.filename,
+        log=args.log == 1,
+    )
     if success:
         for this, that in tqdm(zip(args.this.split("+"), args.that.split("+"))):
-            content = [line.replace(this, that) for line in content]
+            if args.whole_line:
+                content = [that if line == this else line for line in content]
+            else:
+                content = [line.replace(this, that) for line in content]
+
+        if args.save == 1:
+            success = file.save_text(
+                args.filename,
+                content,
+                log=args.log == 1,
+            )
+
+        if success and args.cat:
+            for line in content:
+                if any(this in line for this in args.this.split("+")):
+                    logger.info(line)
 
-        success = file.save_text(args.filename, content)
 elif args.task == "size":
-    print(string.pretty_bytes(file.size(args.filename)))
+    size = file.size(args.filename)
+    print(string.pretty_bytes(size) if args.pretty == 1 else size)
     success = True
 else:
     success = None

bluer_objects/file/functions.py

@@ -117,25 +117,29 @@ def copy(
         return False
 
     if log:
-        logger.info(f"{NAME}: {source} -> {destination}")
+        logger.info(f"{source} -copy-> {destination}")
 
     return True
 
 
 def delete(
     filename: str,
+    log: bool = False,
 ) -> bool:
     if not os.path.isfile(filename):
         return True
 
     try:
         os.remove(filename)
-
-        return True
     except:
         crash_report(f"{NAME}: delete({filename}): failed.")
         return False
 
+    if log:
+        logger.info(f"deleted {filename}.")
+
+    return True
+
 
 def download(
     url: str,
@@ -150,6 +154,8 @@ def download(
         return True
 
     try:
+        os.makedirs(path(filename), exist_ok=True)
+
         # https://stackoverflow.com/a/27406501
         with urllib3.PoolManager().request(
             "GET", url, preload_content=False
@@ -236,6 +242,7 @@ def list_of(
 def move(
     source: str,
     destination: str,
+    log: bool = True,
 ) -> bool:
     try:
         os.makedirs(path(destination), exist_ok=True)
@@ -246,6 +253,9 @@ def move(
         crash_report(f"{NAME}: move({source},{destination}): failed.")
         return False
 
+    if log:
+        logger.info(f"{source} -move-> {destination}")
+
     return True
 
 

bluer_objects/file/load.py

@@ -149,21 +149,14 @@ def load_matrix(
 def load_text(
     filename,
     ignore_error=False,
-    count=-1,
     log=False,
 ) -> Tuple[bool, List[str]]:
     success = True
     text = []
 
     try:
-        if count == -1:
-            with open(filename, "r") as fp:
-                text = fp.read()
-            text = text.split("\n")
-        else:
-            # https://stackoverflow.com/a/1767589/10917551
-            with open(filename) as fp:
-                text = [next(fp) for _ in range(count)]
+        with open(filename, "r") as fp:
+            text = fp.read().splitlines()
     except:
         success = False
         if not ignore_error: