bluer-objects 6.223.1__tar.gz → 6.231.1__tar.gz

{bluer_objects-6.223.1/bluer_objects.egg-info → bluer_objects-6.231.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bluer_objects
-Version: 6.223.1
+Version: 6.231.1
 Summary: 🌀 Object management in Bash.
 Home-page: https://github.com/kamangir/bluer-objects
 Author: Arash Abadpour (Kamangir)
@@ -64,6 +64,6 @@ pip install bluer-objects
 
 [![pylint](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/bluer-objects.svg)](https://pypi.org/project/bluer-objects/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/bluer-objects)](https://pypistats.org/packages/bluer-objects)
 
-built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.223.1`](https://github.com/kamangir/bluer-objects).
+built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.231.1`](https://github.com/kamangir/bluer-objects).
 
 built by 🌀 [`blueness-3.118.1`](https://github.com/kamangir/blueness).

{bluer_objects-6.223.1 → bluer_objects-6.231.1}/README.md

@@ -30,4 +30,4 @@ pip install bluer-objects
 
 [![pylint](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/bluer-objects.svg)](https://pypi.org/project/bluer-objects/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/bluer-objects)](https://pypistats.org/packages/bluer-objects)
 
-built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.223.1`](https://github.com/kamangir/bluer-objects).
+built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.231.1`](https://github.com/kamangir/bluer-objects).

bluer_objects-6.231.1/bluer_objects/README/functions.py

@@ -0,0 +1,186 @@
+from typing import List, Dict, Union, Callable
+import os
+import yaml
+
+from blueness import module
+
+from bluer_objects import NAME as MY_NAME
+from bluer_objects.metadata import get_from_object
+from bluer_objects import file, env
+from bluer_objects import markdown
+from bluer_objects.README.utils import (
+    apply_legacy,
+    process_assets,
+    process_details,
+    process_envs,
+    process_help,
+    process_include,
+    process_mermaid,
+    process_objects,
+    process_variable,
+    signature,
+    variables,
+)
+from bluer_objects.logger import logger
+
+MY_NAME = module.name(__file__, MY_NAME)
+
+
+def build(
+    NAME: str,
+    VERSION: str,
+    REPO_NAME: str,
+    items: List[str] = [],
+    template_filename: str = "",
+    filename: str = "",
+    path: str = "",
+    cols: int = 3,
+    ICON: str = "",
+    MODULE_NAME: str = "",
+    macros: Dict[str, str] = {},
+    help_function: Union[Callable[[List[str]], str], None] = None,
+    legacy_mode: bool = True,
+    assets_repo: str = "kamangir/assets",
+) -> bool:
+    if path:
+        if path.endswith(".md"):
+            filename = path
+            template_filename = file.add_suffix(path, "template")
+        else:
+            filename = os.path.join(path, "README.md")
+            template_filename = os.path.join(path, "template.md")
+
+    if not MODULE_NAME:
+        MODULE_NAME = REPO_NAME
+
+    logger.info(
+        "{}.build: {}-{}: {}[{}]: {} -{}> {}".format(
+            MY_NAME,
+            NAME,
+            VERSION,
+            REPO_NAME,
+            MODULE_NAME,
+            template_filename,
+            "+legacy-" if legacy_mode else "",
+            filename,
+        )
+    )
+
+    table_of_items = markdown.generate_table(items, cols=cols) if cols > 0 else items
+
+    success, template = file.load_text(template_filename)
+    if not success:
+        return success
+
+    if legacy_mode:
+        template = apply_legacy(template)
+
+    content: List[str] = []
+    mermaid_started: bool = False
+    for template_line in template:
+        if template_line.startswith("ignore:::"):
+            content += [template_line.split(":::", 1)[1].strip()]
+            continue
+
+        template_line = process_envs(template_line)
+
+        for key, value in variables.items():
+            template_line = template_line.replace(
+                f"get:::{key}",
+                value,
+            )
+
+        if "metadata:::" in template_line:
+            object_name_and_key = template_line.split("metadata:::", 1)[1]
+            if " " in object_name_and_key:
+                object_name_and_key = object_name_and_key.split(" ", 1)[0]
+            if ":::" not in object_name_and_key:
+                object_name_and_key += ":::"
+            object_name, key = object_name_and_key.split(":::", 1)
+
+            value = get_from_object(
+                object_name,
+                key,
+                {},
+                download=True,
+            )
+
+            logger.info(f"metadata[{object_name_and_key}] = {value}")
+
+            if template_line.startswith("metadata:::"):
+                content += (
+                    ["```yaml"]
+                    + yaml.dump(
+                        value,
+                        default_flow_style=False,
+                    ).split("\n")
+                    + ["```"]
+                )
+                continue
+
+            template_line = template_line.replace(
+                f"metadata:::{object_name}:::{key}",
+                str(value),
+            )
+
+        if template_line.startswith("set:::"):
+            process_variable(template_line)
+            continue
+
+        template_line = process_assets(template_line, assets_repo)
+
+        template_line = process_objects(template_line)
+
+        if template_line.startswith("details:::"):
+            content += process_details(template_line)
+            continue
+
+        if "items:::" in template_line:
+            content += table_of_items
+            continue
+
+        if "include:::" in template_line:
+            content += process_include(
+                template_line,
+                file.path(template_filename),
+            )
+            continue
+
+        if "signature:::" in template_line:
+            content += signature(
+                REPO_NAME,
+                NAME,
+                ICON,
+                MODULE_NAME,
+                VERSION,
+            )
+            continue
+
+        if "help:::" in template_line:
+            if help_function is None:
+                logger.warning("help_function not found.")
+            else:
+                content += process_help(
+                    template_line,
+                    help_function,
+                )
+            continue
+
+        content_section = [template_line]
+        if template_line.startswith("```mermaid"):
+            mermaid_started = True
+            logger.info("🧜🏽‍♀️  detected ...")
+        elif mermaid_started and template_line.startswith("```"):
+            mermaid_started = False
+        elif mermaid_started:
+            if '"' in template_line and ":::folder" not in template_line:
+                content_section = process_mermaid(template_line)
+        else:
+            for macro, macro_value in macros.items():
+                if macro in template_line:
+                    content_section = macro_value
+                    break
+
+        content += content_section
+
+    return file.save_text(filename, content)

bluer_objects-6.231.1/bluer_objects/README/utils.py

@@ -0,0 +1,235 @@
+from typing import List, Tuple, Dict, Union, Callable
+
+from bluer_options.env import get_env
+from bluer_objects import file
+from bluer_objects import env
+from bluer_objects import NAME as MY_NAME, ICON as MY_ICON
+
+from bluer_objects.logger import logger
+
+
+variables: Dict[str, str] = {}
+
+
+def apply_legacy_on_line(line: str) -> str:
+    for before, after in {
+        "yaml:::": "metadata:::",
+        "--help--": "help:::",
+        "--include": "include:::",
+        "--table--": "items:::",
+        "--signature--": "signature:::",
+    }.items():
+        line = line.replace(before, after)
+    return line
+
+
+def apply_legacy(template: List[str]) -> List[str]:
+    logger.info("applying legacy conversions...")
+    template = [apply_legacy_on_line(line) for line in template]
+    return template
+
+
+def process_assets(
+    template_line: str,
+    assets_repo: str,
+) -> str:
+    if "assets:::" in template_line:
+        template_line = " ".join(
+            [
+                (
+                    (
+                        "![image](https://github.com/{}/blob/main/{}?raw=true)".format(
+                            assets_repo,
+                            token.split(":::")[1].strip(),
+                        )
+                        if any(
+                            token.endswith(extension)
+                            for extension in ["png", "jpg", "jpeg", "gif"]
+                        )
+                        else "[{}](https://github.com/{}/blob/main/{})".format(
+                            file.name_and_extension(token.split(":::")[1].strip()),
+                            assets_repo,
+                            token.split(":::")[1].strip(),
+                        )
+                    )
+                    if token.startswith("assets:::")
+                    else token
+                )
+                for token in template_line.split(" ")
+            ]
+        )
+
+    return template_line
+
+
+def process_details(template_line: str) -> List[str]:
+    suffix = template_line.split(":::", 1)[1]
+    if suffix:
+        content_section = [
+            "",
+            "<details>",
+            f"<summary>{suffix}</summary>",
+            "",
+        ]
+    else:
+        content_section = [
+            "",
+            "</details>",
+            "",
+        ]
+
+    return content_section
+
+
+def process_envs(template_line: str) -> str:
+    while "env:::" in template_line:
+        env_name = template_line.split("env:::", 1)[1]
+        if " " in env_name:
+            env_name = env_name.split(" ", 1)[0]
+        else:
+            if ":::" in env_name:
+                env_name = env_name.split(":::", 1)[0]
+
+        env_value = get_env(env_name)
+
+        template_line = template_line.replace(
+            f"env:::{env_name}",
+            env_value,
+        )
+        logger.info(f"{env_name} -> {env_value}")
+
+    return template_line
+
+
+def process_help(
+    template_line: str,
+    help_function: Union[Callable[[List[str]], str], None] = None,
+) -> List[str]:
+    help_command = template_line.split("help:::")[1].strip()
+
+    tokens = help_command.strip().split(" ")[1:]
+
+    help_content = help_function(tokens)
+    if not help_content:
+        logger.warning(f"help not found: {help_command}: {tokens}")
+        return False
+
+    logger.info(f"+= help: {help_command}")
+    print(help_content)
+    content_section = [
+        "```bash",
+        help_content,
+        "```",
+    ]
+
+    return content_section
+
+
+def process_include(
+    template_line: str,
+    template_path: str,
+) -> List[str]:
+    include_filename_relative = template_line.split(" ")[1].strip()
+    include_filename = file.absolute(
+        include_filename_relative,
+        template_path,
+    )
+
+    success, content_section = file.load_text(include_filename)
+    if not success:
+        return success
+
+    content_section = [
+        line for line in content_section if not line.startswith("used by:")
+    ]
+
+    include_title = (template_line.split(" ", 2) + ["", "", ""])[2]
+    if include_title:
+        content_section = [f"## {include_title}"] + content_section[1:]
+
+    if "include:::noref" not in template_line:
+        content_section += [
+            "using [{}]({}).".format(
+                file.name(include_filename),
+                include_filename_relative,
+            )
+        ]
+
+    logger.info(f"{MY_NAME}.build: including {include_filename} ...")
+
+    return content_section
+
+
+def process_mermaid(template_line: str) -> List[str]:
+    template_line_pieces = template_line.split('"')
+    if len(template_line_pieces) != 3:
+        logger.error(f"🧜🏽‍♀️  mermaid line not in expected format: {template_line}.")
+        return False
+
+    template_line_pieces[1] = (
+        template_line_pieces[1]
+        .replace("<", "&lt;")
+        .replace(">", "&gt;")
+        .replace(" ", "<br>")
+        .replace("~~", " ")
+    )
+
+    return ['"'.join(template_line_pieces)]
+
+
+def process_objects(template_line: str) -> str:
+    if "object:::" in template_line:
+        template_line = " ".join(
+            [
+                (
+                    "[{}](https://{}.{}/{}.tar.gz)".format(
+                        token.split(":::")[1].strip(),
+                        env.S3_PUBLIC_STORAGE_BUCKET,
+                        env.S3_STORAGE_ENDPOINT_URL.split("https://", 1)[1],
+                        token.split(":::")[1].strip(),
+                    )
+                    if token.startswith("object:::")
+                    else token
+                )
+                for token in template_line.split(" ")
+            ]
+        )
+
+    return template_line
+
+
+def process_variable(template_line: str):
+    key, value = template_line.split("set:::", 1)[1].split(" ", 1)
+    variables[key] = value
+    logger.info(f"{key} = {value}")
+
+
+def signature(
+    REPO_NAME: str,
+    NAME: str,
+    ICON: str,
+    MODULE_NAME: str,
+    VERSION: str,
+) -> List[str]:
+    return [
+        "",
+        " ".join(
+            [
+                f"[![pylint](https://github.com/kamangir/{REPO_NAME}/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/{REPO_NAME}/actions/workflows/pylint.yml)",
+                f"[![pytest](https://github.com/kamangir/{REPO_NAME}/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/{REPO_NAME}/actions/workflows/pytest.yml)",
+                f"[![bashtest](https://github.com/kamangir/{REPO_NAME}/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/{REPO_NAME}/actions/workflows/bashtest.yml)",
+                f"[![PyPI version](https://img.shields.io/pypi/v/{MODULE_NAME}.svg)](https://pypi.org/project/{MODULE_NAME}/)",
+                f"[![PyPI - Downloads](https://img.shields.io/pypi/dd/{MODULE_NAME})](https://pypistats.org/packages/{MODULE_NAME})",
+            ]
+        ),
+        "",
+        "built by {} [`{}`]({}), based on {}[`{}-{}`]({}).".format(
+            MY_ICON,
+            "bluer README",
+            "https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README",
+            f"{ICON} " if ICON else "",
+            NAME,
+            VERSION,
+            f"https://github.com/kamangir/{REPO_NAME}",
+        ),
+    ]

{bluer_objects-6.223.1 → bluer_objects-6.231.1}/bluer_objects/__init__.py

@@ -4,7 +4,7 @@ ICON = "🌀"
 
 DESCRIPTION = f"{ICON} Object management in Bash."
 
-VERSION = "6.223.1"
+VERSION = "6.231.1"
 
 REPO_NAME = "bluer-objects"
 

{bluer_objects-6.223.1 → bluer_objects-6.231.1}/bluer_objects/storage/WebDAV.py

@@ -4,6 +4,7 @@ from webdav3.client import Client
 from bluer_objects.storage.base import StorageInterface
 from bluer_objects import env, file, path
 from bluer_objects import objects
+from bluer_objects.storage.policies import DownloadPolicy
 from bluer_objects.logger import logger
 
 
@@ -56,6 +57,7 @@ class WebDAVInterface(StorageInterface):
         object_name: str,
         filename: str = "",
         log: bool = True,
+        policy: DownloadPolicy = DownloadPolicy.NONE,
     ) -> bool:
         local_path = objects.path_of(
             object_name=object_name,
@@ -80,6 +82,7 @@ class WebDAVInterface(StorageInterface):
             object_name=object_name,
             filename=filename,
             log=log,
+            policy=policy,
         )
 
     def upload(

{bluer_objects-6.223.1 → bluer_objects-6.231.1}/bluer_objects/storage/WebDAVrequest.py

@@ -8,6 +8,7 @@ from tqdm import tqdm
 from bluer_objects.storage.base import StorageInterface
 from bluer_objects import env, file, path
 from bluer_objects import objects
+from bluer_objects.storage.policies import DownloadPolicy
 from bluer_objects.logger import logger
 
 
@@ -136,6 +137,7 @@ class WebDAVRequestInterface(StorageInterface):
         object_name: str,
         filename: str = "",
         log: bool = True,
+        policy: DownloadPolicy = DownloadPolicy.NONE,
     ) -> bool:
         if filename:
             local_path = objects.path_of(
@@ -176,6 +178,7 @@ class WebDAVRequestInterface(StorageInterface):
                     object_name=object_name,
                     filename=filename,
                     log=log,
+                    policy=policy,
                 )
 
             logger.error(f"failed to download: {response.status_code}")
@@ -193,6 +196,7 @@ class WebDAVRequestInterface(StorageInterface):
                 object_name=object_name,
                 filename=filename_,
                 log=log,
+                policy=policy,
             ):
                 return False
 

{bluer_objects-6.223.1 → bluer_objects-6.231.1}/bluer_objects/storage/WebDAVzip.py

@@ -5,9 +5,10 @@ from webdav3.client import Client
 from tqdm import tqdm
 
 from bluer_objects.storage.base import StorageInterface
-from bluer_objects import env, file, path
+from bluer_objects import env
 from bluer_objects import objects
-from bluer_objects.host import zip, unzip
+from bluer_objects.host import unzip
+from bluer_objects.storage.policies import DownloadPolicy
 from bluer_objects.logger import logger
 
 
@@ -65,6 +66,7 @@ class WebDAVzipInterface(StorageInterface):
         object_name: str,
         filename: str = "",
         log: bool = True,
+        policy: DownloadPolicy = DownloadPolicy.NONE,
     ) -> bool:
         object_path = objects.object_path(
             object_name=object_name,
@@ -99,6 +101,7 @@ class WebDAVzipInterface(StorageInterface):
         return super().download(
             object_name=object_name,
             log=log,
+            policy=policy,
         )
 
     def ls(

{bluer_objects-6.223.1 → bluer_objects-6.231.1}/bluer_objects/storage/__init__.py

@@ -5,6 +5,7 @@ from bluer_objects.storage.base import StorageInterface
 from bluer_objects.storage.WebDAV import WebDAVInterface
 from bluer_objects.storage.WebDAVrequest import WebDAVRequestInterface
 from bluer_objects.storage.WebDAVzip import WebDAVzipInterface
+from bluer_objects.storage.policies import DownloadPolicy
 from bluer_objects import env
 from bluer_objects.logger import logger
 
@@ -35,11 +36,13 @@ def download(
     object_name: str,
     filename: str = "",
     log: bool = True,
+    policy: DownloadPolicy = DownloadPolicy.NONE,
 ) -> bool:
     return interface.download(
         object_name=object_name,
         filename=filename,
         log=log,
+        policy=policy,
     )
 
 

{bluer_objects-6.223.1 → bluer_objects-6.231.1}/bluer_objects/storage/base.py

@@ -3,6 +3,7 @@ import glob
 from typing import Tuple, List
 
 from bluer_objects import objects
+from bluer_objects.storage.policies import DownloadPolicy
 from bluer_objects.logger import logger
 
 
@@ -18,13 +19,19 @@ class StorageInterface:
         object_name: str,
         filename: str = "",
         log: bool = True,
+        policy: DownloadPolicy = DownloadPolicy.NONE,
     ) -> bool:
         if log:
             logger.info(
-                "{}.download {}{}".format(
+                "{}.download {}{}{}".format(
                     self.__class__.__name__,
                     object_name,
                     f"/{filename}" if filename else "",
+                    (
+                        ""
+                        if policy == DownloadPolicy.NONE
+                        else " - policy:{}".format(policy.name.lower())
+                    ),
                 )
             )
 

bluer_objects-6.231.1/bluer_objects/storage/policies.py

@@ -0,0 +1,7 @@
+from enum import Enum, auto
+
+
+class DownloadPolicy(Enum):
+    NONE = auto()
+    DOESNT_EXIST = auto()
+    DIFFERENT = auto()

{bluer_objects-6.223.1 → bluer_objects-6.231.1}/bluer_objects/storage/s3.py

@@ -11,6 +11,7 @@ from bluer_objects.storage.base import StorageInterface
 from bluer_objects.env import ABCLI_OBJECT_ROOT
 from bluer_objects import env, file, path
 from bluer_objects import objects
+from bluer_objects.storage.policies import DownloadPolicy
 from bluer_objects.logger import logger
 
 
@@ -121,6 +122,7 @@ class S3Interface(StorageInterface):
         object_name: str,
         filename: str = "",
         log: bool = True,
+        policy: DownloadPolicy = DownloadPolicy.NONE,
     ) -> bool:
         if filename:
             local_path = objects.path_of(
@@ -129,6 +131,11 @@ class S3Interface(StorageInterface):
                 create=True,
             )
 
+            if policy == DownloadPolicy.DOESNT_EXIST and file.exists(local_path):
+                if log:
+                    logger.info(f"✅ {filename}")
+                return True
+
             if not path.create(file.path(local_path)):
                 return False
 
@@ -160,6 +167,7 @@ class S3Interface(StorageInterface):
                 object_name=object_name,
                 filename=filename,
                 log=log,
+                policy=policy,
             )
 
         success, list_of_files = self.ls(
@@ -174,6 +182,7 @@ class S3Interface(StorageInterface):
                 object_name=object_name,
                 filename=filename_,
                 log=log,
+                policy=policy,
             ):
                 return False
 

{bluer_objects-6.223.1 → bluer_objects-6.231.1/bluer_objects.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: bluer_objects
-Version: 6.223.1
+Version: 6.231.1
 Summary: 🌀 Object management in Bash.
 Home-page: https://github.com/kamangir/bluer-objects
 Author: Arash Abadpour (Kamangir)
@@ -64,6 +64,6 @@ pip install bluer-objects
 
 [![pylint](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/bluer-objects/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/bluer-objects.svg)](https://pypi.org/project/bluer-objects/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/bluer-objects)](https://pypistats.org/packages/bluer-objects)
 
-built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.223.1`](https://github.com/kamangir/bluer-objects).
+built by 🌀 [`bluer README`](https://github.com/kamangir/bluer-objects/tree/main/bluer_objects/README), based on 🌀 [`bluer_objects-6.231.1`](https://github.com/kamangir/bluer-objects).
 
 built by 🌀 [`blueness-3.118.1`](https://github.com/kamangir/blueness).

{bluer_objects-6.223.1 → bluer_objects-6.231.1}/bluer_objects.egg-info/SOURCES.txt

@@ -85,6 +85,7 @@ bluer_objects/.abcli/web/where_am_i.sh
 bluer_objects/README/__init__.py
 bluer_objects/README/functions.py
 bluer_objects/README/items.py
+bluer_objects/README/utils.py
 bluer_objects/file/__init__.py
 bluer_objects/file/__main__.py
 bluer_objects/file/classes.py
@@ -143,6 +144,7 @@ bluer_objects/storage/WebDAVzip.py
 bluer_objects/storage/__init__.py
 bluer_objects/storage/__main__.py
 bluer_objects/storage/base.py
+bluer_objects/storage/policies.py
 bluer_objects/storage/s3.py
 bluer_objects/testing/__init__.py
 bluer_objects/testing/__main__.py