bluer-objects 6.104.1__py3-none-any.whl → 6.377.1__py3-none-any.whl

bluer_objects/mlflow/lock/functions.py

@@ -0,0 +1,121 @@
+import time
+
+from blueness import module
+from bluer_options import string
+
+from bluer_objects import NAME, env
+from bluer_objects.mlflow.tags import get_tags, set_tags
+from bluer_objects.logger import logger
+
+
+NAME = module.name(__file__, NAME)
+
+
+def lock(
+    object_name: str,
+    lock_name: str = "lock",
+    timeout: int = -1,
+    verbose: bool = True,
+) -> bool:
+    logger.info(
+        "{}.lock: {}.{}{}".format(
+            NAME,
+            object_name,
+            lock_name,
+            "" if timeout == -1 else " @ {}".format(string.pretty_duration(timeout)),
+        )
+    )
+
+    start_time = time.time()
+    while True:
+        if timeout > 0 and time.time() - start_time > timeout:
+            if verbose:
+                logger.warning(
+                    "{}.lock: {}.{} timeout.".format(
+                        NAME,
+                        object_name,
+                        lock_name,
+                    )
+                )
+            return False
+
+        success, list_of_tags = get_tags(object_name=object_name)
+        if not success:
+            return False
+
+        lock_value = list_of_tags.get(lock_name, "")
+        if lock_value:
+            if verbose:
+                logger.warning(
+                    "{}.lock: {}.{} is locked by {}.".format(
+                        NAME,
+                        object_name,
+                        lock_name,
+                        lock_value,
+                    )
+                )
+            time.sleep(env.MLFLOW_LOCK_WAIT_FOR_CLEARANCE)
+            continue
+
+        lock_value = string.random()
+        if not set_tags(
+            object_name=object_name,
+            tags={lock_name: lock_value},
+            log=verbose,
+            icon="🔒",
+        ):
+            return False
+
+        time.sleep(env.MLFLOW_LOCK_WAIT_FOR_EXCLUSIVITY)
+
+        success, list_of_tags = get_tags(object_name=object_name)
+        if not success:
+            return False
+
+        lock_value_read = list_of_tags.get(lock_name, "")
+        if lock_value_read != lock_value:
+            if verbose:
+                logger.warning(
+                    "{}.lock: {}.{} is relocked by {} != {}.".format(
+                        NAME,
+                        object_name,
+                        lock_name,
+                        lock_value_read,
+                        lock_value,
+                    )
+                )
+            time.sleep(env.MLFLOW_LOCK_WAIT_FOR_CLEARANCE)
+            continue
+
+        break
+
+    logger.info(
+        "{}.lock: {}.{} is locked by {}.".format(
+            NAME,
+            object_name,
+            lock_name,
+            lock_value,
+        )
+    )
+    return True
+
+
+def unlock(
+    object_name: str,
+    lock_name: str = "lock",
+    verbose: bool = True,
+) -> bool:
+    logger.info(
+        "{}.unlock: {}.{}".format(
+            NAME,
+            object_name,
+            lock_name,
+        )
+    )
+
+    return set_tags(
+        object_name=object_name,
+        tags={lock_name: ""},
+        log=verbose,
+        icon="🔒",
+    )

bluer_objects/mlflow/logging.py

@@ -6,7 +6,7 @@ import mlflow
 from blueness import module
 from bluer_options.logger import crash_report
 
-from bluer_objects import file, objects, NAME
+from bluer_objects import file, objects, NAME, env
 from bluer_objects.mlflow.runs import start_run, end_run
 from bluer_objects.logger import logger
 
@@ -22,25 +22,28 @@ def log_artifacts(
 
     object_path = objects.object_path(object_name, create=True)
 
-    try:
-        mlflow.log_artifacts(object_path)
+    if env.MLFLOW_LOG_ARTIFACTS:
+        try:
+            mlflow.log_artifacts(object_path)
 
-        logger.info("⬆️  {}".format(object_name))
+            logger.info("⬆️  {}".format(object_name))
 
-        # https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.register_model
-        # https://stackoverflow.com/a/71447758/17619982
-        if model_name:
-            mv = mlflow.register_model(
-                "runs:/{}".format(mlflow.active_run().info.run_id),
-                model_name,
-                await_registration_for=0,
-            )
+            # https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.register_model
+            # https://stackoverflow.com/a/71447758/17619982
+            if model_name:
+                mv = mlflow.register_model(
+                    "runs:/{}".format(mlflow.active_run().info.run_id),
+                    model_name,
+                    await_registration_for=0,
+                )
 
-            logger.info("*️⃣  {} -> {}.{}".format(object_name, mv.name, mv.version))
+                logger.info("*️⃣  {} -> {}.{}".format(object_name, mv.name, mv.version))
 
-    except:
-        crash_report(f"{NAME}.log_artifacts({object_name})")
-        return False
+        except:
+            crash_report(f"{NAME}.log_artifacts({object_name})")
+            return False
+    else:
+        logger.info("skipped log artifacts.")
 
     return end_run(object_name)
 
@@ -51,31 +54,34 @@ def log_run(object_name: str) -> bool:
 
     object_path = objects.object_path(object_name, create=True)
 
-    counts: Dict[str, int] = {}
-    skipped_count = 0
-    for extension in "dot,gif,jpeg,jpg,json,png,sh,xml,yaml".split(","):
-        for filename in glob.glob(
-            os.path.join(object_path, f"*.{extension}"),
-        ):
-            filename_name = file.name(filename)
-
-            counts[len(filename_name)] = counts.get(len(filename_name), 0) + 1
-
-            if any(
-                [
-                    file.size(filename) > 10 * 1024 * 1024,
-                    filename_name.startswith("thumbnail"),
-                    counts[len(filename_name)] > 20,
-                ]
+    if env.MLFLOW_LOG_ARTIFACTS:
+        counts: Dict[str, int] = {}
+        skipped_count = 0
+        for extension in "dot,gif,jpeg,jpg,json,png,sh,xml,yaml".split(","):
+            for filename in glob.glob(
+                os.path.join(object_path, f"*.{extension}"),
             ):
-                logger.info(f"skipping {filename}")
-                skipped_count += 1
-                continue
-
-            mlflow.log_artifact(filename)
-            logger.info(f"⬆️  {filename}")
-
-    if skipped_count:
-        logger.info(f"skipped {skipped_count:,} file(s).")
+                filename_name = file.name(filename)
+
+                counts[len(filename_name)] = counts.get(len(filename_name), 0) + 1
+
+                if any(
+                    [
+                        file.size(filename) > 10 * 1024 * 1024,
+                        filename_name.startswith("thumbnail"),
+                        counts[len(filename_name)] > 20,
+                    ]
+                ):
+                    logger.info(f"skipping {filename}")
+                    skipped_count += 1
+                    continue
+
+                mlflow.log_artifact(filename)
+                logger.info(f"⬆️  {filename}")
+
+        if skipped_count:
+            logger.info(f"skipped {skipped_count:,} file(s).")
+    else:
+        logger.info("skipped log artifacts.")
 
     return end_run(object_name)

bluer_objects/pdf/__init__.py

@@ -0,0 +1 @@
+

bluer_objects/pdf/__main__.py

@@ -0,0 +1,78 @@
+import argparse
+
+from blueness import module
+from blueness.argparse.generic import sys_exit
+
+from bluer_objects import NAME
+from bluer_objects.pdf.convert.batch import batch as batch_convert
+from bluer_objects.pdf.convert.convert import convert
+from bluer_objects.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+parser = argparse.ArgumentParser(NAME)
+parser.add_argument(
+    "task",
+    type=str,
+    help="convert",
+)
+parser.add_argument(
+    "--path_prefix",
+    type=str,
+)
+parser.add_argument(
+    "--object_name",
+    type=str,
+)
+parser.add_argument(
+    "--suffixes",
+    type=str,
+)
+parser.add_argument(
+    "--combine",
+    type=int,
+    default=0,
+    help="0 | 1",
+)
+
+parser.add_argument(
+    "--count",
+    type=int,
+    default=-1,
+    help="-1: all",
+)
+parser.add_argument(
+    "--list_missing",
+    type=int,
+    default=1,
+    help="0 | 1",
+)
+parser.add_argument(
+    "--use_metadata",
+    type=int,
+    default=0,
+    help="0 | 1",
+)
+args = parser.parse_args()
+
+success = False
+if args.task == "convert":
+    if args.use_metadata == 1:
+        success = batch_convert(
+            object_name=args.object_name,
+            combine=args.combine == 1,
+            count=args.count,
+            list_missing=args.list_missing == 1,
+        )
+    else:
+        success = convert(
+            path_prefix=args.path_prefix,
+            list_of_suffixes=args.suffixes.split(","),
+            object_name=args.object_name,
+            combine=args.combine == 1,
+            count=args.count,
+        )
+else:
+    success = None
+
+sys_exit(logger, NAME, args.task, success)

bluer_objects/pdf/convert/batch.py

@@ -0,0 +1,54 @@
+from typing import List
+
+from blueness import module
+
+from bluer_objects import NAME
+from bluer_objects.metadata import get_from_object
+from bluer_objects.pdf.convert.convert import convert
+from bluer_objects.pdf.convert.missing import list_missing_docs
+from bluer_objects.env import abcli_path_git
+from bluer_objects.logger import logger
+
+
+NAME = module.name(__file__, NAME)
+
+
+def batch(
+    object_name: str,
+    combine: bool,
+    count: int = -1,
+    list_missing: bool = True,
+) -> bool:
+    logger.info(
+        "{}.batch: {}{}{}{}".format(
+            NAME,
+            object_name,
+            " + combine" if combine else "",
+            "" if count == -1 else f" {count} files",
+            " + list missing" if list_missing else "",
+        )
+    )
+
+    list_of_suffixes: List[str] = get_from_object(
+        object_name,
+        "pdf",
+        [],
+    )
+
+    if not convert(
+        path_prefix=abcli_path_git,
+        list_of_suffixes=list_of_suffixes,
+        object_name=object_name,
+        combine=combine,
+        count=count,
+        incremental=False,
+    ):
+        return False
+
+    if list_missing:
+        list_missing_docs(
+            object_name=object_name,
+            list_of_suffixes=list_of_suffixes,
+        )
+
+    return True

bluer_objects/pdf/convert/combination.py

@@ -0,0 +1,32 @@
+from typing import List
+from PyPDF2 import PdfMerger
+
+from bluer_options.logger import crash_report
+
+from bluer_objects import objects
+from bluer_objects.logger import logger
+
+
+def combine_pdfs(
+    list_of_pdfs: List[str],
+    object_name: str,
+) -> bool:
+    logger.info(f"combining {len(list_of_pdfs)} pdf(s)...")
+    combined_filename = objects.path_of(
+        filename="release.pdf",
+        object_name=object_name,
+    )
+
+    try:
+        merger = PdfMerger()
+        for filename in list_of_pdfs:
+            merger.append(filename)
+
+        merger.write(combined_filename)
+        merger.close()
+    except Exception as e:
+        crash_report(e)
+        return False
+
+    logger.info(f"-> {combined_filename}")
+    return True

bluer_objects/pdf/convert/convert.py

@@ -0,0 +1,111 @@
+from tqdm import tqdm
+from typing import List
+import os
+from PIL import Image
+
+from blueness import module
+
+from bluer_objects import NAME
+from bluer_objects import file, path
+from bluer_objects.metadata import get_from_object, post_to_object
+from bluer_objects.pdf.convert.combination import combine_pdfs
+from bluer_objects.pdf.convert.image import convert_image
+from bluer_objects.pdf.convert.md import convert_md
+from bluer_objects.pdf.convert.pdf import convert_pdf
+from bluer_objects.logger import logger
+
+
+NAME = module.name(__file__, NAME)
+
+
+def convert(
+    path_prefix: str,
+    list_of_suffixes: List[str],
+    object_name: str,
+    combine: bool,
+    count: int = -1,
+    incremental: bool = True,
+) -> bool:
+    logger.info(f"path_prefix: {path_prefix}")
+
+    list_of_pdfs: List[str] = (
+        get_from_object(
+            object_name,
+            "list_of_pdfs",
+            [],
+        )
+        if incremental
+        else []
+    )
+    if incremental:
+        logger.info(f"found {len(list_of_pdfs)} pdf(s)...")
+
+    list_of_pdfs_len_target = -1 if count == -1 else len(list_of_pdfs) + count
+    for suffix in tqdm(list_of_suffixes):
+        if (
+            list_of_pdfs_len_target != -1
+            and len(list_of_pdfs) >= list_of_pdfs_len_target
+        ):
+            logger.info(f"max count {count}, stopping.")
+            break
+
+        logger.info(
+            "{}.convert {} -> {}".format(
+                NAME,
+                suffix,
+                object_name,
+            )
+        )
+
+        source_filename = os.path.join(path_prefix, suffix)
+        if path.exists(source_filename):
+            source_filename = os.path.join(source_filename, "README.md")
+            suffix = os.path.join(suffix, "README.md")
+
+        if source_filename.endswith(".md"):
+            if not convert_md(
+                source_filename,
+                suffix,
+                object_name,
+                list_of_pdfs,
+            ):
+                return False
+        elif file.extension(source_filename) == "pdf":
+            if not convert_pdf(
+                source_filename,
+                suffix,
+                object_name,
+                list_of_pdfs,
+            ):
+                return False
+        elif file.extension(source_filename) in [
+            extension.split(".", 1)[1]
+            for extension in Image.registered_extensions().keys()
+        ]:
+            if not convert_image(
+                source_filename,
+                suffix,
+                object_name,
+                list_of_pdfs,
+            ):
+                return False
+        else:
+            logger.error(f"{source_filename}: cannot convert to pdf.")
+            return False
+
+    if incremental:
+        logger.info(f"{len(list_of_pdfs)} pdf(s) so far ...")
+        if not post_to_object(
+            object_name,
+            "list_of_pdfs",
+            list_of_pdfs,
+        ):
+            return False
+
+    if combine:
+        return combine_pdfs(
+            list_of_pdfs,
+            object_name,
+        )
+
+    return True

bluer_objects/pdf/convert/image.py

@@ -0,0 +1,53 @@
+from PIL import Image
+
+from bluer_options.logger import crash_report
+
+from bluer_objects import file, objects, path
+from bluer_objects.env import abcli_path_git
+from bluer_objects.logger import logger
+
+
+def convert_image(
+    source_filename: str,
+    suffix: str,
+    object_name: str,
+    list_of_pdfs: list[str],
+) -> bool:
+    logger.info("🌠 image found!")
+    filename_pdf = file.add_extension(
+        objects.path_of(
+            filename="docs/{}".format(
+                (
+                    suffix.split(abcli_path_git, 1)[1]
+                    if abcli_path_git in suffix
+                    else suffix
+                ),
+            ),
+            object_name=object_name,
+        ),
+        "pdf",
+    )
+
+    if filename_pdf not in list_of_pdfs:
+        list_of_pdfs.append(filename_pdf)
+
+    if file.exists(filename_pdf):
+        logger.info(f"✅ {filename_pdf}")
+        return True
+
+    if not path.create(
+        file.path(filename_pdf),
+        log=True,
+    ):
+        return False
+
+    try:
+        image = Image.open(source_filename)
+        image = image.convert("RGB")
+        image.save(filename_pdf)
+    except Exception as e:
+        crash_report(e)
+        return False
+
+    logger.info(f"-> {filename_pdf}")
+    return True

bluer_objects/pdf/convert/md.py

@@ -0,0 +1,97 @@
+import pypandoc
+import subprocess
+import os
+
+from bluer_options.logger import crash_report
+
+from bluer_objects import file, objects, path
+from bluer_objects.env import abcli_path_git
+from bluer_objects.logger import logger
+
+css = """
+    <style>
+        body { font-family: sans-serif; margin: 2cm; }
+        img { max-width: 100%; height: auto; }
+        table { width: 100%; border-collapse: collapse; word-break: break-word; }
+        th, td { border: 1px solid #ccc; padding: 4px; vertical-align: top; }
+        code, pre { white-space: pre-wrap; }
+    </style>
+    """
+
+
+def convert_md(
+    source_filename: str,
+    suffix: str,
+    object_name: str,
+    list_of_pdfs: list[str],
+) -> bool:
+    filename_html = file.add_extension(
+        objects.path_of(
+            filename="docs/{}".format(
+                (
+                    suffix.split(abcli_path_git, 1)[1]
+                    if abcli_path_git in suffix
+                    else suffix
+                ),
+            ),
+            object_name=object_name,
+        ),
+        "html",
+    )
+    filename_pdf = file.add_extension(
+        filename_html,
+        "pdf",
+    )
+
+    if filename_pdf not in list_of_pdfs:
+        list_of_pdfs.append(filename_pdf)
+
+    if file.exists(filename_pdf):
+        logger.info(f"✅ {filename_pdf}")
+        return True
+
+    logger.info(f"{source_filename} -> {filename_pdf}")
+
+    try:
+        with open(source_filename, "r", encoding="utf-8") as f:
+            markdown_text = f.read()
+
+        html_text = pypandoc.convert_text(
+            markdown_text,
+            "html",
+            format="md",
+        )
+
+        html_text = (
+            f"<!DOCTYPE html><html><head>{css}</head><body>{html_text}</body></html>"
+        )
+
+        if not path.create(
+            file.path(filename_html),
+            log=True,
+        ):
+            return (False,)
+
+        with open(
+            filename_html,
+            "w",
+            encoding="utf-8",
+        ) as f:
+            f.write(html_text)
+
+        subprocess.run(
+            [
+                "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
+                "--headless",
+                "--disable-gpu",
+                "--no-margins",
+                f"--print-to-pdf={filename_pdf}",
+                os.path.abspath(filename_html),
+            ],
+            check=True,
+        )
+    except Exception as e:
+        crash_report(e)
+        return False
+
+    return True