plexus-python-pulse-demeter 1.1.126__py3-none-any.whl

plexus/pulse/demeter/tools/main.py

@@ -0,0 +1,306 @@
+# This magic comment is used by `argcomplete` to identify that this script requires
+# argument completion support when using shell autocompletion features.
+
+# PYTHON_ARGCOMPLETE_OK
+
+# `argcomplete` is a Python library that provides support for command-line argument autocompletion in
+# shells like bash and zsh.
+# By including the `PYTHON_ARGCOMPLETE_OK` comment at the top of the script, it signals to `argcomplete` that
+# this script is compatible with its autocompletion features. When users set up `argcomplete` in their shell,
+# they can enjoy autocompletion for the arguments defined in this script, enhancing the user experience when
+# running the command-line tool.
+
+# !!! IMPORTANT: DO NOT import heavy dependencies at the top level of this script and the initialization
+# code of the classes derived from `CommandTool`.
+#
+# `argcomplete` works by analyzing the argument parser defined in the script and generating the necessary
+# autocompletion scripts for the shell. Thus, to reduce the cost of loading this script when autocompletion
+# is triggered, we should avoid importing heavy dependencies at the top level of the script. Instead, we can
+# import them within the functions or methods where they are actually used. This way, when `argcomplete` loads
+# the script for autocompletion, it won't incur the overhead of importing unnecessary modules, leading to
+# faster autocompletion performance.
+
+import abc
+import argparse
+import logging
+
+
+class CommandTool(abc.ABC):
+    @classmethod
+    @abc.abstractmethod
+    def commands(cls) -> list[str]:
+        pass
+
+    @classmethod
+    @abc.abstractmethod
+    def desc(cls) -> str:
+        pass
+
+    @classmethod
+    @abc.abstractmethod
+    def make_parser(cls, parser: argparse.ArgumentParser) -> None:
+        pass
+
+    @abc.abstractmethod
+    def run(self, args: argparse.Namespace):
+        pass
+
+
+class TagCacheUploadCommandTool(CommandTool):
+
+    @classmethod
+    def commands(cls) -> list[str]:
+        return ["tag-cache", "upload"]
+
+    @classmethod
+    def desc(cls) -> str:
+        return "Upload the tag cache SQLite DB file to the Demeter API server for ingestion."
+
+    @classmethod
+    def make_parser(cls, parser: argparse.ArgumentParser) -> None:
+        parser.add_argument("tag_cache_file", type=str, help="Path to the tag cache SQLite DB file.")
+        parser.add_argument("--api-url",
+                            type=str,
+                            default="http://localhost:8000",
+                            help="URL of the Demeter API server.")
+
+    def run(self, args: argparse.Namespace) -> int:
+        import shutil
+        import tempfile
+        from pathlib import Path
+
+        import requests
+        from plexus.common.utils.pathutils import make_path
+        from plexus.pulse.common.utils.tagutils import TagCache
+        from plexus.pulse.common.utils.tagutils import predefined_tagsets
+
+        from plexus.pulse.demeter.app.api.tagging_endpoints import router as tagging_router
+
+        tag_cache_file_path = make_path(args.tag_cache_file)
+        if not tag_cache_file_path.is_file():
+            raise ValueError(f"Tag cache file <'{args.tag_cache_file}'> does not exist or is not a file")
+
+        api_url = args.api_url.rstrip("/")
+
+        logging.info(f"Uploading tag cache from file <'%s'> to API at <'%s'>", tag_cache_file_path, api_url)
+
+        with tempfile.NamedTemporaryFile(suffix=tag_cache_file_path.suffix, delete=False) as temp_file:
+            temp_file_path = Path(temp_file.name)
+
+        try:
+
+            shutil.copy2(tag_cache_file_path, temp_file_path)
+            upload_file_path = temp_file_path
+
+            tag_cache = TagCache(file_path=upload_file_path)
+            if args.prune_undefined:
+                tag_cache.remove_records(tagsets=list(predefined_tagsets().values()), tagset_inverted=True)
+
+            with open(upload_file_path, "rb") as fh:
+                response = requests.post(f"{api_url}/{tagging_router.prefix}/upload_tag_cache/",
+                                         files={"tag_cache_file": fh})
+                if response.status_code != 200:
+                    logging.error("Failed to upload tag cache. Status code <%s>, Response:\n%s",
+                                  response.status_code,
+                                  response.text)
+                    return -1
+
+            logging.info(
+                "Tag cache uploaded successfully to API at <'%s'> with <%d> tag targets and <%d> tags records were uploaded.",
+                api_url,
+                len(tag_cache.query_targets()),
+                len(list(tag_cache.iter_record_and_targets())),
+            )
+
+            return 0
+
+        finally:
+            if temp_file_path is not None and temp_file_path.exists():
+                temp_file_path.unlink()
+
+
+class TagCacheValidateCommandTool(CommandTool):
+
+    @classmethod
+    def commands(cls) -> list[str]:
+        return ["tag-cache", "validate"]
+
+    @classmethod
+    def desc(cls) -> str:
+        return "Validate the tag cache SQLite DB file and print its contents in a human-readable format."
+
+    @classmethod
+    def make_parser(cls, parser: argparse.ArgumentParser) -> None:
+        parser.add_argument("tag_cache_file", type=str, help="Path to the tag cache SQLite DB file.")
+
+    def run(self, args: argparse.Namespace) -> int:
+        import uuid as py_uuid
+
+        import rich.console
+        from plexus.common.utils.pathutils import make_path
+        from plexus.pulse.common.utils.ormutils import print_model_results
+        from plexus.pulse.common.utils.tagutils import TagCache
+        from plexus.pulse.common.utils.tagutils import predefined_tagsets
+        from plexus.pulse.common.utils.tagutils import versioned_identifier
+
+        from plexus.pulse.demeter.app.models.registry.tagging import DataTagRecord, DataTagTarget
+
+        console = rich.console.Console()
+
+        tag_cache_file_path = make_path(args.tag_cache_file)
+        if not tag_cache_file_path.is_file():
+            raise ValueError(f"Tag cache file <'{args.tag_cache_file}'> does not exist or is not a file")
+
+        logging.info(f"Loading tag cache from file <'%s'>", tag_cache_file_path)
+
+        tag_cache = TagCache(file_path=tag_cache_file_path)
+
+        errors_count = 0
+
+        for target in tag_cache.query_targets():
+            try:
+                DataTagTarget(
+                    uuid=py_uuid.uuid4(),
+                    tagger_name=target.tagger_name,
+                    tagger_version=target.tagger_version,
+                    vehicle_name=target.vehicle_name,
+                    begin_dt=target.begin_dt,
+                    end_dt=target.end_dt,
+                )
+            except Exception as e:
+                errors_count += 1
+                console.print(
+                    f"[red]Error:[/red] Failed to parse tag target [bold]{target}[/bold]. Exception: {str(e)}")
+
+            for record in tag_cache.with_target(target.identifier).iter_records():
+                versioned_namespace = versioned_identifier(record.tagset_namespace, record.tagset_version)
+                if versioned_namespace not in predefined_tagsets():
+                    errors_count += 1
+                    console.print(
+                        f"[red]Error:[/red] Tagset [bold]{versioned_namespace}[/bold] is not predefined.")
+
+                try:
+                    DataTagRecord(
+                        target_uuid=py_uuid.uuid4(),
+                        vehicle_name=target.vehicle_name,
+                        begin_dt=record.begin_dt,
+                        end_dt=record.end_dt,
+                        tagset_namespace=record.tagset_namespace,
+                        tagset_version=record.tagset_version or
+                                       (
+                                           predefined_tagsets().get(record.tagset_namespace).version
+                                           if record.tagset_namespace in predefined_tagsets()
+                                           else None
+                                       ),
+                        tag=record.tag,
+                        props=record.props,
+                    )
+                except Exception as e:
+                    errors_count += 1
+                    console.print(
+                        f"[red]Error:[/red] Failed to parse tag record [bold]{record}[/bold]. Exception: {str(e)}")
+
+        logging.info("Tag cache validation completed with <%d> error(s).", errors_count)
+
+        print_model_results(tag_cache.query_targets())
+        print_model_results(tag_cache.iter_record_and_targets())
+
+        return 0 if errors_count == 0 else -1
+
+
+class DatasetValidateCommandTool(CommandTool):
+
+    @classmethod
+    def commands(cls) -> list[str]:
+        return ["dataset", "validate"]
+
+    @classmethod
+    def desc(cls) -> str:
+        return "Validate standalone samples from a specified dataset source directory."
+
+    @classmethod
+    def make_parser(cls, parser: argparse.ArgumentParser) -> None:
+        parser.add_argument("dataset_root", type=str, help="Root directory of the dataset.")
+        parser.add_argument("--api-url",
+                            type=str,
+                            default="http://localhost:8000",
+                            help="URL of the Demeter API server.")
+
+    def run(self, args: argparse.Namespace) -> int:
+        from pathlib import Path
+
+        import requests
+        from cloudpathlib import AnyPath, S3Path
+
+        from plexus.pulse.demeter.app.api.dataset_internal_endpoints import router as internal_router
+        from plexus.pulse.demeter.app.core.vault import make_progressed_client
+        from plexus.pulse.demeter.app.models.registry.dataset import SampleLayout
+        from plexus.pulse.demeter.utils.datautils import collect_standalone_dataset_samples
+
+        dataset_root = args.dataset_root
+
+        logging.info(f"Starting sample collection from dataset root <'%s'>.", dataset_root)
+
+        api_url = args.api_url.rstrip("/")
+
+        logging.info(f"Fetching sample layouts from API at <'%s'>.", api_url)
+
+        response = requests.get(f"{api_url}/{internal_router.prefix}/layouts")
+        response.raise_for_status()
+
+        layouts = [SampleLayout.model_validate(item) for item in response.json()]
+
+        logging.info(f"Retrieved <%d> sample layouts from API.", len(layouts))
+
+        if isinstance(AnyPath(dataset_root), S3Path):
+            with make_progressed_client() as s3_client:
+                dataset_root = s3_client.CloudPath(dataset_root)
+                _, _, errors = collect_standalone_dataset_samples(dataset_root, layouts)
+        else:
+            _, _, errors = collect_standalone_dataset_samples(Path(dataset_root).absolute(), layouts)
+
+        if len(errors) > 0:
+            print("Validation errors found:")
+            for error in errors:
+                print(f"- {error}")
+            return 1
+
+        logging.info("Sample collection completed successfully with no errors.")
+
+        return 0
+
+
+def main():
+    import argcomplete
+    from plexus.common.utils.argutils import ParserTree
+
+    tools: list[type[CommandTool]] = [
+        TagCacheUploadCommandTool,
+        TagCacheValidateCommandTool,
+        DatasetValidateCommandTool,
+    ]
+
+    parser_tree = ParserTree(argparse.ArgumentParser(description="Pulse Demeter Python tools"))
+    for tool in tools:
+        parser = parser_tree.add_subcommand_parser(tool.commands(), help=tool.desc())
+        tool.make_parser(parser)
+
+    argcomplete.autocomplete(parser_tree.root_node.parser)
+
+    from plexus.pulse.demeter.utils.config import config
+
+    logging.basicConfig(level=config().get("plexus", "logging.level"),
+                        format=config().get("plexus", "logging.format"))
+
+    commands, args = parser_tree.parse_args()
+    for tool in tools:
+        if commands == tool.commands():
+            try:
+                SystemExit(tool().run(args))
+            except Exception as e:
+                logging.exception(f"An error occurred while executing the command '%s': %s", " ".join(commands), str(e))
+                SystemExit(-1)
+
+
+if __name__ == "__main__":
+    main()

plexus/pulse/demeter/utils/config.py

@@ -0,0 +1,25 @@
+from plexus.common.utils.config import Config
+from plexus.common.utils.funcutils import singleton
+from plexus.common.utils.pathutils import make_path
+
+
+@singleton
+def config() -> Config:
+    default_items: list[tuple[str, str, str]] = [
+        ("pulse.apps.database", "host", ""),
+        ("pulse.apps.database", "port", "5432"),
+        ("pulse.apps.database", "username", ""),
+        ("pulse.apps.database", "password", ""),
+        ("pulse.apps.database", "database", "pulse-apps"),
+        ("pulse.apps.vault", "access_key_id", ""),
+        ("pulse.apps.vault", "secret_access_key", ""),
+        ("pulse.apps.vault", "endpoint_url", "http://localhost:9000"),
+    ]
+
+    from plexus.pulse.common.utils.config import config as pulse_config
+
+    instance = pulse_config(make_path("~/.plexus-pulse.cfg", expand=True, normalize=True, absolute=True))
+    instance.restore()
+    instance.update(default_items, overwrite=False)
+
+    return instance