motoko 0.0.1__tar.gz

motoko-0.0.1/.gitignore

@@ -0,0 +1,17 @@
+BD-*-runs
+.bd
+*.txt
+*.pyc
+*.xyz
+*.mtp
+*.cfg
+*.als
+examples/**/*.yaml
+!examples/alotf/**/*.mtp
+!examples/alotf/**/*.cfg
+!examples/alotf/**/*.yaml
+examples/**/databases/*
+examples/**/db/*
+!files/**/*
+*.egg-info
+tests/db

motoko-0.0.1/.gitlab-ci.yml

@@ -0,0 +1,41 @@
+
+stages:
+  - version
+  - build
+  - test
+  - code_quality
+  - deploy
+
+#-------------------------------------------------------------------------------
+test:
+  stage: test
+  image: python:3.10.9
+  script:
+    - pip install pytest pep8 pyright
+    - pip install .
+    - python3 -m pytest
+
+
+package:
+  stage: build
+  image: python:latest
+  script:
+    - pip install build
+    - python3 -m build
+  artifacts:
+    when: on_success
+    paths:
+      - dist/
+    expire_in: 30 mins
+
+python_pypi:
+  stage: deploy
+  image: python:latest
+  needs:
+    - job: package
+  script:
+    - pip install twine
+    - TWINE_PASSWORD=${PYPI_TOKEN} TWINE_USERNAME=__token__
+          python3 -m twine upload  --verbose dist/*
+  only:
+    - tags

motoko-0.0.1/.pre-commit-config.yaml

@@ -0,0 +1,25 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-json
+      - id: check-yaml
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.11.8
+    hooks:
+      - id: ruff  # Linter
+        args: [ --fix ]
+      - id: ruff-format  # Formatter
+      - id: ruff  # Check annotations and docstrings
+        args: [ --select, "D,ANN0,ANN2", --fix, --unsafe-fixes ]
+        stages: [ manual ]
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.6
+    hooks:
+    - id: codespell
+      additional_dependencies:
+        - tomli

motoko-0.0.1/PKG-INFO

@@ -0,0 +1,65 @@
+Metadata-Version: 2.4
+Name: motoko
+Version: 0.0.1
+Summary: Versatile Workflow Manager
+Author-email: Guillaume Anciaux <guillaume.anciaux@epfl.ch>, Max Ludwig Hodapp <MaxLudwig.Hodapp@mcl.at>
+License: GPL-3.0-or-later
+Project-URL: homepage, https://gitlab.com/blackdynamite/motoko
+Project-URL: repository, https://gitlab.com/blackdynamite/motoko
+Project-URL: documentation, https://gitlab.com/blackdynamite/motoko
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: ase<4.0.0,>=3.24.0
+Requires-Dist: blackdynamite>=1.1.9
+
+# Install the package in development mode with dependencies
+
+```
+pip install -e .
+```
+
+
+# Run the tests
+
+```
+pytest
+```
+
+
+# How to use Kusanagi
+
+## Command line interface (CLI)
+
+Kusanagi is provided with a few command lines.
+In a Kusanagi directory equipped with a `motoko.yaml`file,
+you can initialize the work flow with:
+
+```
+motoko create workflow_dir
+cd workflow_dir
+```
+
+where `workflow_dir` is such a directory
+
+You can then fetch for info of the current state:
+
+```
+motoko info
+```
+
+and to be more verbose
+
+```
+motoko info --verbose
+```
+
+You can finally kill every running daemon with:
+
+```
+motoko kill
+```
+
+
+## Python interface
+
+TODO

motoko-0.0.1/README.md

@@ -0,0 +1,51 @@
+# Install the package in development mode with dependencies
+
+```
+pip install -e .
+```
+
+
+# Run the tests
+
+```
+pytest
+```
+
+
+# How to use Kusanagi
+
+## Command line interface (CLI)
+
+Kusanagi is provided with a few command lines.
+In a Kusanagi directory equipped with a `motoko.yaml`file,
+you can initialize the work flow with:
+
+```
+motoko create workflow_dir
+cd workflow_dir
+```
+
+where `workflow_dir` is such a directory
+
+You can then fetch for info of the current state:
+
+```
+motoko info
+```
+
+and to be more verbose
+
+```
+motoko info --verbose
+```
+
+You can finally kill every running daemon with:
+
+```
+motoko kill
+```
+
+
+## Python interface
+
+TODO

motoko-0.0.1/motoko/bd_study.py

@@ -0,0 +1,115 @@
+"""Scripts for creating and launching BlackDynamite studies"""
+
+import os
+import subprocess
+import shutil
+from BlackDynamite.scripts import createDB, bd_zeo_server
+from BlackDynamite import bdparser
+
+################################################################
+
+
+def create_bd_study(task_manager):
+    if not os.path.exists(task_manager.study_dir):
+        raise RuntimeError(f"not existing study dir {task_manager.study_dir}")
+
+    # Stop any existing BD server and delete files
+    if os.path.exists(task_manager.study_dir):
+        subprocess.run(["canYouDigIt", "server", "stop"], cwd=task_manager.study_dir)
+        # make a bd command to do the cleaning (to be working in remate also)
+        subprocess.run(["git", "clean", "-fxd", task_manager.study_dir])
+
+    os.makedirs(task_manager.bd_dir, exist_ok=True)
+
+    cwd = os.getcwd()
+    os.chdir(task_manager.study_dir)
+    # TODO: BD seems not to allow to create servers using tcp => fix this in BD
+    createDB.main(["--truerun", "--study", task_manager.study])
+    bd_zeo_server.main(["--action", "stop"])
+    bd_zeo_server.main(
+        [
+            "--action",
+            "start",
+            "--host",
+            task_manager.host,
+            "--study",
+            task_manager.study,
+        ]
+    )
+    os.chdir(cwd)
+
+
+################################################################
+def remote_launch(study, study_dir, study_files, host, ssh, local_dir):
+    try:
+        # Remove previous database
+        cmd = f"mkdir -p {study_dir}; "
+        cmd += f"cd {study_dir}; "
+        cmd += "canYouDigIt server stop; "
+        cmd += "rm -rf * .bd"
+        ssh.cmd(cmd)
+        ssh.copy(study_files, study_dir)
+        # Initialize database
+        cmd = f"cd {study_dir}; "
+        cmd += "canYouDigIt init --truerun; "
+        cmd += "canYouDigIt server stop; "
+        cmd += f"canYouDigIt server start --host zeo://{host} --study {study}"
+        ssh.cmd(cmd)
+    except subprocess.CalledProcessError as e:
+        print(e.stderr)
+
+    # Assume that we want to access from the current work dir, create jobs/runs etc., we
+    # need all files and a dummy .db folder in it
+    if local_dir is not None:
+        local_study_dir = local_dir + "/" + study
+        os.makedirs(local_study_dir + "/.bd", exist_ok=True)
+        for f in study_files:
+            try:
+                shutil.copy(f, local_study_dir)
+            except shutil.SameFileError:
+                pass
+
+
+################################################################
+def create_bd_studies(
+    workflow,
+    validated=None,
+):
+    if validated is None or not validated:
+        validated = bdparser.validate_question(
+            "Reset and drop content of Kusanagi database", {"yes": False}, False
+        )
+    if validated is False:
+        return
+
+    for name, task_manager in workflow.task_managers.items():
+        create_bd_study(task_manager)
+
+
+################################################################
+class SSH:
+    def __init__(self, host, user, port=None, profile=""):
+        self.host = host
+        self.user = user
+        self.profile = profile
+
+        self.remote = f"{user}@{host}"
+
+        self.ssh_cmd = ["ssh"]
+        self.scp_cmd = ["scp"]
+        if port is not None:
+            self.ssh_cmd += ["-p", f"{port}"]
+            self.scp_cmd += ["-P", f"{port}"]
+
+    def cmd(self, cmd):
+        if self.profile == "":
+            result = subprocess.run(self.ssh_cmd + [self.remote, cmd])
+        else:
+            result = subprocess.run(
+                self.ssh_cmd + [self.remote, f"source {self.profile}; {cmd}"]
+            )
+        print(result.stdout)
+
+    def copy(self, files, dest):
+        result = subprocess.run(self.scp_cmd + files + [f"{self.remote}:{dest}"])
+        print(result.stdout)

motoko-0.0.1/motoko/daemon.py

@@ -0,0 +1,13 @@
+import subprocess
+import json
+import os
+
+
+def start_daemon(script, inputs):
+    cmd = ["python", script.__file__, json.dumps(inputs)]
+    print("Execute: ", *cmd)
+    return subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        cwd=os.getcwd(),
+    )

motoko-0.0.1/motoko/scripts/create_studies.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+import os
+from motoko.workflow import Workflow
+
+command = "create"
+command_help = "Create the sub studies of the workflow"
+
+
+def populate_arg_parser(parser):
+    parser.add_argument(
+        "--validated",
+        action="store_true",
+        help="Answer yes to validation questions",
+    )
+    parser.add_argument(
+        "directory",
+        type=str,
+        default="./",
+        help="Directory where the description of the workflow is provided",
+    )
+
+
+def main(args):
+    fullpath = os.path.join(args.directory, "motoko.yaml")
+
+    wf = Workflow(fullpath)
+    wf.create(args.validated)
+    return wf

motoko-0.0.1/motoko/scripts/info.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+
+import subprocess
+from motoko.workflow import Workflow
+
+command = "info"
+command_help = "Get info from sub-studies"
+
+
+def populate_arg_parser(parser):
+    parser.add_argument("--verbose", action="store_true", help="show verbose details")
+
+
+def main(args):
+    fullpath = "motoko.yaml"
+    wf = Workflow(fullpath)
+
+    for name, task_manager in wf.task_managers.items():
+        print("*" * 30)
+        print(f"TaskManager: {name}")
+        print("*" * 30)
+        if not args.verbose:
+            subprocess.call("canYouDigIt info", cwd=task_manager.study_dir, shell=True)
+
+        else:
+            subprocess.call(
+                "canYouDigIt jobs info", cwd=task_manager.study_dir, shell=True
+            )
+            subprocess.call(
+                "canYouDigIt runs info", cwd=task_manager.study_dir, shell=True
+            )
+            subprocess.call(
+                "canYouDigIt launch_daemon --status",
+                cwd=task_manager.study_dir,
+                shell=True,
+            )

motoko-0.0.1/motoko/scripts/kill.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+import subprocess
+
+from motoko.workflow import Workflow
+
+command = "kill"
+command_help = "Kill all the running daemons"
+
+
+def populate_arg_parser(parser):
+    parser.add_argument("--verbose", action="store_true", help="show verbose details")
+
+
+def main(args):
+    fullpath = "motoko.yaml"
+
+    wf = Workflow(fullpath)
+
+    for name, task_manager in wf.task_managers.items():
+        print(f"Kill daemons in study: {task_manager.study}")
+        cmds = ["canYouDigIt launch_daemon --stop", "canYouDigIt server stop"]
+        for cmd in cmds:
+            if args.verbose:
+                print(f"({task_manager.study}) {cmd}")
+            subprocess.call(cmd, cwd=task_manager.study_dir, shell=True)

motoko-0.0.1/motoko/scripts/launcher.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+
+from motoko.workflow import Workflow
+
+command = "launcher"
+command_help = "Spawn the launcher daemon"
+
+
+def populate_arg_parser(parser):
+    pass
+
+
+def main(args):
+    fullpath = "motoko.yaml"
+    wf = Workflow(fullpath)
+    wf.start_launcher_daemons()
+
+    # for name, task_manager in wf.task_managers.items():
+    #     subprocess.call(
+    #         "canYouDigIt launch_daemon --start --detach",
+    #         cwd=task_manager.study_dir,
+    #         shell=True,
+    #     )

motoko-0.0.1/motoko/scripts/main.py

@@ -0,0 +1,37 @@
+import argparse
+
+from motoko.scripts import create_studies, info, kill, launcher, orchestrator
+
+commands = [create_studies, info, kill, launcher, orchestrator]
+
+
+def main() -> None:
+    """Entry point for the command line interface."""
+    args = parse_args()
+
+    for command in commands:
+        if args.command == command.command:
+            command.main(args)
+            break
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.prog = "motoko"
+
+    # Create subparsers for each command
+    command_parsers = parser.add_subparsers(dest="command", help="command to run")
+    command_parsers.required = True
+
+    for command in commands:
+        command_parser = command_parsers.add_parser(
+            command.command, help=command.command_help
+        )
+        command.populate_arg_parser(command_parser)
+
+    args = parser.parse_args()
+    return args
+
+
+if __name__ == "__main__":
+    main()

motoko-0.0.1/motoko/scripts/orchestrator.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import importlib.util
+from BlackDynamite.scripts.launch_daemon import find_pids
+
+from motoko.workflow import Workflow
+
+command = "orchestrator"
+command_help = "Start the workflow"
+
+
+def populate_arg_parser(parser):
+    subparsers = parser.add_subparsers(dest="what")
+    parser_start = subparsers.add_parser("start", help="start orchestrator daemon")
+
+    parser_start.add_argument(
+        "--detach",
+        "-d",
+        action="store_true",
+        help="For starting: run in detach/daemon mode with Zdaemon manager",
+    )
+    parser_start.add_argument(
+        "--wait",
+        type=str,
+        default=5,
+        help="Waiting time between state checks in seconds",
+    )
+
+    # Add arguments specific to workflow
+
+    fullpath = "motoko.yaml"
+    wf = Workflow(fullpath)
+
+    fname, _ = wf.orchestrator_script.split(".")
+    file_path = os.path.join(wf.directory, fname + ".py")
+    module_name = "orchestrator"
+    print(f"loading {file_path}")
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)
+    parser_function = getattr(module, "populate_arg_parser")
+    parser_function(parser_start)
+
+    # parser_stop = subparsers.add_parser("stop", help="stop orchestrator daemon")
+    # parser_stat = subparsers.add_parser(
+    #    "status", help="get status of detached orchestrator daemon"
+    # )
+
+
+def main(args):
+    fullpath = "motoko.yaml"
+    wf = Workflow(fullpath)
+    wf_root_dir = wf.directory
+    wf_conf_dir = os.path.join(wf_root_dir, ".wf_conf")
+
+    conf_fname = os.path.join(wf_conf_dir, "orch_daemon.conf")
+
+    if args.what == "start" and not args.detach:
+        params = vars(args)
+        wf.execute(**params)
+        sys.exit(0)
+
+    elif args.what == "start" and args.detach:
+        exclude = ["--detach", "-d"]
+        argv = sys.argv[3:]
+        clargs = " ".join([a for a in argv if a not in exclude])
+
+        prog = f"motoko {command} start {clargs}"
+
+        zdaemon_conf = f"""
+<runner>
+ program {prog}
+ socket-name {wf_conf_dir}/orch_daemon.socket
+ transcript {wf_conf_dir}/orch.log
+ exit-codes 0
+</runner>
+"""
+
+        os.makedirs(wf_conf_dir, exist_ok=True)
+        with open(conf_fname, "w") as f:
+            f.write(zdaemon_conf)
+
+        os.system(f"zdaemon -C {conf_fname} start")
+        os.system(f"zdaemon -C {conf_fname} status")
+
+    elif args.what == "status":
+        print("ZDaemon status:")
+        os.system(f"zdaemon -C {conf_fname} status")
+
+        pids = find_pids(wf_root_dir)
+        print(f"Running orchestrator pids: {pids}")
+
+    elif args.what == "stop":
+        os.system(f"zdaemon -C {conf_fname} stop")
+        pids = find_pids(wf_root_dir)
+        if pids:
+            print(f"Killing: {pids}")
+            pids = [str(e) for e in pids]
+            os.system(f"kill -9 {' '.join(pids)}")
+
+        # shutil.rmtree(wf_conf_dir)