formed 0.0.1__py3-none-any.whl

formed/__init__.py

@@ -0,0 +1,3 @@
+from importlib.metadata import version
+
+__version__ = version("formed")

formed/__main__.py

@@ -0,0 +1,20 @@
+import logging
+import os
+
+from formed.commands import main
+
+if os.environ.get("FORMED_DEBUG"):
+    LEVEL = logging.DEBUG
+else:
+    level_name = os.environ.get("FORMED_LOG_LEVEL", "INFO")
+    LEVEL = logging._nameToLevel.get(level_name, logging.INFO)
+
+logging.basicConfig(format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", level=LEVEL)
+
+
+def run() -> None:
+    main(prog="formed")
+
+
+if __name__ == "__main__":
+    run()

formed/commands/__init__.py

@@ -0,0 +1,21 @@
+import argparse
+from typing import Optional
+
+from formed import __version__
+from formed.commands import workflow  # noqa: F401
+from formed.commands.subcommand import Subcommand
+
+
+def create_subcommand(prog: Optional[str] = None) -> Subcommand:
+    parser = argparse.ArgumentParser(usage="%(prog)s", prog=prog)
+    parser.add_argument(
+        "--version",
+        action="version",
+        version="%(prog)s " + __version__,
+    )
+    return Subcommand(parser)
+
+
+def main(prog: Optional[str] = None) -> None:
+    app = create_subcommand(prog)
+    app()

formed/commands/subcommand.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import argparse
+import re
+from collections import defaultdict
+from collections.abc import Callable
+from typing import ClassVar, NamedTuple, Optional, TypeVar, cast
+
+Subclass = TypeVar("Subclass", bound="Subcommand")
+Registry = dict[type["Subcommand"], dict[str, type["Subcommand"]]]
+
+
+class SubcommandInfo(NamedTuple):
+    name: str
+    usage: str | None = None
+    description: str | None = None
+    epilog: str | None = None
+
+
+class Subcommand:
+    _registry: ClassVar[Registry] = defaultdict(dict)
+    _func_key: ClassVar[str] = "__func"
+    _cmd_info: ClassVar[SubcommandInfo]
+
+    @classmethod
+    def register(
+        cls,
+        name: str | None = None,
+        usage: str | None = None,
+        description: str | None = None,
+        epilog: str | None = None,
+        exist_ok: bool = False,
+    ) -> Callable[[type[Subclass]], type[Subclass]]:
+        registry = Subcommand._registry[cls]
+
+        def wrapper(subclass: type[Subclass]) -> type[Subclass]:
+            info = SubcommandInfo(
+                name=name or cls.camel_to_snake(subclass.__name__),
+                usage=usage,
+                description=description or subclass.__doc__,
+                epilog=epilog,
+            )
+            subclass._cmd_info = info
+
+            if not exist_ok and name in registry:
+                raise ValueError(f"Subcommand '{name}' was already registered.")
+
+            registry[info.name] = subclass
+            return subclass
+
+        return wrapper
+
+    @classmethod
+    def get_info(cls) -> SubcommandInfo:
+        if not hasattr(cls, "_cmd_info"):
+            cls._cmd_info = SubcommandInfo(
+                name=cls.camel_to_snake(cls.__name__),
+                description=cls.__doc__,
+            )
+        return cls._cmd_info
+
+    @staticmethod
+    def camel_to_snake(text: str) -> str:
+        underscored = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", text)
+        return re.sub("([a-z0-9])([A-Z])", r"\1_\2", underscored).lower()
+
+    def __init__(
+        self,
+        parser_or_subparsers: argparse.ArgumentParser | argparse._SubParsersAction | None = None,
+        subcommand_info: SubcommandInfo | None = None,
+    ) -> None:
+        cls = type(self)
+        info = subcommand_info or self.get_info()
+
+        if isinstance(parser_or_subparsers, argparse.ArgumentParser):
+            self._parser = parser_or_subparsers
+        elif isinstance(parser_or_subparsers, argparse._SubParsersAction):
+            self._parser = parser_or_subparsers.add_parser(
+                name=info.name,
+                usage=info.usage,
+                description=info.description,
+                epilog=info.epilog,
+            )
+        else:
+            self._parser = argparse.ArgumentParser()
+
+        self.setup()
+        self._parser.set_defaults(**{self._func_key: self.run})
+
+        registry = Subcommand._registry[cls]
+        if registry:
+            subparsers = self.parser.add_subparsers()
+            for subclass in registry.values():
+                subclass(subparsers)
+
+    def __call__(self, args: argparse.Namespace | None = None) -> None:
+        if not args:
+            args = self.parser.parse_args()
+
+        func = cast(
+            Optional[Callable[[argparse.Namespace], None]],
+            getattr(args, self._func_key, None),
+        )
+        if func:
+            func(args)
+        else:
+            self.parser.print_help()
+
+    @property
+    def parser(self) -> argparse.ArgumentParser:
+        return self._parser
+
+    def setup(self) -> None:
+        """setup parser"""
+
+    def run(self, args: argparse.Namespace) -> None:
+        self.parser.print_help()

formed/commands/workflow.py

@@ -0,0 +1,92 @@
+import argparse
+import pathlib
+import sys
+from logging import getLogger
+
+from formed.settings import load_formed_settings
+from formed.workflow import WorkflowExecutionID, WorkflowExecutionInfo, WorkflowGraph
+
+from .subcommand import Subcommand
+
+logger = getLogger(__name__)
+
+
+@Subcommand.register("workflow")
+class WorkflowCommand(Subcommand):
+    def setup(self) -> None:
+        self.parser.add_argument(
+            "--settings",
+            type=str,
+            default=None,
+            help="workflow environment settings file path",
+        )
+
+
+@WorkflowCommand.register("run")
+class WorkflowRunCommand(Subcommand):
+    def setup(self) -> None:
+        self.parser.add_argument(
+            "config",
+            type=str,
+            help="config file path",
+        )
+        self.parser.add_argument(
+            "--execution-id",
+            type=WorkflowExecutionID,
+            default=None,
+            help="execution id",
+        )
+        self.parser.add_argument(
+            "--step",
+            type=str,
+            default=None,
+            help="step name to execute",
+        )
+        self.parser.add_argument(
+            "--overrides",
+            type=str,
+            default=None,
+            help="overrides jsonnet file path",
+        )
+
+    def run(self, args: argparse.Namespace) -> None:
+        formed_settings = load_formed_settings(args.settings)
+        settings = formed_settings.workflow
+        config_path = pathlib.Path(args.config)
+
+        logger.info(f"Load workflow from {config_path}")
+        graph = WorkflowGraph.from_jsonnet(config_path, overrides=args.overrides)
+        if args.step is not None:
+            graph = graph.get_subgraph(args.step)
+
+        execution = WorkflowExecutionInfo(graph, id=args.execution_id)
+
+        organizer = settings.organizer
+        executor = settings.executor
+
+        if execution.id is not None and organizer.exists(execution.id):
+            logger.error(f"Execution {execution.id} already exists")
+            sys.exit(1)
+
+        organizer.run(executor, execution)
+
+
+@WorkflowCommand.register("remove")
+class WorkflowRemoveCommand(Subcommand):
+    def setup(self) -> None:
+        self.parser.add_argument(
+            "execution_id",
+            type=WorkflowExecutionID,
+            help="execution id",
+        )
+
+    def run(self, args: argparse.Namespace) -> None:
+        formed_settings = load_formed_settings(args.settings)
+        settings = formed_settings.workflow
+        organizer = settings.organizer
+
+        if not organizer.exists(args.execution_id):
+            logger.error(f"Execution {args.execution_id} does not exist")
+            sys.exit(1)
+
+        organizer.remove(args.execution_id)

formed/common/astutils.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import ast
+import itertools
+import textwrap
+
+
+class ASTNormalizer(ast.NodeTransformer):
+    def __init__(self) -> None:
+        self.global_name_counter = itertools.count()
+        self.scoped_name_counter = itertools.count()
+        self.global_name_mapping: dict[str, str] = {}
+        self.scope_stack: list[dict[str, str]] = [{}]
+
+    def find_name_in_scopes(self, name: str) -> str | None:
+        for scope in reversed(self.scope_stack):
+            if name in scope:
+                return scope[name]
+        return None
+
+    def visit_Name(self, node: ast.Name) -> ast.Name:
+        new_name = self.find_name_in_scopes(node.id)
+        if new_name is not None:
+            node.id = new_name
+        return node
+
+    def visit_arg(self, node: ast.arg) -> ast.arg:
+        new_name = self.find_name_in_scopes(node.arg)
+        if new_name is not None:
+            node.arg = new_name
+        return node
+
+    def visit_FunctionDef(self, node: ast.FunctionDef) -> ast.FunctionDef:
+        if node.name not in self.global_name_mapping:
+            self.global_name_mapping[node.name] = f"var{next(self.global_name_counter)}"
+        node.name = self.global_name_mapping[node.name]
+        self.scope_stack.append({})
+        self.scoped_name_counter = itertools.count()
+        for arg in node.args.args:
+            new_name = f"var{next(self.scoped_name_counter)}"
+            self.scope_stack[-1][arg.arg] = new_name
+        self.generic_visit(node)
+        self.scope_stack.pop()
+        return node
+
+    def visit_Assign(self, node: ast.Assign) -> ast.Assign:
+        for target in node.targets:
+            if isinstance(target, ast.Name):
+                new_name = self.find_name_in_scopes(target.id)
+                if new_name is None:
+                    new_name = f"var{next(self.scoped_name_counter)}"
+                    self.scope_stack[-1][target.id] = new_name
+                target.id = new_name
+        self.generic_visit(node)
+        return node
+
+
+def normalize_source(source: str) -> str:
+    source = textwrap.dedent(source)
+    tree = ast.parse(source)
+    normalizer = ASTNormalizer()
+    tree = normalizer.visit(tree)
+    return ast.dump(tree)

formed/common/base58.py

@@ -0,0 +1,30 @@
+from typing import Final, Union
+
+_ALPHABET: Final[bytes] = b"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
+
+
+def b58encode(s: Union[str, bytes]) -> bytes:
+    if isinstance(s, str):
+        s = s.encode("ascii")
+    original_length = len(s)
+    s = s.lstrip(b"\0")
+    stripped_length = len(s)
+    n = int.from_bytes(s, "big")
+    res = bytearray()
+    while n:
+        n, r = divmod(n, 58)
+        res.append(_ALPHABET[r])
+    return _ALPHABET[0:1] * (original_length - stripped_length) + bytes(reversed(res))
+
+
+def b58decode(v: Union[str, bytes]) -> bytes:
+    v = v.rstrip()
+    if isinstance(v, str):
+        v = v.encode("ascii")
+    original_length = len(v)
+    v = v.lstrip(_ALPHABET[0:1])
+    stripped_length = len(v)
+    n = 0
+    for c in v:
+        n = n * 58 + _ALPHABET.index(c)
+    return n.to_bytes(original_length - stripped_length + (n.bit_length() + 7) // 8, "big")

formed/common/dag.py

@@ -0,0 +1,111 @@
+import sys
+from collections.abc import Callable, Hashable
+from typing import Generic, TextIO, TypeVar
+
+T_Node = TypeVar("T_Node", bound=Hashable)
+
+
+class DAG(Generic[T_Node]):
+    def __init__(self, dependencies: dict[T_Node, set[T_Node]]) -> None:
+        nodes = set(dependencies.keys() | set(node for deps in dependencies.values() for node in deps))
+        empty_dependencies: dict[T_Node, set[T_Node]] = {node: set() for node in nodes}
+
+        self._dependencies = {**empty_dependencies, **dependencies}
+
+    def __len__(self) -> int:
+        return len(self._dependencies)
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, DAG):
+            return NotImplemented
+        return self._dependencies == other._dependencies
+
+    def __hash__(self) -> int:
+        return hash(frozenset((node, frozenset(deps)) for node, deps in self._dependencies.items()))
+
+    @property
+    def nodes(self) -> set[T_Node]:
+        return set(self._dependencies.keys())
+
+    def subgraph(self, nodes: set[T_Node]) -> "DAG":
+        return DAG({node: nodes & deps for node, deps in self._dependencies.items() if node in nodes})
+
+    def in_degree(self, node: T_Node) -> int:
+        return len(self._dependencies[node])
+
+    def successors(self, node: T_Node) -> set[T_Node]:
+        return {successor for successor, deps in self._dependencies.items() if node in deps}
+
+    def weekly_connected_components(self) -> set["DAG"]:
+        groups: list[set[T_Node]] = []
+
+        for node, deps in self._dependencies.items():
+            current_group = {node} | deps
+            for group in groups:
+                if group & current_group:
+                    group.update(current_group)
+                    break
+            else:
+                groups.append(current_group)
+
+        for i, group in enumerate(groups):
+            for other_group in groups[i + 1 :]:
+                if group & other_group:
+                    group.update(other_group)
+                    other_group.clear()
+
+        groups = [group for group in groups if group]
+
+        return {DAG({node: self._dependencies[node] for node in group}) for group in groups}
+
+    def visualize(
+        self,
+        *,
+        indent: int = 2,
+        output: TextIO = sys.stdout,
+        rename: Callable[[T_Node], str] = str,
+    ) -> None:
+        locations: dict[T_Node, tuple[int, int]] = {}
+
+        def _process_dag(dag: DAG, level: int) -> None:
+            for subdag in sorted(dag.weekly_connected_components(), key=lambda g: sorted(g.nodes)):
+                _process_component(subdag, level)
+
+        def _process_component(dag: DAG, level: int) -> None:
+            sources = set(node for node in dag.nodes if dag.in_degree(node) == 0)
+            for i, node in enumerate(sorted(sources)):
+                if node in locations:
+                    raise ValueError(f"Node {node} already placed")
+                locations[node] = (len(locations), level + i)
+
+            subdag = dag.subgraph(dag.nodes - sources)
+            _process_dag(subdag, level + len(sources))
+
+        _process_dag(self, 0)
+
+        a = [[" "] * level * indent for _, level in sorted(locations.values())]
+        for node, (position, level) in sorted(locations.items(), key=lambda x: x[1][0]):
+            successors = sorted(self.successors(node), key=lambda v: locations[v][0])
+            if not successors:
+                continue
+
+            last_successor_position, last_successor_level = locations[successors[-1]]
+            for successor_position in range(position + 1, last_successor_position):
+                a[successor_position][level * indent] = "│"
+            for successor in successors[:-1]:
+                successor_position, successor_level = locations[successor]
+                a[successor_position][level * indent] = (
+                    "┼" if level > 0 and a[successor_position][level * indent - 1] == "─" else "├"
+                )
+                for offset in range(level * indent + 1, successor_level * indent):
+                    a[successor_position][offset] = "─"
+            a[last_successor_position][level * indent] = (
+                "┴" if level > 0 and a[last_successor_position][level * indent - 1] == "─" else "╰"
+            )
+            for offset in range(level * indent + 1, last_successor_level * indent):
+                a[last_successor_position][offset] = "─"
+
+        output.writelines(
+            "".join(a[position]) + "• " + rename(node) + "\n"
+            for node, (position, _) in sorted(locations.items(), key=lambda x: x[1][0])
+        )

formed/common/git.py

@@ -0,0 +1,76 @@
+import dataclasses
+import os
+import subprocess
+from os import PathLike
+from pathlib import Path
+from typing import Optional, Union
+
+
+@dataclasses.dataclass(frozen=True)
+class GitInfo:
+    commit: str
+    branch: str
+
+
+class GitClient:
+    @staticmethod
+    def is_available() -> bool:
+        return (
+            subprocess.run(["git", "--version"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0
+        )
+
+    def __init__(self, directory: Union[str, PathLike]) -> None:
+        if not self.is_available():
+            raise RuntimeError("Git is not available")
+
+        self._directory = Path(directory)
+
+    def root(self) -> Path:
+        return Path(os.popen(f"git -C {self._directory} rev-parse --show-toplevel").read().strip())
+
+    def is_initialized(self) -> bool:
+        return (
+            subprocess.run(
+                ["git", "status"], cwd=self._directory, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+            ).returncode
+            == 0
+        )
+
+    def diff(self) -> str:
+        return os.popen(f"git -C {self._directory} diff").read()
+
+    def show(
+        self,
+        commit: Optional[str] = None,
+        format: Optional[str] = None,
+        no_patch: bool = False,
+    ) -> str:
+        command = f"git -C {self._directory} show"
+        if commit:
+            command += f" {commit}"
+        if format:
+            command += f" --format='{format}'"
+        if no_patch:
+            command += " --no-patch"
+        return os.popen(command).read()
+
+    def get_commit(self) -> str:
+        return self.show(no_patch=True, format="%H").strip()
+
+    def get_branch(self) -> str:
+        return os.popen(f"git -C {self._directory} rev-parse --abbrev-ref HEAD").read().strip()
+
+    def get_info(self) -> GitInfo:
+        commit = self.get_commit()
+        branch = self.get_branch()
+        return GitInfo(commit, branch)
+
+
+def get_git_info(directory: Optional[Union[str, PathLike]] = None) -> Optional[GitInfo]:
+    if not GitClient.is_available():
+        return None
+    directory = directory or Path.cwd()
+    client = GitClient(directory)
+    if not client.is_available() or not client.is_initialized():
+        return None
+    return client.get_info()

formed/common/hashutils.py

@@ -0,0 +1,70 @@
+import hashlib
+import io
+from typing import Any
+
+import dill
+
+
+def hash_object_bytes(o: Any) -> bytes:
+    m = hashlib.blake2b()
+    with io.BytesIO() as buffer:
+        dill.dump(o, buffer)
+        m.update(buffer.getbuffer())
+    return m.digest()
+
+
+def hash_object(o: Any) -> int:
+    return int.from_bytes(hash_object_bytes(o), "big")
+
+
+def murmurhash3(key: str, seed: int = 0) -> int:
+    length = len(key)
+
+    remainder = length & 3
+    n = length - remainder
+    h1 = seed
+    c1 = 0xCC9E2D51
+    c2 = 0x1B873593
+    i = 0
+
+    while i < n:
+        k1 = (
+            (ord(key[i]) & 0xFF)
+            | ((ord(key[i + 1]) & 0xFF) << 8)
+            | (ord(key[i + 2]) & 0xFF) << 16
+            | (ord(key[i + 3]) & 0xFF) << 24
+        )
+        i += 4
+
+        k1 = ((((k1 & 0xFFFF) * c1) + ((((k1 >> 16) * c1) & 0xFFFF) << 16))) & 0xFFFFFFFF
+        k1 = (k1 << 15) | (k1 >> 17)
+        k1 = ((((k1 & 0xFFFF) * c2) + ((((k1 >> 16) * c2) & 0xFFFF) << 16))) & 0xFFFFFFFF
+
+        h1 ^= k1
+        h1 = (h1 << 13) | (h1 >> 19)
+        h1b = ((((h1 & 0xFFFF) * 5) + ((((h1 >> 16) * 5) & 0xFFFF) << 16))) & 0xFFFFFFFF
+        h1 = ((h1b & 0xFFFF) + 0x6B64) + ((((h1b >> 16) + 0xE654) & 0xFFFF) << 16)
+
+    k1 = 0
+    if remainder == 3:
+        k1 ^= (ord(key[i + 2]) & 0xFF) << 16
+        k1 ^= (ord(key[i + 1]) & 0xFF) << 8
+        k1 ^= ord(key[i]) & 0xFF
+    elif remainder == 2:
+        k1 ^= (ord(key[i + 1]) & 0xFF) << 8
+        k1 ^= ord(key[i]) & 0xFF
+    elif remainder == 1:
+        k1 ^= ord(key[i]) & 0xFF
+    k1 = ((((k1 & 0xFFFF) * c1) + ((((k1 >> 16) * c1) & 0xFFFF) << 16))) & 0xFFFFFFFF
+    k1 = (k1 << 15) | (k1 >> 17)
+    k1 = ((((k1 & 0xFFFF) * c2) + ((((k1 >> 16) * c2) & 0xFFFF) << 16))) & 0xFFFFFFFF
+    h1 ^= k1
+
+    h1 ^= length
+    h1 ^= h1 >> 16
+    h1 = ((((h1 & 0xFFFF) * 0x85EBCA6B) + ((((h1 >> 16) * 0x85EBCA6B) & 0xFFFF) << 16))) & 0xFFFFFFFF
+    h1 ^= h1 >> 13
+    h1 = ((((h1 & 0xFFFF) * 0xC2B2AE35) + ((((h1 >> 16) * 0xC2B2AE35) & 0xFFFF) << 16))) & 0xFFFFFFFF
+    h1 ^= h1 >> 16
+
+    return h1 >> 0