m-strat 0.0.2__py3-none-any.whl

m_strat/__init__.py

@@ -0,0 +1 @@
+# Init

m_strat/api/__init__.py

@@ -0,0 +1 @@
+# init

m_strat/api/dag/__init__.py

@@ -0,0 +1,4 @@
+from .blueprint_register import *
+from .executor import *
+from .graph_manager import *
+from .node import *

m_strat/api/dag/blueprint_register.py

@@ -0,0 +1,197 @@
+import importlib.util
+import inspect
+import json
+import os
+
+from m_strat.api.dag.node import BlueprintNode
+from m_strat.api.logger import m_log
+
+
+class BlueprintRegister:
+    """Manages Blueprint Nodes in a project"""
+
+    def __init__(self, meta_file_path: str = "configs/bp_files_meta.jsonl"):
+        self.blueprints = {}
+        self.schemas = {}
+        self.files_found = []
+
+        if meta_file_path:
+            self.load_from_meta_file(meta_file_path)
+        self.meta_file_path = meta_file_path
+
+    def register_node(
+        self, name: str, func: callable, schema: dict, overwrite: bool = True
+    ):
+        """Adds a method to the project registry
+        Args:
+            name (str): The new name of the blueprint node being created (can be given via BlueprintNode decorator)
+            func (callable): The function that will be used to execute the node logic
+            schema (dict[str, type]): Used to guide the UI and create an automated form in the application, not enforced
+            overwrite (bool, optional): Allows a node to be overwritten if the name already exists. Defaults to True.
+        """
+        assert overwrite or self.blueprints.get(name, None) is None, (
+            f"blueprint of '{name}' already exists, overwrite not allowed"
+        )
+        self.blueprints[name] = func
+        self.schemas[name] = schema
+
+    def unregister_node(self, name: str):
+        """Remove a node from the registry by name
+        Returns:
+           tuple[dict,dict] : blueprint callable and schema respectively
+        """
+        return (self.blueprints.pop(name, None), self.schemas.pop(name, None))
+
+    def get_blueprint(self, name: str):
+        """Get the callable of a blueprint by name"""
+        return self.blueprints.get(name, None)
+
+    def refresh_blueprints(self):
+        self.blueprints = {}
+        self.schemas = {}
+        old_files_found = list(self.files_found)
+        self.files_found = []
+        for f in old_files_found:
+            self.add_from_file(f)
+
+    def save_meta_file(self, meta_file_path="configs/bp_files_meta.jsonl"):
+        """Generates a jsonl file of where all the blueprints were retrieved from, so they can be loaded easily
+        Args:
+            meta_file_path (str, optional): relative path of wheree this generated file will save. Defaults to "configs/bp_files_meta.jsonl".
+        """
+        if not self.files_found:
+            m_log.info("Attempted to save blueprints, but no blueprints registered")
+            return
+        with open(meta_file_path, "w", encoding="utf-8") as meta_f:
+            meta_f.writelines(
+                json.dumps(bp_file_path) + "\n"
+                for bp_file_path in set(self.files_found)
+            )
+        m_log.info(f"Blueprint data saved {meta_file_path}")
+
+    def load_from_meta_file(self, meta_file_path: str | None = None):
+        """Loads and use each line as a path to retrieve saved blueprint reference from py files .
+        Args:
+            meta_file_path (str, optional): The path where the meta file resides. Defaults to "configs/bp_files_meta.jsonl".
+        Returns:
+           list[str] : Returns a list of all the names of the blueprints that were added
+        """
+        m_log.info("Loading Blueprint Configuration Data...")
+        if meta_file_path is None:
+            meta_file_path = self.meta_file_path
+        if not os.path.exists(meta_file_path):
+            m_log.warning(f"bp_reg: path does not exist:: {meta_file_path}")
+            return []
+        added = []
+        with open(meta_file_path, "r", encoding="utf8") as meta_f:
+            for i, line in enumerate(meta_f):
+                if not line.strip():
+                    continue
+                try:
+                    next_path = json.loads(line.strip())
+                    added.extend(self.add_from_file(next_path))
+                except (OSError, ValueError, json.JSONDecodeError) as e:
+                    m_log.warning(
+                        f"Line at {i} in {meta_file_path} failed to read: {e}"
+                    )
+        return added
+
+    def get_schema(self, name: str):
+        """Gets the (dict[str,type]) schema given a name of a Blueprint Node"""
+        return self.schemas.get(name, None)
+
+    def add_from_file(self, file_path: str):
+        """uses importlib to check for any Blueprint Nodes and attempts to add them to the system
+        Args:
+            file_path (str): relative path of the .py blueprint file resides
+        Returns:
+            list[str] : Returns a list of all the names of the blueprints that were added
+        """
+        if not os.path.exists(file_path):
+            m_log.warning(f"bp_reg: path does not exist:: {file_path}")
+            return []
+
+        module_name = os.path.splitext(os.path.basename(file_path))[0]
+        spec = importlib.util.spec_from_file_location(module_name, file_path)
+        added = []
+        assert (
+            spec is not None and spec.loader
+        ), f"""Provided file does not have a spec or spec loader: ||{file_path} ||
+            This can happen because:
+            1. If the file exstension is wrong (not .py)
+            2. File permission issues such as file locked (in use elsewhere e.g )
+            3. Unfriendly path characters: path contains unusual unicode characters or file name is not python friendly
+            4. Some issue which I simply have not thought of in this moment.
+            """
+        module = importlib.util.module_from_spec(spec)
+        module.BlueprintNode = BlueprintNode
+        try:
+            spec.loader.exec_module(module)
+        except Exception as e:
+            # flagrantly assume the error is because the module should not be run like this!
+            m_log.debug(
+                f"Spec Loader did not like something when bp register used exec_module (can ignore typically): {e}"
+            )
+
+        for _, obj in inspect.getmembers(module, inspect.isfunction):
+            if getattr(obj, "__is_m_strat_node__", False):
+                self.register_node(
+                    obj.__m_strat_node_name__, obj, obj.__m_strat_node_schema__
+                )
+                added.append(obj.__m_strat_node_name__)
+
+        if len(added) > 0:
+            self.files_found.append(file_path)
+        return added
+
+    def add_from_folder(self, folder_path, add_subfolders=True):
+        """
+            From a directory, adds all Blueprint Nodes to the register
+        Args:
+            folder_path (str): The relative directory path
+            add_subfolders (bool, optional): Whether to add Blueprint Nodes from subdirectories (of any depth) . Defaults to True.
+        Returns:
+            list[str] : Returns a list of all the names of the blueprints that were added
+        """
+        if not os.path.isdir(folder_path):
+            m_log.warning(f"bp_reg: path does not exist:: {folder_path}")
+            return []
+        added_nodes = []
+        m_log.info(f"Adding folder blueprint folder: {folder_path}")
+        if add_subfolders:
+            for root, _, files in os.walk(folder_path):
+                for f_name in files:
+                    if f_name.endswith(".py") and not f_name.startswith("__"):
+                        full_path = os.path.join(root, f_name)
+                        next_added = []
+                        try:
+                            next_added = self.add_from_file(full_path)
+                        except Exception as e:
+                            m_log.error(
+                                f"Blueprint tried to add Blueprints from |{f_name}| in directory |{root}| but failed: \n {e} \n{'*' * 12}\n ...Skipping!"
+                            )
+                            continue
+
+                        added_nodes.extend(next_added)
+        else:
+            for f_name in os.listdir(folder_path):
+                if f_name.endswith(".py") and not f_name.startswith("__"):
+                    full_path = os.path.join(folder_path, f_name)
+                    if os.path.isfile(full_path):
+                        added_nodes.extend(self.add_from_file(full_path))
+        m_log.info(
+            f"Finished adding blueprint director, {len(added_nodes)} blueprint(s) found"
+        )
+        return added_nodes
+
+    def get_all_blueprints(self):
+        """Copies the blueprint callable dictionary, returning all blueprints
+        NB: Blueprints are given by reference, modifications will alter them!
+        """
+        return dict(self.blueprints)
+
+    def get_all_schemas(self):
+        """Copies the schema dictionary, returning all blueprint schemas
+        NB: Blueprints are given by reference, modifications will alter them!
+        """
+        return dict(self.schemas)

m_strat/api/dag/executor.py

@@ -0,0 +1,34 @@
+from m_strat.api.logger import m_log
+
+_EXECUTORS = {}
+
+
+# Defines an exectuion method for Node system. Must have (targetnodes, init_context) as arguments. Must return context
+# name muse be unique
+def Executor(name: str):
+    def decorator(func):
+        assert _EXECUTORS.get(name, None) is None, f"Executor {name} already exists"
+        _EXECUTORS[name] = func
+        return func
+
+    return decorator
+
+
+# Standard linear-non parallelised running, executes in same thread without modification
+@Executor(name="Sequential")
+def execute_sequential(target_nodes: list[tuple], init_context: dict):
+    context = init_context
+    m_log.info("Started Sequential running")
+    for func, params, node_info in target_nodes:
+        m_log.debug(f"Running Node {node_info.id}")
+        try:
+            context = func(context, params)
+        except Exception as e:
+            m_log.error(
+                f"Pipeline execution failed at '{node_info.id}' using blueprint '{node_info.blueprint_name}':\n {e}"
+            )
+            raise  # NB you should always raiserrors when making your own Executor!!
+        m_log.debug(f"Node{node_info.id} Completed")
+
+    m_log.info(f"Finished Executing {len(target_nodes)} node(s)")
+    return context

m_strat/api/dag/graph_manager.py

@@ -0,0 +1,232 @@
+import json
+import os
+from collections import defaultdict, deque
+from dataclasses import asdict
+
+from m_strat.api.project_management.wrappers import Serializable
+
+from .node import ExecutionNode
+
+
+class GraphManager(Serializable):
+    """Manages a single DAG which is used to build a ML pipeline from Blueprint Nodes"""
+
+    def __init__(
+        self,
+        name: str,
+        load_path: str | None = None,
+        load: bool = False,
+        dir: str = "exec_graphs",
+    ):
+        """Instansiates a Graphmanager object used to manage a DAG
+        Args:
+            name (str): The designated label of this DAG
+            load_path (str, optional): Path to json config . Defaults to None.
+            load (bool, optional): If true attemps to load the graph by name or load path. Defaults to False.
+            dir (str, optional): Path to where this graph is stored. Defaults to "exec_graphs".
+
+        Raises:
+            ValueError: if no name or load_path is supplied and load is true as no data can be loaded.
+        """
+        self.nodes = {"start": ExecutionNode("start", "", "")}
+        self.edges = defaultdict(list)
+        self.name = name
+        self.dir = dir
+        if load and (name or load_path):
+            self.load(name=name, load_path=load_path)
+        elif load:
+            raise ValueError("GraphManager cannot load without a valid graph name")
+
+    @staticmethod
+    def get_file_path(name: str, dir: str):
+        """From the name and directory, returns what the graph data file should be"""
+        return os.path.join(dir, f"gm_{name}.json")
+
+    @staticmethod
+    def get_name_from_filename(path: str):
+        """if given a standardised graph data file, extracts the name of the graph"""
+        base_name: str = os.path.splitext(os.path.basename(path))[0]
+        return base_name[3:]
+
+    def create_add_new_node(self, node_id: str, blueprint_name: str, param_id: str):
+        """From data supplied, creates an ExecutionNode and adds it to the graph
+        Args:
+            node_id (str): human readable name of node
+            blueprint_name (str): name of the blueprint in blueprint Register
+            param_id (str): name of node params from paramater manager
+        """
+        new_node = ExecutionNode(
+            id=node_id, blueprint_name=blueprint_name, param_id=param_id
+        )
+        self.add_node(new_node)
+
+    def add_node(self, new_node: ExecutionNode):
+        """Adds a node to the graph"""
+        assert self.nodes.get(new_node.id, None) is None, "Node Already exists"
+        self.nodes[new_node.id] = new_node
+
+    def change_node_param_template(self, node_id: str, new_param_id: str):
+        """Changes a node parameter template (i.e. the "params" parsed into the ExecutionNode at runtime)
+        Args:
+            node_id (str): instance name of the node within the DAG
+            new_param_id (str): the new parameter id for the node to use
+        """
+        assert isinstance(new_param_id, str) and new_param_id, (
+            "New Parameter id Invalid"
+        )
+        node = self.nodes.get(node_id, None)
+        assert node is not None, "Node not found"
+        node.param_id = new_param_id
+
+    def remove_node(self, node_id: str):
+        """
+            Removes a node from the DAG (and its edges)
+        Args:
+            node_id(str): the target node to be removed (DAG name not blueprint name!)
+        """
+        assert node_id != "start", (
+            "Cannot Remove or Alter Start Node, alter it's edges, or parse different start point to build_execution instead"
+        )
+        self.edges.pop(node_id, [])
+        self.nodes.pop(node_id)
+        for e_v_lst in self.edges.values():
+            if node_id in e_v_lst:
+                e_v_lst.remove(node_id)
+
+    def add_edge(self, node_from: str, node_to: str):
+        """Connects 2 nodes together, representing execution order
+        Args:
+            node_from (str): Node from which the connection is to be made
+            node_to (str): Node the connection ends at.
+        """
+        assert node_to != "start", "All edges on start node must be outgoing"
+        assert node_from != node_to, "A node cannot have an edge connecting to itself"
+        assert self.nodes.get(node_from, None) is not None, (
+            f"Add Edge From Node {node_from} not present in manager"
+        )
+        assert self.nodes.get(node_to, None) is not None, (
+            f"Add Edge To Node {node_to} not present in manager"
+        )
+        neighbours = self.edges[node_from]
+        if node_to not in neighbours:
+            neighbours.append(node_to)
+
+    def remove_edge(self, node_from: str, node_to: str, allow_inverse: bool = False):
+        """Removes 2 connecting edges of a node, allow inverse checks with the nodes in the opposite direction"""
+        self.edges[node_from].remove(node_to)
+        if allow_inverse:
+            inverse_edge = self.edges.get(node_to, None)
+            if not inverse_edge:
+                return
+            if node_from in inverse_edge:
+                inverse_edge.remove(node_from)
+
+    def get_edges(self):
+        """Gets all edges as a set of  tuples  (of -> from, to) and returns them
+        Returns:
+            set: tuple of (node_from, node_to)
+        """
+        edge_tuples = set()
+        for node_from, neighbours in self.edges.items():
+            for node_to in neighbours:
+                edge_tuples.add((node_from, node_to))
+        return edge_tuples
+
+    def get_nodes(self):
+        """Returns a shallow copy of all nodes"""
+        return dict(self.nodes)
+
+    def get_node(self, node_id: str):
+        """Gets a node via node_id from this dag, returns None if not present"""
+        return self.nodes.get(node_id, None)
+
+    def build_execution(self, start_node_id="start"):
+        """Generates a linear execution of nodes from star_node_id until there are ends
+        Args:
+            start_node_id (str, optional): String of first node_id. Defaults to "start".
+        Returns:
+            list[ExecutionNode]: returns the order of nodes by execution from start->finish
+        NB: Currently does not detect loops, parallelisation is flattened currently.
+        """
+        in_degree = {node_id: 0 for node_id in self.nodes}
+
+        for neighbors in self.edges.values():
+            for n in neighbors:
+                in_degree[n] += 1
+
+        queue = deque([start_node_id])
+        exec_order = []
+
+        while queue:
+            current = queue.popleft()
+
+            exec_order.append(self.nodes.get(current))
+
+            for neighbor in self.edges.get(current, []):
+                in_degree[neighbor] -= 1
+                if in_degree[neighbor] == 0:
+                    queue.append(neighbor)
+
+        return exec_order
+
+    def load(self, name: str | None = None, load_path: str | None = None):
+        """using the load path primarily or the name if not supplied. load selection order: load_path->name->self.name
+        Args:
+            name (str, optional): Name of graph manager. Defaults to None.
+            load_path (str, optional): Path to graph configuration file. Priortised. Defaults to None.
+        """
+        assert load_path is not None or name is not None or self.name is not None, (
+            "At least one identifier required to find the graph manager"
+        )
+        if load_path is None:
+            load_path = self.get_file_path(name, self.dir)
+
+        with open(load_path, "r", encoding="utf-8") as load_file:
+            load_data = json.load(load_file)
+            self.edges = defaultdict(list, load_data["edges"])
+            self.name = load_data["name"] if name is None else name
+            node_data = {}
+            for node_id, exec_dict in load_data["nodes"].items():
+                node_data[node_id] = ExecutionNode(
+                    id=node_id,
+                    blueprint_name=exec_dict["blueprint_name"],
+                    param_id=exec_dict["param_id"],
+                    pos=exec_dict.get("pos", None),
+                )
+            self.nodes = node_data
+
+    def save(self):
+        """using the name of the current graph, generates a name and saves the configuration as json"""
+        save_path = self.get_file_path(self.name, self.dir)
+        save_data = {"edges": self.edges, "name": self.name}
+        save_data["nodes"] = {
+            node_id: asdict(exec_node) for node_id, exec_node in self.nodes.items()
+        }
+        with open(save_path, "w", encoding="utf-8") as save_file:
+            json.dump(save_data, save_file)
+
+    def duplicate(self, new_name=""):
+        """Duplicates the current graph manager into another"""
+        if not new_name:
+            new_name = f"{self.name}_copy"
+        dupe = GraphManager(name=new_name, load=False, dir=self.dir)
+        for n in self.nodes.values():
+            if n.id == "start":
+                if n.pos:
+                    dupe.nodes["start"].pos = list(n.pos)
+                continue
+            n_copy = ExecutionNode(
+                id=n.id,
+                blueprint_name=n.blueprint_name,
+                param_id=n.param_id,
+                pos=list(n.pos) if n.pos else None,
+            )
+            dupe.add_node(n_copy)
+        for node_name, neighbours in self.edges.items():
+            for neigh in neighbours:
+                dupe.add_edge(node_name, neigh)
+        return dupe
+
+    @classmethod
+    def file_type(cls):
+        return "json"

m_strat/api/dag/node.py

@@ -0,0 +1,56 @@
+from dataclasses import dataclass
+
+# types that are allowed witihn blueprint params. Modification may cause error
+__ALLOWED_SCHEMAS = {
+    int,
+    float,
+    bool,
+    str,
+    list,
+    bytes,
+}
+
+
+def BlueprintNode(name: str, schema: dict[str:type] | None = None):
+    """Designates a function as a DAG node, Function must have arguments (context,params) ONLY and MUST always return context
+    Args:
+        name (str): Unique name of blueprint
+        schema (dict[str,type], optional): Dictionay. Defaults to {}.
+    """
+    if schema is None:
+        schema = {}
+
+    def decorator(node_func):
+        assert isinstance(name, str), "Node Name must be a string"
+        assert len(name) > 0, "Node name is empty"
+
+        if schema:
+            assert all(
+                v in __ALLOWED_SCHEMAS and isinstance(k, str) for k, v in schema.items()
+            ), "Invalid Schema keys or values"
+
+        node_func.__is_m_strat_node__ = True
+        node_func.__m_strat_node_name__ = name
+        node_func.__m_strat_node_schema__ = schema
+        return node_func
+
+    return decorator
+
+
+@dataclass
+class ExecutionNode:
+    id: str  # The human readable 'name' of the node instance
+    param_id: str  # the node parmater identifier (at runtime is retireved as params)
+    blueprint_name: (
+        str  # the name of the blueprint, must exactly match the BlueprintNode name
+    )
+    pos: list = None  # optional positional information for GUIs
+
+    def to_dict(self):
+        """Converts datalcass to a dictionary, used for serialization"""
+        return {
+            "id": self.id,
+            "blueprint_name": self.blueprint_name,
+            "param_id": self.param_id,
+            "pos": self.pos,
+        }

m_strat/api/ecs/__init__.py

@@ -0,0 +1,3 @@
+from .component_dispatcher import *
+from .ecs_register import *
+from .entity import *