thread-order 1.0.0__py3-none-any.whl

thread_order/__init__.py

@@ -0,0 +1,51 @@
+from importlib import metadata as _metadata
+import importlib
+from os import getenv
+
+__all__ = [
+    'Scheduler',
+    'DAGraph',
+    'configure_logging',
+    'ThreadProxyLogger',
+    'dmark',
+    'mark',
+    'default_workers',
+    '__version__']
+
+def __getattr__(name):
+    if name == 'Scheduler':
+        from .scheduler import Scheduler
+        return Scheduler
+    if name == 'DAGraph':
+        from .graph import DAGraph
+        return DAGraph
+    if name == 'configure_logging':
+        from .logger import configure_logging
+        return configure_logging
+    if name == 'ThreadProxyLogger':
+        from .logger import ThreadProxyLogger
+        return ThreadProxyLogger
+    if name == 'dmark':
+        from .scheduler import dmark
+        return dmark
+    if name == 'mark':
+        from .scheduler import mark
+        return mark
+    if name == 'default_workers':
+        from .scheduler import default_workers
+        return default_workers
+    # If the requested attribute isn't one of the known top-level symbols,
+    # try to lazily import a submodule (e.g. `thread_order.scheduler`) so
+    # attribute lookups such as those used by mocking/patching succeed.
+    try:
+        return importlib.import_module(f"{__name__}.{name}")
+    except Exception:
+        raise AttributeError(name)
+
+try:
+    __version__ = _metadata.version(__name__)
+except _metadata.PackageNotFoundError:
+    __version__ = '1.0.0'
+
+if getenv('DEV'):
+    __version__ = f'{__version__}+dev'

thread_order/graph.py

@@ -0,0 +1,142 @@
+import threading
+import logging
+from collections import defaultdict
+
+def log_candidates(candidates, number):
+    """ log a debug message describing how many candidate nodes were found
+    """
+    logger = logging.getLogger(threading.current_thread().name)
+    count = len(candidates)
+    if not candidates:
+        message = 'but found no candidates eligible for submission'
+    else:
+        base = f"and found {count} candidate{'s' if count != 1 else ''} eligible for submission"
+        message = f"{base} {', '.join(candidates)}"
+    logger.debug(f'requested {number} {message}')
+
+class DAGraph:
+
+    def __init__(self):
+        """ initialize an empty DAG with parent and child adjacency mappings
+        """
+        self._parents = defaultdict(list)
+        self._children = defaultdict(set)
+        self._original_parents = {}
+
+    def add(self, name, after=None):
+        """ add a new node with optional dependencies
+
+            All items in `after` must already exist in the DAG.
+            Raises ValueError if the node already exists, dependencies are unknown,
+            or the addition would introduce a cycle.
+        """
+        logger = logging.getLogger(threading.current_thread().name)
+        after = after or []
+        logger.debug(f'add {name} dependent on {after}')
+        if name in self._parents:
+            raise ValueError(f'{name} has already been added')
+        unknowns = [dep for dep in after if dep not in self._parents]
+        if unknowns:
+            raise ValueError(f'{name} depends on unknown {unknowns}')
+        self._parents[name] = []
+        self._original_parents[name] = list(after) if after else []
+        for dep in after:
+            self._parents[name].append(dep)
+            self._children[dep].add(name)
+        # defensive: future refactor may allow updating deps
+        if self._has_cycle():
+            # rollback this node to keep DAG consistent
+            for dep in after:
+                self._children[dep].discard(name)
+            self._parents.pop(name, None)
+            raise ValueError(f'adding {name} will create a cycle')
+
+    def remove(self, name):
+        """ remove a completed node and detach it from all dependent children
+
+            Cleans up parent and child relationships and drops the node completely
+            once it has no remaining edges.
+        """
+        logger = logging.getLogger(threading.current_thread().name)
+        for child in self._children.pop(name, ()):
+            logger.debug(f'removing {name} as a dependency from {child}')
+            try:
+                self._parents[child].remove(name)
+            except ValueError:
+                # defensive: graph might already be partially cleaned
+                pass
+
+        if name in self._parents and not self._parents[name]:
+            logger.debug(f'removing {name} from dependency graph')
+            self._parents.pop(name, None)
+
+    def ready(self, active=None):
+        """ return a list of nodes whose dependencies are satisfied and not active
+        """
+        if active is None:
+            active = set()
+        return [name for name, deps in self._parents.items() if not deps and name not in active]
+
+    def get_candidates(self, active, number, sort=True):
+        """ return up to `number` ready nodes, optionally sorted for stable scheduling
+
+            Also logs the candidate list for visibility.
+        """
+        candidates = self.ready(active)
+        if sort:
+            candidates = sorted(candidates)
+        log_candidates(candidates, number)
+        return candidates[:number]
+
+    def _has_cycle(self):
+        """ return True if DAGraph contains a cycle
+        """
+        visited = set()
+        stack = set()
+
+        def visit(node):
+            if node in stack:
+                return True
+            if node in visited:
+                return False
+            visited.add(node)
+            stack.add(node)
+            for neighbor in self._parents[node]:
+                if visit(neighbor):
+                    return True
+            stack.remove(node)
+            return False
+        return any(visit(node) for node in self._parents)
+
+    def is_empty(self):
+        """ return True if the DAGraph has no nodes
+        """
+        return not self._parents and not self._children
+
+    def __repr__(self):
+        """ return a human-readable representation of the dependency graph
+        """
+        parents = '\n'.join(f'{n}: {self._parents[n]}' for n in sorted(self._parents))
+        children = '\n'.join(
+            f'{n}: {sorted(list(self._children[n]))}' for n in sorted(self._children))
+        return f'Parents:\n{parents}\nChildren:\n{children}'
+
+    def nodes(self):
+        """ return an iterable of node names in the graph
+        """
+        return self._parents.keys()
+
+    def parents_of(self, name):
+        """ return a list of parent nodes (dependencies) for a given node
+        """
+        return list(self._parents.get(name, []))
+
+    def children_of(self, name):
+        """ return a list of child nodes (dependents) for a given node
+        """
+        return list(self._children.get(name, []))
+
+    def original_parents_of(self, name):
+        """ return a list of original parent nodes (dependencies) for a given node
+        """
+        return list(self._original_parents.get(name, []))

thread_order/graph_summary.py

@@ -0,0 +1,313 @@
+"""
+Graph summary formatting helpers for threaded_order.
+
+This module produces a human-readable summary of a DAGraph instance, used
+by the CLI to display the dependency graph.
+"""
+
+def _graph_get_nodes_and_ids(dag):
+    """ return a sorted list of node names and a stable numeric ID mapping.
+
+        IDs are deterministic based on sorted node order.
+        Returns:
+            nodes: [name, ...]
+            ids: {name: numeric_id}
+    """
+    nodes = sorted(dag.nodes())
+    ids = {}
+    for idx, name in enumerate(nodes):
+        ids[name] = idx
+    return nodes, ids
+
+def _graph_build_indegree_and_adj(dag, nodes):
+    """ build indegree table and adjacency (outgoing edges) table.
+
+        Returns:
+            indegree: {node: number_of_parents}
+            adj:      {node: [sorted_child_nodes]}
+            num_edges: total edge count
+    """
+    indegree = {}
+    adj = {}
+    num_edges = 0
+
+    for name in nodes:
+        parents = dag.parents_of(name)
+        children = dag.children_of(name)
+
+        indegree[name] = len(parents)
+        children_sorted = sorted(children) if children else []
+        adj[name] = children_sorted
+        num_edges += len(children_sorted)
+
+    return indegree, adj, num_edges
+
+def _graph_find_roots_and_leaves(nodes, indegree, adj):
+    """ identify root nodes (no parents) and leaf nodes (no children).
+
+        Returns:
+            roots: [node, ...]
+            leaves: [node, ...]
+    """
+    roots = []
+    leaves = []
+
+    for name in nodes:
+        if indegree[name] == 0:
+            roots.append(name)
+        if not adj[name]:
+            leaves.append(name)
+
+    return roots, leaves
+
+def _graph_compute_levels(nodes, roots, indegree, adj, ids):
+    """ compute topological levels using a Kahn-style layering algorithm.
+
+        Each level contains nodes that can run in parallel after prior levels complete.
+
+        Returns:
+            levels: [ [node1, node2], [node3], ... ]
+    """
+    levels = []
+    if not nodes:
+        return levels
+
+    indeg = {}
+    for name, value in indegree.items():
+        indeg[name] = value
+
+    queue = sorted(roots, key=lambda n: ids[n])
+    seen = set()
+
+    while queue:
+        level = []
+        next_queue = []
+
+        for name in queue:
+            if name in seen:
+                continue
+            seen.add(name)
+            level.append(name)
+
+            for dst in adj[name]:
+                indeg[dst] -= 1
+                if indeg[dst] == 0:
+                    next_queue.append(dst)
+
+        if level:
+            level.sort(key=lambda n: ids[n])  # noqa: E731
+            levels.append(level)
+
+        queue = sorted(next_queue, key=lambda n: ids[n])
+
+    if not levels:
+        levels = [nodes]
+
+    return levels
+
+def _graph_format_header(num_nodes, num_edges, roots, leaves, levels_count, ids):
+    """ format the summary header section:
+            Graph: X nodes, Y edges
+            Roots: [id], [id], ...
+            Leaves: [id], [id], ...
+            Levels: Z
+    """
+    lines = []
+    lines.append(f'Graph: {num_nodes} nodes, {num_edges} edges')
+
+    if roots:
+        root_ids = ', '.join(f'[{ids[name]}]' for name in roots)
+        lines.append(f'Roots: {root_ids}')
+
+    if leaves:
+        leaf_ids = ', '.join(f'[{ids[name]}]' for name in leaves)
+        lines.append(f'Leaves: {leaf_ids}')
+
+    lines.append(f'Levels: {levels_count}')
+    return lines
+
+def _graph_format_nodes(nodes, ids):
+    """ format the Nodes: section:
+            [id] node_name
+    """
+    lines = ['Nodes:']
+    for name in nodes:
+        lines.append(f'  [{ids[name]}] {name}')
+    return lines
+
+def _graph_format_edges(nodes, adj, ids):
+    """ format the Edges: section:
+            [src_id] -> [child_id], ...
+            or
+            [src_id] -> (none)
+    """
+    lines = ['Edges:']
+    for name in nodes:
+        children = adj[name]
+        if children:
+            targets = ', '.join(f'[{ids[c]}]' for c in children)
+        else:
+            targets = '(none)'
+        lines.append(f'  [{ids[name]}] -> {targets}')
+    return lines
+
+def _graph_compute_longest_chains(nodes, levels, adj):
+    """ compute the longest dependency chains in the DAG.
+
+    Uses the topological levels to derive a topo order, then computes the
+    longest distance (in edges) from any root to each node.
+
+    Returns:
+        max_len: int, length in edges of the longest chain
+        chains: list of chains, each a [node, node, ...] list
+    """
+    if not nodes:
+        return 0, []
+
+    topo_order = []
+    for level in levels:
+        for name in level:
+            topo_order.append(name)
+
+    dist = {name: 0 for name in nodes}
+    prev = {name: None for name in nodes}
+
+    for src in topo_order:
+        for dst in adj[src]:
+            cand = dist[src] + 1
+            if cand > dist[dst]:
+                dist[dst] = cand
+                prev[dst] = src
+
+    if not dist:
+        return 0, []
+
+    max_len = max(dist.values())
+    if max_len == 0:
+        # no edges; treat single nodes as trivial chains
+        return 0, [[nodes[0]]] if nodes else []
+
+    end_nodes = [name for name, d in dist.items() if d == max_len]
+
+    chains = []
+    for end in end_nodes:
+        chain = []
+        cur = end
+        while cur is not None:
+            chain.append(cur)
+            cur = prev[cur]
+        chains.append(list(reversed(chain)))
+
+    # keep it readable: limit to a few longest chains
+    return max_len, chains[:3]
+
+def _graph_find_hotspots(nodes, indegree, adj, fan_in_min=2, fan_out_min=2):
+    """ identify potential bottlenecks / hotspots:
+
+        - high fan-in: nodes that depend on many others
+        - high fan-out: nodes that unlock many dependents
+
+        Returns:
+            high_in: [node, ...]
+            high_out: [node, ...]
+    """
+    high_in = [name for name in nodes if indegree[name] >= fan_in_min]
+    high_out = [name for name in nodes if len(adj[name]) >= fan_out_min]
+    return high_in, high_out
+
+def _graph_format_stats(max_chain_len, chains, high_in, high_out, indegree, adj):
+    """ format a Stats: section showing longest chains and fan-in/out hotspots.
+    """
+    lines = []
+    lines.append('Stats:')
+    lines.append(f'  Longest chain length (edges): {max_chain_len}')
+
+    if chains:
+        lines.append('  Longest chains:')
+        for chain in chains:
+            path = ' -> '.join(chain)
+            lines.append(f'    {path}')
+
+    if high_in:
+        lines.append('  High fan-in nodes (many dependencies):')
+        for name in high_in:
+            lines.append(f'    {name} (indegree={indegree[name]})')
+
+    if high_out:
+        lines.append('  High fan-out nodes (many dependents):')
+        for name in high_out:
+            lines.append(f'    {name} (children={len(adj[name])})')
+
+    return lines
+
+def format_graph_summary(dag):
+    """ produce the full human-readable DAG summary used by the CLI.
+
+        Example output:
+
+            Graph: 18 nodes, 21 edges
+            Roots: [0], [1]
+            Leaves: [6], [7]
+            Levels: 4
+
+            Nodes:
+            [0] tests/test_a.py::test_A
+            ...
+
+            Edges:
+            [0] -> [2]
+            ...
+
+            Stats:
+            Longest chain length (edges): 3
+            Longest chains:
+                test_a -> test_c -> test_d -> test_f
+            High fan-in nodes (many dependencies):
+                test_f (indegree=2)
+            High fan-out nodes (many dependents):
+                test_a (children=2)
+
+        Returns:
+            A single string containing the formatted summary.
+    """
+    nodes, ids = _graph_get_nodes_and_ids(dag)
+
+    if not nodes:
+        return 'Graph: 0 nodes, 0 edges'
+
+    indegree, adj, num_edges = _graph_build_indegree_and_adj(dag, nodes)
+    roots, leaves = _graph_find_roots_and_leaves(nodes, indegree, adj)
+    levels = _graph_compute_levels(nodes, roots, indegree, adj, ids)
+    max_chain_len, chains = _graph_compute_longest_chains(nodes, levels, adj)
+    high_in, high_out = _graph_find_hotspots(nodes, indegree, adj)
+
+    lines = []
+
+    header_lines = _graph_format_header(
+        num_nodes=len(nodes),
+        num_edges=num_edges,
+        roots=roots,
+        leaves=leaves,
+        levels_count=len(levels),
+        ids=ids,
+    )
+    lines.extend(header_lines)
+    lines.append('')
+
+    lines.extend(_graph_format_nodes(nodes, ids))
+    lines.append('')
+
+    lines.extend(_graph_format_edges(nodes, adj, ids))
+    lines.append('')
+
+    stats_lines = _graph_format_stats(
+        max_chain_len=max_chain_len,
+        chains=chains,
+        high_in=high_in,
+        high_out=high_out,
+        indegree=indegree,
+        adj=adj,
+    )
+    lines.extend(stats_lines)
+
+    return '\n'.join(lines)

thread_order/logger.py

@@ -0,0 +1,143 @@
+import os
+import re
+import sys
+import threading
+import logging
+try:
+    from colorama import init
+    from colorama import Fore, Style
+    HAS_COLOR = True
+except ImportError:
+    HAS_COLOR = False
+
+if HAS_COLOR:
+
+    class ColoredFormatter(logging.Formatter):
+        LEVEL_COLORS = {
+            logging.DEBUG: Style.BRIGHT + Fore.CYAN,
+            logging.INFO: Style.BRIGHT + Fore.BLUE,
+            logging.WARNING: Style.BRIGHT + Fore.YELLOW,
+            logging.ERROR: Style.BRIGHT + Fore.RED,
+            logging.CRITICAL: Style.BRIGHT + Fore.RED,
+        }
+        DEFAULT_HIGHLIGHTS = [
+            (re.compile(r'\bPASSED\b', re.IGNORECASE), Fore.GREEN),
+            (re.compile(r'\bFAILED\b', re.IGNORECASE), Fore.RED),
+            (re.compile(r'\bSKIPPED\b', re.IGNORECASE), Fore.YELLOW),
+            (re.compile(r'Scheduler::State:\s*(\{.*?^})', re.DOTALL | re.MULTILINE), Fore.MAGENTA)
+        ]
+
+        def __init__(self, workers, *args, **kwargs):
+            self.highlights = kwargs.pop('highlights', []) or self.DEFAULT_HIGHLIGHTS
+            self.verbose = kwargs.pop('verbose', False)
+            super().__init__(*args, **kwargs)
+
+        def _apply_highlights(self, message):
+            out = message
+            for pattern, color in self.highlights:
+                def replace(m):
+                    text = m.group(0)
+                    return f'{color}{text}{Style.RESET_ALL}{Fore.WHITE}'
+                out = pattern.sub(replace, out)
+            return out
+
+        def format(self, record):
+            timestamp = self.formatTime(record, self.datefmt)
+            level_color = self.LEVEL_COLORS.get(record.levelno, Fore.WHITE)
+
+            raw_msg = record.getMessage()
+            colored_msg = self._apply_highlights(raw_msg)
+
+            # only log thread name if it's not MainThread
+            # to avoid cluttering the logs with 'MainThread' entries
+            if record.threadName == 'MainThread':
+                thread_name = ''
+            else:
+                thread_name = f'[{record.threadName}] '
+
+            if self.verbose:
+                msg = (
+                    f"{Style.DIM}{timestamp}{Style.RESET_ALL} "
+                    f"{level_color}{record.levelname:<5}{Style.RESET_ALL} "
+                    f"{thread_name}"
+                    f"{Fore.WHITE}{record.funcName}: {colored_msg}{Style.RESET_ALL}"
+                )
+            else:
+                msg = (
+                    f"{Style.DIM}{timestamp}{Style.RESET_ALL} "
+                    f"{thread_name}"
+                    f"{colored_msg}{Style.RESET_ALL}"
+                )
+            if record.exc_info:
+                msg += '\n' + self.formatException(record.exc_info)
+
+            return msg
+
+class MainThreadAwareFormatter(logging.Formatter):
+
+    def __init__(self, main_fmt, thread_fmt, workers, thread_prefix='thread_', *args, **kwargs):
+        super().__init__(fmt=main_fmt, *args, **kwargs)
+        self.main_fmt = main_fmt
+        self.thread_fmt = thread_fmt
+        self.thread_prefix = thread_prefix
+
+    def format(self, record):
+        if record.threadName == 'MainThread':
+            self._style._fmt = self.main_fmt
+        else:
+            self._style._fmt = self.thread_fmt
+        return super().format(record)
+
+class ThreadProxyLogger:
+    def __getattr__(self, name):
+        return getattr(logging.getLogger(threading.current_thread().name), name)
+
+def configure_logging(workers, prefix='thread', add_stream_handler=False, highlights=None,
+                      verbose=False):
+
+    root_logger = logging.getLogger()
+    if getattr(root_logger, '_logging_initialized', False):
+        return
+
+    root_logger.setLevel(logging.DEBUG)
+    root_logger.handlers.clear()
+
+    file_formatter = logging.Formatter(
+        '%(asctime)s %(levelname)-5s [%(threadName)s] %(funcName)s: %(message)s')
+
+    if add_stream_handler or verbose:
+        if HAS_COLOR:
+            init(autoreset=False)
+            stream_formatter = ColoredFormatter(workers, highlights=highlights, verbose=verbose)
+        else:
+            main_fmt = '%(asctime)s %(message)s'
+            thread_fmt = '%(asctime)s [%(threadName)s] %(message)s'
+            stream_formatter = MainThreadAwareFormatter(main_fmt, thread_fmt, workers)
+
+        stream_handler = logging.StreamHandler(sys.stderr)
+        stream_handler.setFormatter(stream_formatter)
+        stream_handler.setLevel(logging.DEBUG if verbose else logging.INFO)
+        root_logger.addHandler(stream_handler)
+
+        root_logger._logging_initialized = True
+
+    base = os.path.splitext(os.path.basename(sys.argv[0]))[0]
+
+    def add_handler(name):
+        filename = f'{base}_{name}.log'
+        logger = logging.getLogger(name)
+        if not any(
+            isinstance(handler, logging.FileHandler)
+            and getattr(handler, 'baseFilename', '').endswith(filename)
+            for handler in logger.handlers
+        ):
+            fhandler = logging.FileHandler(filename, mode='a', encoding='utf-8')
+            fhandler.setLevel(logging.DEBUG)
+            fhandler.setFormatter(file_formatter)
+            logger.addHandler(fhandler)
+        logger.setLevel(logging.DEBUG)
+        return logger
+
+    add_handler(threading.current_thread().name)
+    for index in range(workers):
+        add_handler(f'{prefix}_{index}')