scriptplan 0.9.0__py3-none-any.whl

scriptplan/scheduler/batch_processor.py

@@ -0,0 +1,238 @@
+"""BatchProcessor module for parallel job execution.
+
+The BatchProcessor class can be used to run code blocks of the program as
+separate processes. Multiple pieces of code can be submitted to be executed
+in parallel. The number of CPU cores to use is limited at object creation time.
+"""
+
+import threading
+import multiprocessing
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, as_completed
+from typing import Callable, Any, Optional, List, Dict
+from dataclasses import dataclass, field
+
+
+@dataclass
+class JobInfo:
+    """Storage container for batch job related information.
+
+    Contains job id, process id, stdout/stderr data and return value.
+    """
+
+    job_id: int
+    func: Callable
+    tag: Any = None
+    pid: Optional[int] = None
+    ret_val: Optional[int] = None
+    stdout: str = ''
+    stderr: str = ''
+    args: tuple = field(default_factory=tuple)
+    kwargs: dict = field(default_factory=dict)
+
+    @property
+    def jobId(self):
+        """Alias for Ruby compatibility."""
+        return self.job_id
+
+    @property
+    def retVal(self):
+        """Alias for Ruby compatibility."""
+        return self.ret_val
+
+
+def _worker_function(func, args, kwargs):
+    """Worker function that runs in subprocess and returns result."""
+    try:
+        result = func(*args, **kwargs)
+        return (0, result, '', '')
+    except Exception as e:
+        import traceback
+        return (1, None, '', traceback.format_exc())
+
+
+class BatchProcessor:
+    """Run code blocks in parallel processes.
+
+    Submitted jobs are queued and scheduled to the given number of CPUs.
+    Usage:
+        1. Create a BatchProcessor object with max CPU cores
+        2. Use queue() to submit jobs
+        3. Use wait() to wait for completion and process results
+    """
+
+    def __init__(self, max_cpu_cores: int = None):
+        """Create a BatchProcessor object.
+
+        Args:
+            max_cpu_cores: Maximum number of simultaneous processes.
+                          Defaults to CPU count.
+        """
+        if max_cpu_cores is None:
+            max_cpu_cores = multiprocessing.cpu_count()
+        self._max_cpu_cores = max_cpu_cores
+
+        self._to_run_queue: List[JobInfo] = []
+        self._running_jobs: Dict[int, JobInfo] = {}
+        self._completed_jobs: List[JobInfo] = []
+
+        self._lock = threading.Lock()
+        self._jobs_in = 0
+        self._jobs_out = 0
+
+        self._executor = None
+
+    @property
+    def maxCpuCores(self):
+        """Return the maximum number of CPU cores to use."""
+        return self._max_cpu_cores
+
+    def queue(self, tag: Any = None, func: Callable = None, *args, **kwargs):
+        """Add a new job to the job queue.
+
+        Args:
+            tag: Optional data to identify the job upon completion.
+            func: The function to execute in a separate process.
+            *args: Positional arguments to pass to the function.
+            **kwargs: Keyword arguments to pass to the function.
+        """
+        with self._lock:
+            if self._jobs_out > 0:
+                raise RuntimeError("You cannot call queue() while wait() is running!")
+
+            job = JobInfo(
+                job_id=self._jobs_in,
+                func=func,
+                tag=tag,
+                args=args,
+                kwargs=kwargs
+            )
+            self._jobs_in += 1
+            self._to_run_queue.append(job)
+
+    def wait(self, callback: Callable[[JobInfo], None] = None):
+        """Wait for all jobs to complete.
+
+        Args:
+            callback: Optional function called with each JobInfo as jobs complete.
+        """
+        if self._jobs_in == 0:
+            return
+
+        # Create executor
+        self._executor = ProcessPoolExecutor(max_workers=self._max_cpu_cores)
+
+        try:
+            # Submit all jobs
+            futures = {}
+            for job in self._to_run_queue:
+                future = self._executor.submit(
+                    _worker_function, job.func, job.args, job.kwargs
+                )
+                futures[future] = job
+
+            # Wait for completion and process results
+            for future in as_completed(futures):
+                job = futures[future]
+                try:
+                    ret_code, result, stdout, stderr = future.result()
+                    job.ret_val = ret_code
+                    job.stdout = stdout
+                    job.stderr = stderr
+                except Exception as e:
+                    job.ret_val = 1
+                    job.stderr = str(e)
+
+                self._jobs_out += 1
+                self._completed_jobs.append(job)
+
+                if callback:
+                    callback(job)
+
+        finally:
+            self._executor.shutdown(wait=True)
+            self._executor = None
+
+        # Reset for reuse
+        self._to_run_queue.clear()
+        self._running_jobs.clear()
+        self._completed_jobs.clear()
+        self._jobs_in = 0
+        self._jobs_out = 0
+
+    def cancel(self):
+        """Cancel all pending jobs."""
+        if self._executor:
+            self._executor.shutdown(wait=False, cancel_futures=True)
+            self._executor = None
+
+
+class ThreadBatchProcessor:
+    """Thread-based batch processor for lighter workloads.
+
+    Uses threads instead of processes, suitable for I/O-bound tasks.
+    """
+
+    def __init__(self, max_threads: int = None):
+        """Create a ThreadBatchProcessor object.
+
+        Args:
+            max_threads: Maximum number of simultaneous threads.
+                        Defaults to CPU count * 5.
+        """
+        if max_threads is None:
+            max_threads = multiprocessing.cpu_count() * 5
+        self._max_threads = max_threads
+
+        self._to_run_queue: List[JobInfo] = []
+        self._jobs_in = 0
+        self._jobs_out = 0
+        self._lock = threading.Lock()
+        self._executor = None
+
+    def queue(self, tag: Any = None, func: Callable = None, *args, **kwargs):
+        """Add a new job to the job queue."""
+        with self._lock:
+            job = JobInfo(
+                job_id=self._jobs_in,
+                func=func,
+                tag=tag,
+                args=args,
+                kwargs=kwargs
+            )
+            self._jobs_in += 1
+            self._to_run_queue.append(job)
+
+    def wait(self, callback: Callable[[JobInfo], None] = None):
+        """Wait for all jobs to complete."""
+        if self._jobs_in == 0:
+            return
+
+        self._executor = ThreadPoolExecutor(max_workers=self._max_threads)
+
+        try:
+            futures = {}
+            for job in self._to_run_queue:
+                future = self._executor.submit(job.func, *job.args, **job.kwargs)
+                futures[future] = job
+
+            for future in as_completed(futures):
+                job = futures[future]
+                try:
+                    result = future.result()
+                    job.ret_val = 0
+                except Exception as e:
+                    job.ret_val = 1
+                    job.stderr = str(e)
+
+                self._jobs_out += 1
+
+                if callback:
+                    callback(job)
+
+        finally:
+            self._executor.shutdown(wait=True)
+            self._executor = None
+
+        self._to_run_queue.clear()
+        self._jobs_in = 0
+        self._jobs_out = 0

scriptplan/scheduler/scoreboard.py

@@ -0,0 +1,120 @@
+import math
+from scriptplan.utils.time import TimeInterval
+
+class Scoreboard:
+    def __init__(self, start, end, granularity, init_val=None):
+        self.startDate = start
+        self.endDate = end
+        self.resolution = granularity
+        
+        # Calculate size
+        # Ruby: ((endDate - startDate) / resolution).ceil + 1
+        diff = (end - start).total_seconds() if hasattr(end - start, 'total_seconds') else (end - start)
+        self.size = math.ceil(diff / granularity) + 1
+        
+        self.clear(init_val)
+
+    def clear(self, init_val=None):
+        self.sb = [init_val] * self.size
+
+    def idxToDate(self, idx, forceIntoProject=False):
+        if forceIntoProject:
+            if idx < 0:
+                return self.startDate
+            if idx >= self.size:
+                return self.endDate
+        elif idx < 0 or idx >= self.size:
+            raise IndexError(f"Index {idx} is out of scoreboard range ({self.size - 1})")
+        
+        from datetime import timedelta
+        return self.startDate + timedelta(seconds=idx * self.resolution)
+
+    def dateToIdx(self, date, forceIntoProject=True):
+        diff = (date - self.startDate).total_seconds() if hasattr(date - self.startDate, 'total_seconds') else (date - self.startDate)
+        idx = int(diff / self.resolution)
+        
+        if forceIntoProject:
+            if idx < 0: return 0
+            if idx >= self.size: return self.size - 1
+        elif idx < 0 or idx >= self.size:
+            raise IndexError(f"Date {date} is out of project time range ({self.startDate} - {self.endDate})")
+        
+        return idx
+
+    def each(self, startIdx=0, endIdx=None):
+        if endIdx is None:
+            endIdx = self.size
+        
+        if startIdx != 0 or endIdx != self.size:
+            for i in range(startIdx, endIdx):
+                yield self.sb[i]
+        else:
+            for entry in self.sb:
+                yield entry
+
+    def each_index(self):
+        for i in range(len(self.sb)):
+            yield i
+
+    def collect(self, func):
+        for i in range(len(self.sb)):
+            self.sb[i] = func(self.sb[i])
+
+    def __getitem__(self, idx):
+        return self.sb[idx]
+
+    def __setitem__(self, idx, value):
+        self.sb[idx] = value
+
+    def get(self, date):
+        return self.sb[self.dateToIdx(date)]
+
+    def set(self, date, value):
+        self.sb[self.dateToIdx(date)] = value
+
+    def collectIntervals(self, iv, minDuration, predicate):
+        startIdx = self.dateToIdx(iv.start)
+        endIdx = self.dateToIdx(iv.end)
+        sIdx = startIdx
+        eIdx = endIdx
+        
+        minDurationSlots = int(minDuration / self.resolution)
+        if minDurationSlots <= 0:
+            minDurationSlots = 1
+            
+        startIdx -= minDurationSlots
+        if startIdx < 0: startIdx = 0
+        endIdx += minDurationSlots
+        if endIdx > self.size - 1: endIdx = self.size - 1
+        
+        intervals = []
+        duration = 0
+        start = 0
+        
+        idx = startIdx
+        while idx <= endIdx:
+            # yield/predicate check
+            val = self.sb[idx] if idx < len(self.sb) else None # Boundary check
+            if predicate(val) and idx < endIdx:
+                if start == 0:
+                    start = idx
+                duration += 1
+            else:
+                if duration > 0:
+                    if duration >= minDurationSlots:
+                        if start < sIdx: start = sIdx
+                        current_idx = idx
+                        if current_idx > eIdx: current_idx = eIdx
+                        
+                        intervals.append(TimeInterval(self.idxToDate(start), self.idxToDate(current_idx)))
+                    duration = 0
+                    start = 0
+            idx += 1
+            
+        return intervals
+
+    def __iter__(self):
+        return iter(self.sb)
+    
+    def __len__(self):
+        return self.size

scriptplan/utils/data_cache.py

@@ -0,0 +1,46 @@
+class DataCache:
+    _instance = None
+    
+    def __init__(self):
+        self._cache = {}
+
+    @classmethod
+    def instance(cls):
+        if cls._instance is None:
+            cls._instance = DataCache()
+        return cls._instance
+    
+    def flush(self):
+        self._cache = {}
+
+    def cached(self, obj, tag, *args, **kwargs):
+        # Simple caching implementation
+        # Key based on object id, tag, and args
+        key = (id(obj), tag, args, tuple(kwargs.items()))
+        if key in self._cache:
+            return self._cache[key]
+        
+        # If block is passed? 
+        # In Python, we can't pass a block like Ruby.
+        # We expect the last argument or a specific argument to be a callable if used like Ruby's block.
+        # But here the caller is likely doing: @dCache.cached(...) do ... end
+        # In Python: dCache.cached(..., lambda: ...)
+        # So the last arg might be the function to execute.
+        
+        # However, treeSumR implementation:
+        # @dCache.cached(self, cacheTag, startIdx, endIdx, *args) do ... end
+        
+        # In Python `ResourceScenario.treeSumR`:
+        # self.dCache.cached(self, cacheTag, startIdx, endIdx, *args, lambda: ...) ?
+        
+        # I haven't implemented treeSumR in Python yet.
+        # If I did, I would pass a callable.
+        
+        # Let's assume the last argument is the callable if it's a function.
+        # Or better, explicit 'calculator' argument.
+        
+        # But since I haven't implemented treeSumR fully in ResourceScenario, this is future proofing.
+        pass
+
+class FileList(list):
+    pass

scriptplan/utils/logger.py

@@ -0,0 +1,243 @@
+"""Log module implementing segmented execution traces.
+
+The Log class implements a filter for segmented execution traces. The
+trace messages are filtered based on their segment name and the nesting
+level of the segments. The class uses a Singleton pattern.
+"""
+
+import sys
+import threading
+from typing import List, Callable, Optional
+
+
+class ANSIColor:
+    """ANSI color codes for terminal output."""
+    GREEN = '\033[32m'
+    RED = '\033[31m'
+    YELLOW = '\033[33m'
+    BLUE = '\033[34m'
+    RESET = '\033[0m'
+
+    @classmethod
+    def green(cls, text: str) -> str:
+        return f"{cls.GREEN}{text}{cls.RESET}"
+
+    @classmethod
+    def red(cls, text: str) -> str:
+        return f"{cls.RED}{text}{cls.RESET}"
+
+    @classmethod
+    def yellow(cls, text: str) -> str:
+        return f"{cls.YELLOW}{text}{cls.RESET}"
+
+    @classmethod
+    def blue(cls, text: str) -> str:
+        return f"{cls.BLUE}{text}{cls.RESET}"
+
+
+class Log:
+    """Singleton class for segmented execution trace logging.
+
+    The trace messages are filtered based on their segment name and the nesting
+    level of the segments.
+    """
+
+    _instance = None
+    _lock = threading.Lock()
+
+    # Class-level variables (equivalent to Ruby's @@)
+    _level = 0
+    _stack: List[str] = []
+    _segments: List[str] = []
+    _silent = True
+    _progress = 0
+    _progressMeter = ''
+
+    def __new__(cls):
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:
+                    cls._instance = super().__new__(cls)
+        return cls._instance
+
+    @classmethod
+    def get_level(cls) -> int:
+        """Get the current log level."""
+        return cls._level
+
+    @classmethod
+    def set_level(cls, level: int):
+        """Set the maximum nesting level that should be shown.
+
+        Segments with a nesting level greater than level will be silently dropped.
+        """
+        cls._level = level
+
+    # Property-style accessors
+    level = property(lambda cls: cls._level)
+
+    @classmethod
+    def get_segments(cls) -> List[str]:
+        """Get the current segment filter list."""
+        return cls._segments
+
+    @classmethod
+    def set_segments(cls, segments: List[str]):
+        """Set the segment filter list.
+
+        Messages not in these segments will be ignored. Messages from segments
+        that are nested into the shown segments will be shown for the next
+        _level nested segments.
+        """
+        cls._segments = segments
+
+    @classmethod
+    def get_silent(cls) -> bool:
+        """Get the silent mode status."""
+        return cls._silent
+
+    @classmethod
+    def set_silent(cls, silent: bool):
+        """Set silent mode. If True, progress information will not be shown."""
+        cls._silent = silent
+
+    @classmethod
+    def enter(cls, segment: str, message: str):
+        """Open a new segment.
+
+        Args:
+            segment: The name of the segment.
+            message: A description of the segment.
+        """
+        if cls._level == 0:
+            return
+
+        cls._stack.append(segment)
+        cls.msg(lambda: f">> [{segment}] {message}")
+
+    @classmethod
+    def exit(cls, segment: str, message: Optional[str] = None):
+        """Close an open segment.
+
+        Will search the stack of open segments for a segment with that name
+        and will close all nested segments as well.
+
+        Args:
+            segment: The name of the segment to close.
+            message: Optional exit message.
+        """
+        if cls._level == 0:
+            return
+
+        if message:
+            cls.msg(lambda: f"<< [{segment}] {message}")
+
+        if segment in cls._stack:
+            while cls._stack:
+                m = cls._stack.pop()
+                if m == segment:
+                    break
+
+    @classmethod
+    def msg(cls, message_func: Callable[[], str]):
+        """Show a log message within the currently active segment.
+
+        The message is the result of the passed callable. The callable will
+        only be evaluated if the message will actually be shown.
+
+        Args:
+            message_func: A callable that returns the message string.
+        """
+        if cls._level == 0:
+            return
+
+        offset = 0
+        if cls._segments:
+            showMessage = False
+            for segment in cls._stack:
+                if segment in cls._segments:
+                    offset = cls._stack.index(segment)
+                    showMessage = True
+                    break
+            if not showMessage:
+                return
+
+        if len(cls._stack) - offset < cls._level:
+            indent = ' ' * (len(cls._stack) - offset)
+            print(indent + message_func(), file=sys.stderr)
+
+    @classmethod
+    def status(cls, message: str):
+        """Print out a status message unless in silent mode."""
+        if cls._silent:
+            return
+        print(message)
+
+    @classmethod
+    def startProgressMeter(cls, text: str):
+        """Start the progress meter display or change the info text.
+
+        While the meter is active, the text cursor is always returned to
+        the start of the same line.
+        """
+        if cls._silent:
+            return
+
+        maxlen = 60
+        text = text.ljust(maxlen)
+        if len(text) > maxlen:
+            text = text[:maxlen]
+        cls._progressMeter = text
+        print(f"{cls._progressMeter} ...", end='\r')
+        sys.stdout.flush()
+
+    @classmethod
+    def stopProgressMeter(cls):
+        """Set the progress meter status to 'done' and move to the next line."""
+        if cls._silent:
+            return
+        print(f"{cls._progressMeter} [      {ANSIColor.green('Done')}      ]")
+
+    @classmethod
+    def activity(cls):
+        """Update the progress indicator to the next symbol.
+
+        May only be called after startProgressMeter.
+        """
+        if cls._silent:
+            return
+
+        indicator = ['-', '\\', '|', '/']
+        cls._progress = (cls._progress + 1) % len(indicator)
+        print(f"{cls._progressMeter} [{indicator[cls._progress]}]", end='\r')
+        sys.stdout.flush()
+
+    @classmethod
+    def progress(cls, percent: float):
+        """Update the progress bar to the given percent completion.
+
+        May only be called after startProgressMeter.
+
+        Args:
+            percent: Completion value between 0.0 and 1.0.
+        """
+        if cls._silent:
+            return
+
+        percent = max(0.0, min(1.0, percent))
+        cls._progress = percent
+
+        length = 16
+        full = int(length * percent)
+        bar = '=' * full + ' ' * (length - full)
+        label = f"{int(percent * 100.0)}%"
+        start = length // 2 - len(label) // 2
+        bar = bar[:start] + label + bar[start + len(label):]
+        print(f"{cls._progressMeter} [{ANSIColor.green(bar)}]", end='\r')
+        sys.stdout.flush()
+
+
+# Convenience function to get the singleton instance
+def get_logger() -> Log:
+    """Return the Log singleton instance."""
+    return Log()