graflag-runner 1.0.0__tar.gz

graflag_runner-1.0.0/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.4
+Name: graflag_runner
+Version: 1.0.0
+Summary: Framework for executing graph anomaly detection methods with resource monitoring
+Author: GraFlag Team
+Requires-Python: >=3.7
+Requires-Dist: psutil>=5.8.0
+Dynamic: author
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

graflag_runner-1.0.0/__init__.py

@@ -0,0 +1,36 @@
+"""
+GraFlag Runner - Framework for executing graph anomaly detection methods.
+
+This package provides:
+- MethodRunner: Main execution wrapper with resource monitoring
+- ResourceMonitor: Real-time CPU, memory, and GPU tracking
+- ResultWriter: Simple API for methods to save standardized results
+- StreamableArray: Wrapper for memory-efficient streaming of large arrays
+- subprocess_utils: Utilities for running subprocesses with real-time output
+- logging: Simple logging functions (debug, info, warning, error, critical, exception)
+"""
+
+from .runner import MethodRunner
+from .results import ResultWriter
+from .streaming import StreamableArray, stream_write_json
+from .subprocess_utils import (
+    run_with_realtime_output,
+    run_command_list,
+    save_output_to_file
+)
+from .logging_utils import debug, info, warning, error, critical, exception
+
+__version__ = "1.0.0"
+__all__ = [
+    "MethodRunner",
+    "ResultWriter",
+    "run_with_realtime_output",
+    "run_command_list",
+    "save_output_to_file",
+    "debug",
+    "info",
+    "warning",
+    "error",
+    "critical",
+    "exception"
+]

graflag_runner-1.0.0/__main__.py

@@ -0,0 +1,6 @@
+"""Allow running as module: python -m graflag_runner"""
+
+from .runner import main
+
+if __name__ == "__main__":
+    main()

graflag_runner-1.0.0/graflag_runner.egg-info/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.4
+Name: graflag_runner
+Version: 1.0.0
+Summary: Framework for executing graph anomaly detection methods with resource monitoring
+Author: GraFlag Team
+Requires-Python: >=3.7
+Requires-Dist: psutil>=5.8.0
+Dynamic: author
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

graflag_runner-1.0.0/graflag_runner.egg-info/SOURCES.txt

@@ -0,0 +1,25 @@
+__init__.py
+__main__.py
+logging_utils.py
+monitor.py
+results.py
+runner.py
+setup.py
+streaming.py
+subprocess_utils.py
+write_utils.py
+./__init__.py
+./__main__.py
+./logging_utils.py
+./monitor.py
+./results.py
+./runner.py
+./streaming.py
+./subprocess_utils.py
+./write_utils.py
+graflag_runner.egg-info/PKG-INFO
+graflag_runner.egg-info/SOURCES.txt
+graflag_runner.egg-info/dependency_links.txt
+graflag_runner.egg-info/entry_points.txt
+graflag_runner.egg-info/requires.txt
+graflag_runner.egg-info/top_level.txt

graflag_runner-1.0.0/graflag_runner.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

graflag_runner-1.0.0/graflag_runner.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+graflag-run = graflag_runner.runner:main

graflag_runner-1.0.0/graflag_runner.egg-info/requires.txt

@@ -0,0 +1 @@
+psutil>=5.8.0

graflag_runner-1.0.0/graflag_runner.egg-info/top_level.txt

@@ -0,0 +1 @@
+graflag_runner

graflag_runner-1.0.0/logging_utils.py

@@ -0,0 +1,47 @@
+"""
+Logging utilities for GraFlag methods.
+
+Provides consistent logging functions across all methods.
+"""
+
+import logging as _logging
+import os
+
+# Configure logging once at module import
+_method_name = os.environ.get("METHOD_NAME", "unknown_method")
+_logging.basicConfig(
+    level=_logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    force=True
+)
+_logger = _logging.getLogger(_method_name)
+
+
+def debug(msg, *args, **kwargs):
+    """Log a debug message."""
+    _logger.debug(msg, *args, **kwargs)
+
+
+def info(msg, *args, **kwargs):
+    """Log an info message."""
+    _logger.info(msg, *args, **kwargs)
+
+
+def warning(msg, *args, **kwargs):
+    """Log a warning message."""
+    _logger.warning(msg, *args, **kwargs)
+
+
+def error(msg, *args, **kwargs):
+    """Log an error message."""
+    _logger.error(msg, *args, **kwargs)
+
+
+def critical(msg, *args, **kwargs):
+    """Log a critical message."""
+    _logger.critical(msg, *args, **kwargs)
+
+
+def exception(msg, *args, **kwargs):
+    """Log an exception message with traceback."""
+    _logger.exception(msg, *args, **kwargs)

graflag_runner-1.0.0/monitor.py

@@ -0,0 +1,157 @@
+"""Resource monitoring for method execution."""
+
+import os
+import time
+import psutil
+import subprocess
+from typing import Optional, Dict
+from pathlib import Path
+import logging
+
+logger = logging.getLogger(__name__)
+
+# Import ResultWriter for spot() functionality
+from .results import ResultWriter
+
+
+class ResourceMonitor:
+    """Monitor CPU, memory, and GPU usage during method execution."""
+    
+    def __init__(self, pid: Optional[int] = None):
+        """
+        Initialize resource monitor.
+        
+        Args:
+            pid: Process ID to monitor (None = current process)
+        """
+        self.pid = pid or os.getpid()
+        self.monitoring = False
+        self.peak_memory_mb = 0
+        self.peak_gpu_mb = 0
+        self.has_gpu = self._check_gpu()
+        
+        # System totals
+        self.total_memory_mb = psutil.virtual_memory().total / (1024 * 1024)
+        self.total_gpu_mb = self._get_total_gpu_memory() if self.has_gpu else 0
+        
+        # ResultWriter for spot() functionality
+        self.result_writer = ResultWriter()
+        
+    def _check_gpu(self) -> bool:
+        """Check if NVIDIA GPU is available."""
+        try:
+            subprocess.run(
+                ["nvidia-smi"], 
+                stdout=subprocess.DEVNULL, 
+                stderr=subprocess.DEVNULL,
+                timeout=2
+            )
+            return True
+        except (FileNotFoundError, subprocess.TimeoutExpired):
+            return False
+    
+    def _get_total_gpu_memory(self) -> float:
+        """Get total GPU memory in MB."""
+        if not self.has_gpu:
+            return 0
+        try:
+            result = subprocess.run(
+                ["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"],
+                capture_output=True,
+                text=True,
+                timeout=2
+            )
+            return float(result.stdout.strip().split('\n')[0])
+        except Exception:
+            return 0
+    
+    def _get_process_memory(self) -> float:
+        """Get current process memory usage in MB (including children)."""
+        try:
+            process = psutil.Process(self.pid)
+            # Get memory for main process
+            memory = process.memory_info().rss / (1024 * 1024)
+            # Add memory for all child processes
+            for child in process.children(recursive=True):
+                try:
+                    memory += child.memory_info().rss / (1024 * 1024)
+                except (psutil.NoSuchProcess, psutil.AccessDenied):
+                    continue
+            return memory
+        except (psutil.NoSuchProcess, psutil.AccessDenied):
+            return 0
+    
+    def _get_gpu_memory(self) -> float:
+        """Get GPU memory usage in MB (all processes in container)."""
+        if not self.has_gpu:
+            return 0
+        try:
+            # In a container, we're the only workload, so track total GPU usage
+            # This is more reliable than trying to track specific PIDs since
+            # PyTorch/CUDA may spawn processes that aren't direct children
+            result = subprocess.run(
+                ["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"],
+                capture_output=True,
+                text=True,
+                timeout=2
+            )
+            
+            if result.returncode == 0 and result.stdout.strip():
+                return float(result.stdout.strip().split('\n')[0])
+            
+            return 0.0
+        except Exception as e:
+            logger.debug(f"Failed to get GPU memory: {e}")
+            return 0
+    
+    def start_monitoring(self, interval: float = 1.0):
+        """
+        Start background monitoring loop.
+        
+        Args:
+            interval: Monitoring interval in seconds
+        """
+        self.monitoring = True
+        
+        logger.info(f"[INFO] Resource monitoring started (PID: {self.pid})")
+        logger.info(f"   Total memory: {self.total_memory_mb:.0f}MB")
+        if self.has_gpu:
+            logger.info(f"   Total GPU memory: {self.total_gpu_mb:.0f}MB")
+        else:
+            logger.info("   GPU: Not available")
+        
+        while self.monitoring:
+            current_memory = self._get_process_memory()
+            current_gpu = self._get_gpu_memory()
+            
+            # Update peaks
+            self.peak_memory_mb = max(self.peak_memory_mb, current_memory)
+            self.peak_gpu_mb = max(self.peak_gpu_mb, current_gpu)
+            
+            # Log to CSV using spot() method
+            if self.result_writer:
+                self.result_writer.spot(
+                    "resources",
+                    memory_mb=round(current_memory, 2),
+                    gpu_mb=round(current_gpu, 2)
+                )
+            
+            time.sleep(interval)
+    
+    def stop_monitoring(self):
+        """Stop background monitoring."""
+        self.monitoring = False
+    
+    def get_summary(self) -> Dict[str, float]:
+        """
+        Get resource usage summary.
+        
+        Returns:
+            Dictionary with peak memory and GPU usage
+        """
+        return {
+            "peak_memory_mb": round(self.peak_memory_mb, 2),
+            "peak_gpu_mb": round(self.peak_gpu_mb, 2) if self.has_gpu else None,
+            "total_memory_mb": round(self.total_memory_mb, 2),
+            "total_gpu_mb": round(self.total_gpu_mb, 2) if self.has_gpu else None,
+        }

graflag_runner-1.0.0/results.py

@@ -0,0 +1,255 @@
+"""Result management and standardization."""
+
+import json
+import csv
+import os
+import time
+from pathlib import Path
+from typing import List, Dict, Any, Optional, Union, Iterator
+from collections import OrderedDict
+import logging
+
+from .streaming import StreamableArray, stream_write_json
+
+logger = logging.getLogger(__name__)
+
+
+class ResultWriter:
+    """
+    Simple API for methods to save standardized results.
+    
+    Usage in method code:
+        from graflag_runner import ResultWriter
+        
+        writer = ResultWriter()
+        writer.save_scores(
+            result_type="TEMPORAL_NODE_SCORES",
+            scores=[[0.1, 0.2], [0.3, 0.4]],
+            timestamps=[0, 1],
+            node_ids=[0, 1]
+        )
+        writer.add_metadata(method_name="TADDY", dataset="uci")
+        writer.finalize()
+    """
+    
+    VALID_RESULT_TYPES = {
+        "NODE_ANOMALY_SCORES",
+        "EDGE_ANOMALY_SCORES",
+        "GRAPH_ANOMALY_SCORES",
+        "TEMPORAL_NODE_ANOMALY_SCORES",
+        "TEMPORAL_EDGE_ANOMALY_SCORES",
+        "TEMPORAL_GRAPH_ANOMALY_SCORES",
+        "NODE_STREAM_ANOMALY_SCORES",
+        "EDGE_STREAM_ANOMALY_SCORES",
+        "GRAPH_STREAM_ANOMALY_SCORES",
+    }
+    
+    def __init__(self):
+        """
+        Initialize result writer.
+        
+        Args:
+            output_dir: Directory to save results.json
+        """
+        self.output_dir = Path(os.environ.get("EXP"))
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        
+        self.results = {
+            "result_type": None,
+            "scores": None,
+            "metadata": {}
+        }
+        
+        # Schema tracking for spot() method
+        self._spot_schemas: Dict[str, OrderedDict] = {}
+    
+    def save_scores(
+        self,
+        result_type: str,
+        scores: Union[List, StreamableArray, Iterator],
+        **kwargs
+    ):
+        """
+        Save anomaly scores with specified result type.
+        
+        Supports both regular lists and streaming for large datasets:
+        - Regular list/array: scores = [[...], [...], ...]
+        - Generator: scores = StreamableArray(generate_rows())
+        - Raw iterator: Will be wrapped in StreamableArray automatically
+        
+        Args:
+            result_type: One of VALID_RESULT_TYPES
+            scores: Anomaly scores (list, StreamableArray, or generator)
+                   Can be a generator for memory-efficient handling of large arrays
+            **kwargs: Additional fields (timestamps, node_ids, edges, etc.)
+        """
+        if result_type not in self.VALID_RESULT_TYPES:
+            raise ValueError(
+                f"Invalid result_type: {result_type}. "
+                f"Must be one of {self.VALID_RESULT_TYPES}"
+            )
+        
+        self.results["result_type"] = result_type
+        
+        # Wrap raw generators/iterators in StreamableArray
+        if hasattr(scores, '__iter__') and hasattr(scores, '__next__'):
+            if not isinstance(scores, StreamableArray):
+                scores = StreamableArray(scores)
+                logger.info("[INFO] Wrapped generator in StreamableArray for streaming")
+        
+        self.results["scores"] = scores
+        
+        # Add optional fields
+        for key, value in kwargs.items():
+            self.results[key] = value
+        
+        if isinstance(scores, StreamableArray):
+            logger.info(f"[OK] Streamable scores registered: {result_type}")
+        else:
+            logger.info(f"[OK] Scores saved: {result_type}")
+    
+    def add_metadata(self, **kwargs):
+        """
+        Add metadata fields.
+        
+        Args:
+            **kwargs: Metadata key-value pairs (method_name, dataset, etc.)
+        """
+        self.results["metadata"].update(kwargs)
+    
+    def add_resource_metrics(
+        self,
+        exec_time_ms: float,
+        peak_memory_mb: float,
+        peak_gpu_mb: Optional[float] = None
+    ):
+        """
+        Add resource consumption metrics.
+        
+        Args:
+            exec_time_ms: Execution time in milliseconds
+            peak_memory_mb: Peak memory usage in MB
+            peak_gpu_mb: Peak GPU memory in MB (optional)
+        """
+        self.results["metadata"]["exec_time_ms"] = round(exec_time_ms, 2)
+        self.results["metadata"]["peak_memory_mb"] = round(peak_memory_mb, 2)
+        if peak_gpu_mb is not None:
+            self.results["metadata"]["peak_gpu_mb"] = round(peak_gpu_mb, 2)
+    
+    def finalize(self) -> Path:
+        """
+        Write results to results.json file.
+        
+        Uses streaming for large score arrays to avoid memory issues.
+        Regular lists are written normally, StreamableArray objects are
+        written row-by-row without loading the entire array into memory.
+        
+        Returns:
+            Path to results.json
+        """
+        output_file = self.output_dir / "results.json"
+        
+        # Validation
+        if self.results["result_type"] is None:
+            raise ValueError("No scores saved. Call save_scores() first.")
+        
+        # Check if we need streaming
+        has_streamable = isinstance(self.results.get("scores"), StreamableArray)
+        
+        if has_streamable:
+            logger.info("[INFO] Writing results with streaming (large data)...")
+            stream_write_json(self.results, output_file)
+        else:
+            # Regular JSON dump for small data
+            logger.info("[INFO] Writing results (standard)...")
+            with open(output_file, 'w') as f:
+                json.dump(self.results, f, indent=2)
+        
+        logger.info(f"[OK] Results written to: {output_file}")
+        return output_file
+    
+    def spot(self, metric_key: str, **metrics):
+        """
+        Track real-time metrics to a CSV file with schema validation.
+        
+        This method is used for monitoring progress during training/execution:
+        - Creates a CSV file named "{metric_key}.csv" in the output directory
+        - First column is always "timestamp" (Unix timestamp)
+        - Subsequent columns are the metric keys provided in **metrics
+        - Schema is locked after first call - subsequent calls must have same keys
+        - Automatically appends new rows on each call
+        
+        Args:
+            metric_key: Identifier for the metric group (e.g., "training", "validation", "resources")
+                       Used as the CSV filename: "{metric_key}.csv"
+            **metrics: Metric key-value pairs to record (e.g., loss=0.5, auc=0.85)
+        
+        Raises:
+            ValueError: If schema changes after first call (different metric keys)
+        
+        Examples:
+            # Track training metrics
+            writer.spot("training", epoch=1, loss=0.5, auc=0.85)
+            writer.spot("training", epoch=2, loss=0.3, auc=0.90)  # Must have same keys
+            
+            # Track resource usage
+            writer.spot("resources", memory_mb=512.5, gpu_mb=2048.0)
+            
+            # Track validation metrics separately
+            writer.spot("validation", epoch=1, val_loss=0.6, val_auc=0.82)
+        """
+        if not metrics:
+            raise ValueError("At least one metric must be provided to spot()")
+        
+        # Get CSV file path
+        csv_file = self.output_dir / f"{metric_key}.csv"
+        
+        # Get current schema (ordered dict to preserve column order)
+        current_schema = OrderedDict(sorted(metrics.items()))
+        
+        # Check if this is the first call for this metric_key
+        if metric_key not in self._spot_schemas:
+            # First call - establish schema
+            self._spot_schemas[metric_key] = current_schema
+            
+            # Create CSV file with header
+            with open(csv_file, 'w', newline='') as f:
+                writer = csv.writer(f)
+                header = ['timestamp'] + list(current_schema.keys())
+                writer.writerow(header)
+            
+            logger.debug(f"[INFO] Created spot metric file: {csv_file}")
+            logger.debug(f"   Schema: {list(current_schema.keys())}")
+        else:
+            # Validate schema matches
+            expected_schema = self._spot_schemas[metric_key]
+            if set(current_schema.keys()) != set(expected_schema.keys()):
+                raise ValueError(
+                    f"Schema mismatch for metric '{metric_key}'.\n"
+                    f"Expected keys: {list(expected_schema.keys())}\n"
+                    f"Provided keys: {list(current_schema.keys())}\n"
+                    f"All spot() calls for the same metric_key must have identical metric keys."
+                )
+        
+        # Append row to CSV
+        timestamp = time.time()
+        with open(csv_file, 'a', newline='') as f:
+            writer = csv.writer(f)
+            # Use the established schema order
+            schema = self._spot_schemas[metric_key]
+            row = [timestamp] + [metrics[key] for key in schema.keys()]
+            writer.writerow(row)
+    
+    @staticmethod
+    def load_results(results_file: str) -> Dict[str, Any]:
+        """
+        Load results from JSON file.
+        
+        Args:
+            results_file: Path to results.json
+            
+        Returns:
+            Results dictionary
+        """
+        with open(results_file, 'r') as f:
+            return json.load(f)

graflag_runner-1.0.0/runner.py

@@ -0,0 +1,326 @@
+"""Main method runner with resource monitoring."""
+
+import os
+import sys
+import json
+import time
+import subprocess
+import threading
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional, Dict, Any
+import logging
+
+from .monitor import ResourceMonitor
+from .results import ResultWriter
+from .subprocess_utils import run_with_realtime_output, save_output_to_file
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+
+class MethodRunner:
+    """
+    Wrapper for executing graph anomaly detection methods.
+    
+    Features:
+    - Automatic resource monitoring (CPU, memory, GPU)
+    - Execution timing
+    - Result standardization
+    - Error handling
+    
+    Usage in Dockerfile:
+        CMD ["python", "-m", "graflag_runner.runner"]
+        
+    Environment variables required:
+        - DATA: Input dataset path
+        - EXP: Experiment output path
+        - METHOD_NAME: Method name
+        - COMMAND: Command to execute (e.g., "python main.py --dataset uci")
+    """
+    
+    def __init__(
+        self,
+        data_dir: str,
+        exp_dir: str,
+        method_name: str,
+        command: str,
+        monitor_interval: float = 1.0,
+        pass_env_args: bool = False,
+        **kwargs
+    ):
+        """
+        Initialize method runner.
+        
+        Args:
+            data_dir: Input dataset directory
+            exp_dir: Experiment output directory
+            method_name: Name of the method
+            command: Command to execute
+            monitor_interval: Resource monitoring interval in seconds (default: 1.0)
+            pass_env_args: Whether to extract env vars starting with _ and pass as CLI args (default: False)
+            **kwargs: Additional configuration
+        """
+        self.data_dir = Path(data_dir)
+        self.exp_dir = Path(exp_dir)
+        self.method_name = method_name
+        self.command = command
+        self.monitor_interval = monitor_interval
+        self.pass_env_args = pass_env_args
+        self.config = kwargs
+        
+        # Extract environment variables starting with _ if requested
+        if self.pass_env_args:
+            self.command = self._build_command_with_env_args()
+        
+        # Create experiment directory
+        self.exp_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Initialize resource monitor (uses spot() method now)
+        self.monitor = ResourceMonitor()
+        
+        logger.info("=" * 60)
+        logger.info(f"GraFlag Runner - {self.method_name}")
+        logger.info("=" * 60)
+        logger.info(f"[INFO] Data: {self.data_dir}")
+        logger.info(f"[INFO] Output: {self.exp_dir}")
+        logger.info(f"[INFO] Command: {self.command}")
+        logger.info(f"[INFO] Monitor interval: {self.monitor_interval}s")
+        logger.info(f"[INFO] Pass env args: {self.pass_env_args}")
+        logger.info("")
+    
+    def _build_command_with_env_args(self) -> str:
+        """
+        Extract environment variables starting with _ and append them as CLI arguments.
+
+        Example:
+            _BATCH_SIZE=128 -> --batch_size 128
+            _LEARNING_RATE=0.001 -> --learning_rate 0.001
+
+        Returns:
+            Command string with appended arguments
+        """
+        env_args = []
+
+        for key, value in os.environ.items():
+            if key.startswith("_"):
+                # Remove leading underscore and convert to lowercase
+                arg_name = key[1:].lower()
+                env_args.append(f"--{arg_name} {value}")
+
+        if env_args:
+            args_str = " ".join(env_args)
+            logger.info(f"[INFO] Extracted env args: {args_str}")
+            return f"{self.command} {args_str}"
+
+        return self.command
+    
+    def _save_status(self, status: str, exec_time_ms: float = None,
+                     resources: dict = None, exit_code: int = None,
+                     error: str = None):
+        """Save execution status to status.json in the experiment directory."""
+        status_data = {
+            "status": status,
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+            "method_name": self.method_name,
+        }
+        if exec_time_ms is not None:
+            status_data["exec_time_ms"] = round(exec_time_ms, 2)
+        if resources is not None:
+            status_data["resources"] = resources
+        if exit_code is not None:
+            status_data["exit_code"] = exit_code
+        if error is not None:
+            status_data["error"] = str(error)
+
+        status_file = self.exp_dir / "status.json"
+        try:
+            with open(status_file, 'w') as f:
+                json.dump(status_data, f, indent=2)
+        except Exception as e:
+            logger.warning(f"Failed to write status.json: {e}")
+
+    def run(self) -> Dict[str, Any]:
+        """
+        Execute method with monitoring.
+
+        Returns:
+            Dictionary with execution summary
+        """
+        # Save initial running status
+        self._save_status("running")
+
+        # Start monitoring in background thread
+        monitor_thread = threading.Thread(
+            target=self.monitor.start_monitoring,
+            args=(self.monitor_interval,),
+            daemon=True
+        )
+        monitor_thread.start()
+
+        # Record start time
+        start_time = time.time()
+
+        try:
+            logger.info("[INFO] Starting method execution...")
+
+            # Execute command with real-time output using utility function
+            return_code, captured_output = run_with_realtime_output(
+                command=self.command,
+                shell=True,
+                cwd=os.getcwd()
+            )
+
+            # Record end time
+            end_time = time.time()
+            exec_time_ms = (end_time - start_time) * 1000
+
+            # Stop monitoring
+            self.monitor.stop_monitoring()
+            monitor_thread.join(timeout=2)
+
+            # Get resource summary
+            resources = self.monitor.get_summary()
+
+            # Log results
+            logger.info("")
+            logger.info("[INFO] Execution Summary:")
+            logger.info(f"   [INFO] Execution time: {exec_time_ms:.2f}ms")
+            logger.info(f"   [INFO] Peak memory: {resources['peak_memory_mb']:.2f}MB")
+            if resources['peak_gpu_mb'] is not None:
+                logger.info(f"   [INFO] Peak GPU memory: {resources['peak_gpu_mb']:.2f}MB")
+            logger.info("")
+
+            # Save captured output to file using utility function
+            output_file = self.exp_dir / "method_output.txt"
+            save_output_to_file(
+                output_lines=captured_output,
+                output_file=str(output_file),
+                header="=== METHOD OUTPUT ===\n"
+            )
+            logger.info(f"[INFO] Full output saved to: {output_file}")
+
+            if return_code == 0:
+                logger.info("[OK] Method execution completed successfully")
+                self._save_status("completed", exec_time_ms, resources, exit_code=0)
+            else:
+                logger.error(f"[FAIL] Method execution failed with exit code {return_code}")
+                logger.error(f"[INFO] Check {output_file} for details")
+                self._save_status("failed", exec_time_ms, resources, exit_code=return_code)
+                raise RuntimeError(f"Method execution failed with exit code {return_code}")
+
+            return {
+                "success": True,
+                "exec_time_ms": exec_time_ms,
+                "resources": resources,
+                "output_file": str(output_file)
+            }
+
+        except Exception as e:
+            # Stop monitoring on error
+            self.monitor.stop_monitoring()
+            # Save failed status (only if not already saved by return_code check)
+            status_file = self.exp_dir / "status.json"
+            try:
+                existing = json.loads(status_file.read_text())
+                if existing.get("status") == "running":
+                    end_time = time.time()
+                    self._save_status("failed", (end_time - start_time) * 1000, error=str(e))
+            except Exception:
+                self._save_status("failed", error=str(e))
+            logger.error(f"[FAIL] Execution error: {e}")
+            raise
+    
+    @classmethod
+    def from_env(cls, pass_env_args: bool = False):
+        """
+        Create runner from environment variables.
+
+        Args:
+            pass_env_args: Whether to pass _* env vars as CLI args (default: False)
+
+        Environment variables:
+            - DATA: Input dataset path
+            - EXP: Experiment output path
+            - METHOD_NAME: Method name
+            - COMMAND: Command to execute
+            - MONITOR_INTERVAL: Resource monitoring interval in seconds (optional, default: 1.0)
+            - SUPPORTED_DATASETS: Comma-separated list of compatible dataset patterns (optional)
+        """
+        data_dir = os.environ.get("DATA")
+        exp_dir = os.environ.get("EXP")
+        method_name = os.environ.get("METHOD_NAME", "Unknown")
+        command = os.environ.get("COMMAND")
+        monitor_interval = float(os.environ.get("MONITOR_INTERVAL", "1.0"))
+        supported_datasets = os.environ.get("SUPPORTED_DATASETS", "")
+
+        if not all([data_dir, exp_dir, command]):
+            raise ValueError(
+                "Missing required environment variables: DATA, EXP, COMMAND"
+            )
+
+        # Validate dataset compatibility if SUPPORTED_DATASETS is specified
+        if supported_datasets:
+            dataset_name = os.path.basename(data_dir.rstrip('/'))
+            patterns = [p.strip() for p in supported_datasets.split(',') if p.strip()]
+
+            is_compatible = False
+            for pattern in patterns:
+                # Support wildcard patterns (e.g., "generaldyg_*", "btc_*")
+                if pattern.endswith('*'):
+                    prefix = pattern[:-1]
+                    if dataset_name.startswith(prefix):
+                        is_compatible = True
+                        break
+                elif pattern == dataset_name:
+                    is_compatible = True
+                    break
+
+            if not is_compatible:
+                logger.error(f"[FAIL] Dataset '{dataset_name}' is not compatible with method '{method_name}'")
+                logger.error(f"   Supported datasets: {', '.join(patterns)}")
+                raise ValueError(
+                    f"Dataset '{dataset_name}' is not compatible with method '{method_name}'. "
+                    f"Supported datasets: {', '.join(patterns)}"
+                )
+        
+        return cls(
+            data_dir=data_dir,
+            exp_dir=exp_dir,
+            method_name=method_name,
+            command=command,
+            monitor_interval=monitor_interval,
+            pass_env_args=pass_env_args
+        )
+
+
+def main():
+    """CLI entry point for running as module."""
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="GraFlag Method Runner")
+    parser.add_argument(
+        "--pass-env-args",
+        action="store_true",
+        help="Extract environment variables starting with _ and pass as CLI arguments"
+    )
+    
+    args = parser.parse_args()
+    
+    try:
+        runner = MethodRunner.from_env(pass_env_args=args.pass_env_args)
+        summary = runner.run()
+        
+        logger.info("[OK] Runner completed successfully")
+        sys.exit(0)
+        
+    except Exception as e:
+        logger.error(f"[FAIL] Runner failed: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

graflag_runner-1.0.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

graflag_runner-1.0.0/setup.py

@@ -0,0 +1,21 @@
+"""Setup script for graflag_runner package."""
+
+from setuptools import setup
+
+setup(
+    name="graflag_runner",
+    version="1.0.0",
+    description="Framework for executing graph anomaly detection methods with resource monitoring",
+    author="GraFlag Team",
+    packages=["graflag_runner"],
+    package_dir={"graflag_runner": "."},
+    install_requires=[
+        "psutil>=5.8.0",
+    ],
+    python_requires=">=3.7",
+    entry_points={
+        "console_scripts": [
+            "graflag-run=graflag_runner.runner:main",
+        ],
+    },
+)

graflag_runner-1.0.0/streaming.py

@@ -0,0 +1,160 @@
+"""Streaming utilities for handling large result data."""
+
+import json
+from typing import Iterator, Any, Union, List
+from pathlib import Path
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class StreamableArray:
+    """
+    Wrapper for large arrays that can be generated on-demand.
+    
+    This allows methods to avoid loading entire large matrices into memory
+    by generating rows/values lazily as needed during JSON serialization.
+    
+    Usage:
+        # Instead of building a huge list:
+        # scores = [[...], [...], ...]  # 10GB in memory
+        
+        # Use a generator:
+        def generate_rows():
+            for i in range(1000000):
+                yield compute_row(i)
+        
+        scores = StreamableArray(generate_rows())
+    """
+    
+    def __init__(self, generator: Iterator):
+        """
+        Initialize with a generator or iterator.
+        
+        Args:
+            generator: Iterator that yields array elements (rows, values, etc.)
+        """
+        self.generator = generator
+    
+    def __iter__(self):
+        """Make this object iterable."""
+        return self.generator
+
+
+def stream_write_json(data: dict, output_path: Path, streamable_keys: List[str] = None):
+    """
+    Write JSON to file with support for streaming large arrays.
+    
+    This function writes JSON incrementally, streaming any StreamableArray objects
+    row-by-row instead of loading them entirely into memory. Regular data is
+    serialized normally.
+    
+    Args:
+        data: Dictionary to serialize to JSON
+        output_path: Path to output JSON file
+        streamable_keys: Keys in data that contain StreamableArray objects
+                        (auto-detected if None)
+    
+    Example:
+        data = {
+            "result_type": "TEMPORAL_EDGE_ANOMALY_SCORES",
+            "scores": StreamableArray(generate_scores()),  # Large!
+            "timestamps": [0, 1, 2, ...],  # Regular list
+            "metadata": {...}
+        }
+        stream_write_json(data, Path("results.json"))
+    """
+    if streamable_keys is None:
+        # Auto-detect StreamableArray objects
+        streamable_keys = [
+            key for key, value in data.items() 
+            if isinstance(value, StreamableArray)
+        ]
+    
+    logger.debug(f"Stream writing JSON with streamable keys: {streamable_keys}")
+    
+    with open(output_path, 'w') as f:
+        f.write('{\n')
+        
+        keys = list(data.keys())
+        for idx, key in enumerate(keys):
+            value = data[key]
+            is_last = (idx == len(keys) - 1)
+            
+            # Write key
+            f.write(f'  {json.dumps(key)}: ')
+            
+            # Handle streamable arrays
+            if isinstance(value, StreamableArray):
+                _stream_write_array(f, value, indent=2)
+            else:
+                # Regular JSON serialization with proper indentation
+                serialized = json.dumps(value, indent=2)
+                # Indent all lines after the first
+                lines = serialized.split('\n')
+                f.write(lines[0])
+                for line in lines[1:]:
+                    f.write('\n  ' + line)
+            
+            # Comma between fields
+            if not is_last:
+                f.write(',')
+            f.write('\n')
+        
+        f.write('}\n')
+    
+    logger.info(f"[OK] Streamed JSON written to: {output_path}")
+
+
+def _stream_write_array(f, streamable: StreamableArray, indent: int = 0):
+    """
+    Internal function to stream write an array element by element.
+    
+    Args:
+        f: File handle to write to
+        streamable: StreamableArray to serialize
+        indent: Indentation level (number of spaces)
+    """
+    indent_str = ' ' * indent
+    f.write('[\n')
+    
+    first = True
+    row_count = 0
+    
+    for element in streamable:
+        if not first:
+            f.write(',\n')
+        
+        # Write element with indentation
+        f.write(f'{indent_str}  ')
+        json.dump(element, f)
+        
+        first = False
+        row_count += 1
+        
+        # Progress logging for very large arrays (every 1000 rows)
+        if row_count % 1000 == 0:
+            logger.debug(f"  ... streamed {row_count} rows")
+    
+    f.write(f'\n{indent_str}]')
+    
+    if row_count >= 1000:
+        logger.info(f"  Streamed total: {row_count} rows")
+
+
+def is_streamable(obj: Any) -> bool:
+    """
+    Check if an object is streamable (generator/iterator).
+    
+    Args:
+        obj: Object to check
+        
+    Returns:
+        True if object is a generator or iterator (excluding strings/lists)
+    """
+    return (
+        isinstance(obj, StreamableArray) or
+        (hasattr(obj, '__iter__') and 
+         hasattr(obj, '__next__') and 
+         not isinstance(obj, (str, bytes, list, tuple, dict)))
+    )

graflag_runner-1.0.0/subprocess_utils.py

@@ -0,0 +1,166 @@
+"""Subprocess utilities for real-time output streaming."""
+
+import os
+import sys
+import subprocess
+from typing import List, Tuple, Optional, Dict
+
+
+def run_with_realtime_output(
+    command: str,
+    shell: bool = True,
+    cwd: Optional[str] = None,
+    env: Optional[Dict[str, str]] = None,
+    stdin_passthrough: bool = True
+) -> Tuple[int, List[str]]:
+    """
+    Run a command with real-time output streaming while capturing all output.
+    
+    This function streams subprocess output in real-time to stdout while also
+    capturing it for later use. Optionally forwards stdin from parent to subprocess,
+    enabling interactive processes and piped input.
+    
+    Args:
+        command: Command to execute (string if shell=True, list if shell=False)
+        shell: Whether to execute through shell (default: True)
+        cwd: Working directory for the command (default: current directory)
+        env: Environment variables dict (default: copy of current environment)
+        stdin_passthrough: Forward stdin from parent to subprocess (default: True)
+                          Set to False to disable stdin (subprocess gets None)
+    
+    Returns:
+        Tuple of (return_code, captured_output_lines)
+        - return_code: Exit code of the process
+        - captured_output_lines: List of output lines (includes newlines)
+    
+    Example:
+        >>> # Basic usage
+        >>> return_code, output = run_with_realtime_output("python train.py")
+        >>> if return_code == 0:
+        >>>     with open("log.txt", "w") as f:
+        >>>         f.writelines(output)
+        
+        >>> # With piped input
+        >>> return_code, output = run_with_realtime_output(
+        >>>     "python process.py",
+        >>>     stdin_passthrough=True  # Allows: echo "data" | python wrapper.py
+        >>> )
+        
+        >>> # Interactive process
+        >>> return_code, output = run_with_realtime_output(
+        >>>     "python interactive.py",
+        >>>     stdin_passthrough=True  # User can type input
+        >>> )
+    """
+    # Set environment variables to disable Python output buffering
+    if env is None:
+        env = os.environ.copy()
+    env['PYTHONUNBUFFERED'] = '1'
+    
+    # Determine stdin handling
+    # If stdin_passthrough=True, inherit parent's stdin (allows piped input & interactive)
+    # If stdin_passthrough=False, subprocess gets no input (stdin=None)
+    stdin_config = None if stdin_passthrough else subprocess.DEVNULL
+    
+    # Start subprocess with piped output and configurable stdin
+    process = subprocess.Popen(
+        command,
+        shell=shell,
+        stdin=stdin_config,  # Inherit stdin or disable it
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,  # Merge stderr into stdout
+        text=True,
+        bufsize=1,  # Line buffered
+        universal_newlines=True,
+        env=env,
+        cwd=cwd
+    )
+    
+    # Capture output while streaming it in real-time
+    captured_output = []
+    for line in process.stdout:
+        # Print to console (forwarded to parent)
+        print(line, end='', flush=True)
+        sys.stdout.flush()  # Force flush
+        # Capture for later use
+        captured_output.append(line)
+    
+    # Wait for process to complete
+    return_code = process.wait()
+    
+    return return_code, captured_output
+
+
+def run_command_list(
+    commands: List[str],
+    cwd: Optional[str] = None,
+    env: Optional[Dict[str, str]] = None,
+    stop_on_error: bool = True,
+    stdin_passthrough: bool = False
+) -> List[Tuple[str, int, List[str]]]:
+    """
+    Run multiple commands sequentially with real-time output.
+    
+    Args:
+        commands: List of commands to execute
+        cwd: Working directory for all commands
+        env: Environment variables dict
+        stop_on_error: If True, stop executing after first failure
+        stdin_passthrough: Forward stdin to commands (default: False for batch jobs)
+    
+    Returns:
+        List of tuples: [(command, return_code, output_lines), ...]
+    
+    Example:
+        >>> results = run_command_list([
+        >>>     "python prepare_data.py",
+        >>>     "python train_model.py"
+        >>> ])
+        >>> for cmd, code, output in results:
+        >>>     if code != 0:
+        >>>         print(f"Failed: {cmd}")
+    """
+    results = []
+    
+    for command in commands:
+        print(f"\n{'='*60}")
+        print(f"Running: {command}")
+        print(f"{'='*60}\n")
+        
+        return_code, output = run_with_realtime_output(
+            command=command,
+            cwd=cwd,
+            env=env,
+            stdin_passthrough=stdin_passthrough
+        )
+        
+        results.append((command, return_code, output))
+        
+        if stop_on_error and return_code != 0:
+            print(f"\n[ERROR] Command failed with exit code {return_code}")
+            break
+    
+    return results
+
+
+def save_output_to_file(
+    output_lines: List[str],
+    output_file: str,
+    header: str = "=== OUTPUT ===\n"
+) -> None:
+    """
+    Save captured output to a file.
+    
+    Args:
+        output_lines: List of output lines to save
+        output_file: Path to output file
+        header: Optional header to prepend to the file
+    
+    Example:
+        >>> return_code, output = run_with_realtime_output("python train.py")
+        >>> save_output_to_file(output, "training.log", "=== TRAINING LOG ===\n")
+    """
+    with open(output_file, 'w') as f:
+        if header:
+            f.write(header)
+        f.writelines(output_lines)