graflag 1.0.0__py3-none-any.whl

graflag/__init__.py

@@ -0,0 +1,20 @@
+"""
+GraFlag - Graph Anomaly Detection Benchmarking Tool
+
+A tool for benchmarking Graph Anomaly Detection methods using Docker Swarm
+across multiple nodes with shared NFS storage.
+"""
+
+from .core import GraFlag, GraFlagError
+from .config import GraflagConfig, CONFIG_FILE
+from .models import (
+    ClusterInfo, MethodInfo, DatasetInfo, ExperimentInfo,
+    ExperimentResults, EvaluationResults, RunProgress,
+)
+
+__version__ = "1.0.0"
+__all__ = [
+    "GraFlag", "GraFlagError", "GraflagConfig", "CONFIG_FILE",
+    "ClusterInfo", "MethodInfo", "DatasetInfo", "ExperimentInfo",
+    "ExperimentResults", "EvaluationResults", "RunProgress",
+]

graflag/api.py

@@ -0,0 +1,217 @@
+"""
+GraFlag Python API for GUI Integration.
+
+Thin wrapper around GraFlag core that provides error-safe access
+and returns structured dataclass objects suitable for GUI/web consumption.
+"""
+
+from typing import Dict, List, Optional, Any, Callable
+import logging
+
+from .core import GraFlag, GraFlagError
+from .config import GraflagConfig
+from .models import (
+    ClusterInfo, MethodInfo, DatasetInfo, ExperimentInfo,
+    ExperimentResults, EvaluationResults, RunProgress,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class GraFlagAPI:
+    """
+    High-level Python API for GraFlag operations.
+
+    All methods return structured data (dataclasses) and catch exceptions
+    to avoid crashing the GUI. Use the core GraFlag class directly for
+    CLI-style usage where exceptions should propagate.
+
+    Usage:
+        api = GraFlagAPI(config_file=".env")
+
+        cluster = api.get_cluster_info()
+        methods = api.list_methods()
+        experiments = api.list_experiments()
+    """
+
+    def __init__(self, config_file: str = ".env", log_level: int = logging.INFO):
+        logging.basicConfig(level=log_level)
+        self.core = GraFlag(config_file)
+        self.config = self.core.config
+
+    # ========================================================================
+    # Cluster
+    # ========================================================================
+
+    def get_cluster_info(self) -> ClusterInfo:
+        """Get cluster status information."""
+        return self.core.status()
+
+    def setup_cluster(self) -> Dict[str, Any]:
+        """Setup GraFlag cluster."""
+        try:
+            self.core.setup()
+            return {"success": True, "message": "Cluster setup completed"}
+        except Exception as e:
+            return {"success": False, "error": str(e)}
+
+    # ========================================================================
+    # Resources
+    # ========================================================================
+
+    def list_methods(self) -> List[MethodInfo]:
+        """List available methods."""
+        try:
+            return self.core.list_methods()
+        except Exception as e:
+            logger.error(f"Error listing methods: {e}")
+            return []
+
+    def get_method_details(self, method_name: str) -> Optional[MethodInfo]:
+        """Get details for a specific method."""
+        try:
+            methods = self.core.list_methods()
+            for m in methods:
+                if m.name == method_name:
+                    return m
+            return None
+        except Exception as e:
+            logger.error(f"Error getting method details: {e}")
+            return None
+
+    def list_datasets(self) -> List[DatasetInfo]:
+        """List available datasets."""
+        try:
+            return self.core.list_datasets()
+        except Exception as e:
+            logger.error(f"Error listing datasets: {e}")
+            return []
+
+    def list_experiments(self, limit: int = 50) -> List[ExperimentInfo]:
+        """List recent experiments."""
+        try:
+            return self.core.list_experiments(limit=limit)
+        except Exception as e:
+            logger.error(f"Error listing experiments: {e}")
+            return []
+
+    def get_experiment_details(self, experiment_name: str) -> Optional[ExperimentInfo]:
+        """Get details for a specific experiment."""
+        try:
+            experiments = self.core.list_experiments(limit=500)
+            for e in experiments:
+                if e.name == experiment_name:
+                    return e
+            return None
+        except Exception as e:
+            logger.error(f"Error getting experiment details: {e}")
+            return None
+
+    # ========================================================================
+    # Run
+    # ========================================================================
+
+    def run(
+        self,
+        method: str,
+        dataset: str,
+        tag: str = "latest",
+        build: bool = False,
+        gpu: bool = True,
+        method_params: Optional[Dict[str, Any]] = None,
+        on_progress: Optional[Callable[[RunProgress], None]] = None,
+    ) -> str:
+        """Run an experiment. Returns experiment name."""
+        return self.core.run(
+            method_name=method,
+            dataset=dataset,
+            tag=tag,
+            build=build,
+            gpu=gpu,
+            method_params=method_params or {},
+        )
+
+    # ========================================================================
+    # Results
+    # ========================================================================
+
+    def get_experiment_results(self, experiment_name: str) -> Optional[ExperimentResults]:
+        """Get experiment results."""
+        try:
+            return self.core.get_experiment_results(experiment_name)
+        except Exception as e:
+            logger.error(f"Error getting results: {e}")
+            return None
+
+    def get_evaluation_results(self, experiment_name: str) -> Optional[EvaluationResults]:
+        """Get evaluation results."""
+        try:
+            return self.core.get_evaluation_results(experiment_name)
+        except Exception as e:
+            logger.error(f"Error getting evaluation: {e}")
+            return None
+
+    def evaluate_experiment(self, experiment_name: str) -> bool:
+        """Run evaluation on an experiment."""
+        self.core.evaluate(experiment_name)
+        return True
+
+    # ========================================================================
+    # Services
+    # ========================================================================
+
+    def list_running_services(self) -> List[Dict[str, str]]:
+        """List running Docker services."""
+        try:
+            return self.core.list_services()
+        except Exception as e:
+            logger.error(f"Error listing services: {e}")
+            return []
+
+    def stop_experiment(self, experiment_name: str) -> bool:
+        """Stop a running experiment."""
+        try:
+            self.core.stop(experiment_name)
+            return True
+        except Exception as e:
+            logger.error(f"Error stopping experiment: {e}")
+            return False
+
+    def delete_experiment(self, experiment_name: str) -> bool:
+        """Stop and delete an experiment."""
+        try:
+            self.core.stop(experiment_name, remove=True)
+            return True
+        except Exception as e:
+            logger.error(f"Error deleting experiment: {e}")
+            return False
+
+    def get_experiment_logs(self, experiment_name: str, tail: int = 100) -> List[str]:
+        """Get recent logs for an experiment."""
+        try:
+            return self.core.get_logs(experiment_name, tail=tail)
+        except Exception as e:
+            logger.error(f"Error getting logs: {e}")
+            return []
+
+    # ========================================================================
+    # File Operations
+    # ========================================================================
+
+    def download_file(self, remote_path: str, local_path: str) -> bool:
+        """Download a file from remote shared directory."""
+        try:
+            self.core.copy_files(remote_path, local_path, recursive=False, from_remote=True)
+            return True
+        except Exception as e:
+            logger.error(f"Error downloading file: {e}")
+            return False
+
+    def download_directory(self, remote_path: str, local_path: str) -> bool:
+        """Download a directory from remote shared directory."""
+        try:
+            self.core.copy_files(remote_path, local_path, recursive=True, from_remote=True)
+            return True
+        except Exception as e:
+            logger.error(f"Error downloading directory: {e}")
+            return False

graflag/cli.py

@@ -0,0 +1,321 @@
+"""Command Line Interface for GraFlag."""
+
+import os
+import sys
+import json
+import argparse
+import logging
+import traceback
+from pathlib import Path
+
+from .core import GraFlag, GraFlagError
+from .config import get_config_path, init_config
+
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger(__name__)
+
+
+def main():
+    """Main CLI interface."""
+    parser = argparse.ArgumentParser(
+        description="GraFlag - Graph Anomaly Detection Benchmarking Tool",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""\
+Examples:
+  graflag setup                                    # Setup cluster
+  graflag setup --reconfigure                      # Re-run config wizard
+  graflag run --method Dummy --dataset Cora        # Run experiment with GPU
+  graflag run -m taddy -d uci --build --params MAX_EPOCH=100 LEARNING_RATE=0.001
+  graflag run -m DeepWalk -d CiteSeer --no-gpu     # Run without GPU
+  graflag run --from-config ./experiments/exp__method__dataset__timestamp/service_config.json
+  graflag status                                   # Show cluster status
+  graflag list methods                             # List available methods
+  graflag list services                            # List running services
+  graflag logs -e exp__dummy__cora__20250924_161245 # Show logs
+  graflag logs -e exp__dummy__cora__20250924_161245 -f # Follow logs
+  graflag stop -e exp__dummy__cora__20250924_161245 # Stop experiment
+  graflag evaluate -e exp__generaldyg__btc_alpha__20251211_120000 # Evaluate
+  graflag copy -s ./data -d datasets -r            # Copy to remote
+  graflag copy --from-remote -s experiments/exp -d ./local # Copy from remote
+  graflag sync                                     # Sync current method dir
+  graflag sync --lib --path ./my-lib/              # Sync a shared library
+  graflag gui                                      # Start web dashboard
+  graflag gui --port 8080 --debug                  # GUI on custom port
+  graflag devcluster --hosts hosts.yml             # Deploy virtual cluster
+  graflag devcluster --hosts hosts.yml --pubkey ~/.ssh/id_rsa.pub
+  graflag devcluster --down                        # Stop and remove cluster
+        """,
+    )
+
+    parser.add_argument(
+        "command",
+        choices=["setup", "run", "status", "list", "copy", "logs", "stop", "evaluate", "sync", "gui", "devcluster"],
+        help="Command to execute",
+    )
+    parser.add_argument(
+        "subcommand", nargs="?",
+        choices=["methods", "datasets", "experiments", "services"],
+        help="Subcommand for list command",
+    )
+    parser.add_argument("--method", "-m", help="Method name for run")
+    parser.add_argument("--dataset", "-d", help="Dataset name for run")
+    parser.add_argument("--tag", "-t", default="latest", help="Docker image tag (default: latest)")
+    parser.add_argument("--build", "-b", action="store_true", help="Build image before running")
+    parser.add_argument("--config", "-c", default=".env", help="Configuration file (default: .env)")
+    parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose logging")
+    parser.add_argument("--source", "-s", nargs='+', help="Source path(s) for copy command")
+    parser.add_argument("--dest", help="Destination path for copy command")
+    parser.add_argument("--recursive", "-r", action="store_true", help="Copy directories recursively")
+    parser.add_argument("--from-remote", action="store_true", help="Copy from remote to local")
+    parser.add_argument("--experiment", "-e", help="Experiment name for logs/stop commands")
+    parser.add_argument("--follow", "-f", action="store_true", help="Follow log output")
+    parser.add_argument("--rm", action="store_true", help="Also delete experiment directory on stop")
+    parser.add_argument("--tee", help="Save logs to file while displaying")
+    parser.add_argument("--gpu", "-g", action="store_true", default=True, help="Enable GPU (default: True)")
+    parser.add_argument("--no-gpu", action="store_false", dest="gpu", help="Disable GPU")
+    parser.add_argument("--params", "-p", nargs='+', metavar="KEY=VALUE", help="Method parameters")
+    parser.add_argument(
+        "--from-config", metavar="CONFIG_FILE",
+        help="Load method/dataset/params from a config file",
+    )
+    parser.add_argument("--lib", action="store_true", help="Sync as a shared library")
+    parser.add_argument("--path", default=None, help="Local path for sync command")
+    # GUI args
+    parser.add_argument("--host", default="0.0.0.0", help="GUI server host (default: 0.0.0.0)")
+    parser.add_argument("--port", default=5000, type=int, help="GUI server port (default: 5000)")
+    parser.add_argument("--debug", action="store_true", help="Enable GUI debug mode")
+    # Devcluster args
+    parser.add_argument("--hosts", default=None, help="Path to hosts.yml for devcluster")
+    parser.add_argument("--pubkey", default=None, help="Path to SSH public key for devcluster")
+    parser.add_argument("--down", action="store_true", help="Stop and remove devcluster")
+    parser.add_argument("--reconfigure", action="store_true", help="Re-run configuration wizard for setup")
+
+    args = parser.parse_args()
+
+    if args.verbose:
+        logging.getLogger().setLevel(logging.DEBUG)
+
+    try:
+        # GUI: start web dashboard (doesn't need GraFlag instance directly)
+        if args.command == "gui":
+            from .gui.server import serve
+            serve(args.config, args.host, args.port, args.debug)
+            return
+
+        # Devcluster: deploy or tear down virtual cluster
+        if args.command == "devcluster":
+            if not args.hosts and not args.down:
+                parser.error("devcluster requires --hosts <path-to-hosts.yml> or --down")
+            from .devcluster.cli import main as devcluster_main
+            devcluster_main(args.hosts, args.pubkey, args.down)
+            return
+
+        # Setup: create or update config interactively
+        if args.command == "setup":
+            config_path = get_config_path(args.config)
+            if not config_path.exists() or args.reconfigure:
+                init_config()
+
+        gf = GraFlag(config_file=args.config)
+
+        if args.command == "setup":
+            gf.setup()
+            # Show status after setup
+            _print_status(gf.status())
+
+        elif args.command == "run":
+            method, dataset, method_params = _parse_run_args(args, parser)
+            gf.run(method, dataset, args.tag, args.build, args.gpu, method_params)
+
+        elif args.command == "status":
+            _print_status(gf.status())
+
+        elif args.command == "list":
+            if args.subcommand == "methods":
+                _print_methods(gf.list_methods())
+            elif args.subcommand == "datasets":
+                _print_datasets(gf.list_datasets())
+            elif args.subcommand == "experiments":
+                _print_experiments(gf.list_experiments())
+            elif args.subcommand == "services":
+                _print_services(gf.list_services())
+            else:
+                parser.error("list command requires subcommand: methods, datasets, experiments, or services")
+
+        elif args.command == "copy":
+            if not args.source or not args.dest:
+                parser.error("copy command requires --source and --dest")
+            gf.copy_files(args.source, args.dest, args.recursive, args.from_remote)
+
+        elif args.command == "logs":
+            if not args.experiment:
+                parser.error("logs command requires --experiment")
+            if args.follow:
+                gf.follow_logs(args.experiment, args.tee)
+            else:
+                gf.show_logs(args.experiment, args.tee)
+
+        elif args.command == "stop":
+            if not args.experiment:
+                parser.error("stop command requires --experiment")
+            gf.stop(args.experiment, remove=args.rm)
+
+        elif args.command == "evaluate":
+            if not args.experiment:
+                parser.error("evaluate command requires --experiment")
+            gf.evaluate(args.experiment)
+
+        elif args.command == "sync":
+            local_path = args.path or os.getcwd()
+            gf.sync(local_path, is_lib=args.lib)
+
+    except GraFlagError as e:
+        logger.error(str(e))
+        sys.exit(1)
+    except KeyboardInterrupt:
+        logger.info("Interrupted by user")
+        sys.exit(0)
+    except Exception as e:
+        logger.error(f"Unexpected error: {e}")
+        if args.verbose:
+            traceback.print_exc()
+        sys.exit(1)
+
+
+# ============================================================================
+# Output Formatting
+# ============================================================================
+
+def _parse_run_args(args, parser):
+    """Parse run arguments from CLI args."""
+    method = args.method
+    dataset = args.dataset
+    method_params = {}
+
+    if args.from_config:
+        config_path = Path(args.from_config)
+        if not config_path.exists():
+            parser.error(f"Config file not found: {args.from_config}")
+
+        try:
+            with open(config_path, 'r') as f:
+                config = json.load(f)
+        except json.JSONDecodeError as e:
+            parser.error(f"Invalid JSON in config file: {e}")
+
+        if not method:
+            method = config.get('method_name')
+        if not dataset:
+            dataset = config.get('dataset')
+
+        env_contents = config.get('env_contents', {})
+        for key, value in env_contents.items():
+            if key.startswith('_'):
+                method_params[key[1:]] = str(value)
+
+    if args.params:
+        for param in args.params:
+            if '=' not in param:
+                parser.error(f"Invalid parameter format: {param}. Use KEY=VALUE.")
+            key, value = param.split('=', 1)
+            method_params[key] = value
+
+    if not method or not dataset:
+        parser.error("run requires --method and --dataset (or --from-config)")
+
+    return method, dataset, method_params
+
+
+def _print_status(cluster_info):
+    """Format and print cluster status."""
+    if cluster_info.error:
+        print(f"[ERROR] {cluster_info.error}")
+        return
+
+    print(f"\n[INFO] Manager: {cluster_info.manager_ip}")
+    print(f"[INFO] Swarm: {'active' if cluster_info.swarm_initialized else 'inactive'}")
+
+    if cluster_info.worker_nodes:
+        print("\n[INFO] Nodes:")
+        for node in cluster_info.worker_nodes:
+            role = "manager" if node.get('is_manager') else "worker"
+            print(f"  - {node['hostname']}: {node['status']} ({role}, {node['availability']})")
+
+    if cluster_info.services:
+        print(f"\n[INFO] Running Services:")
+        print(f"  {'NAME':<50} {'REPLICAS':<15} {'IMAGE':<30}")
+        print("  " + "-" * 95)
+        for svc in cluster_info.services:
+            name = svc['name'][:49]
+            replicas = svc.get('replicas', '')[:14]
+            image = svc.get('image', '')[:29]
+            print(f"  {name:<50} {replicas:<15} {image:<30}")
+    else:
+        print("\n[INFO] Running Services: None")
+
+    print(f"\n[INFO] Shared Directory: {cluster_info.shared_dir}")
+    if cluster_info.shared_contents:
+        print("  Contents:")
+        for item in cluster_info.shared_contents:
+            print(f"    - {item}")
+
+
+def _print_methods(methods):
+    """Format and print method list."""
+    if not methods:
+        print("[INFO] No methods found")
+        return
+
+    print("[INFO] Available Methods:")
+    for m in methods:
+        print(f"  - {m.name} (Supports: {m.supported_data})")
+
+
+def _print_datasets(datasets):
+    """Format and print dataset list."""
+    if not datasets:
+        print("[INFO] No datasets found")
+        return
+
+    print("[INFO] Available Datasets:")
+    for d in datasets:
+        size_str = f" ({d.size_mb:.1f} MB, {d.file_count} files)" if d.size_mb > 0 else ""
+        print(f"  - {d.name}{size_str}")
+
+
+def _print_experiments(experiments):
+    """Format and print experiment list."""
+    if not experiments:
+        print("[INFO] No experiments found")
+        return
+
+    print("[INFO] Recent Experiments:")
+    for e in experiments:
+        tags = f"[{e.status}]"
+        if e.has_results:
+            tags += " [results]"
+        if e.has_evaluation:
+            tags += " [eval]"
+        print(f"  - {e.name}  {tags}")
+
+
+def _print_services(services):
+    """Format and print running services."""
+    if not services:
+        print("\n[INFO] Running Services: None")
+        return
+
+    print("\n[INFO] Running Services:")
+    print(f"  {'NAME':<50} {'REPLICAS':<15} {'IMAGE':<30}")
+    print("  " + "-" * 95)
+    for svc in services:
+        name = svc['name'][:49]
+        replicas = str(svc.get('replicas', ''))[:14]
+        image = svc.get('image', '')[:29]
+        print(f"  {name:<50} {replicas:<15} {image:<30}")
+
+
+if __name__ == "__main__":
+    main()

graflag/config.py

@@ -0,0 +1,128 @@
+"""Configuration management for GraFlag."""
+
+import os
+from pathlib import Path
+from typing import Dict, Optional
+import logging
+
+logger = logging.getLogger(__name__)
+
+CONFIG_DIR = Path.home() / ".config" / "graflag"
+CONFIG_FILE = CONFIG_DIR / "config.env"
+
+DEFAULTS = {
+    "SSH_PORT": "22",
+    "SHARED_DIR": "/shared",
+    "NFS_PORT": "2049",
+}
+
+PROMPTS = [
+    ("MANAGER_IP", "Manager IP address", None),
+    ("SSH_PORT", "SSH port", "22"),
+    ("SSH_KEY", "SSH private key path", "~/.ssh/id_ed25519"),
+    ("SHARED_DIR", "Remote shared directory", "/shared"),
+    ("HOSTS_FILE", "Hosts file (hosts.yml) path", "hosts.yml"),
+]
+
+
+def get_config_path(override: Optional[str] = None) -> Path:
+    """Resolve config file path. Checks override, then cwd .env, then standard location."""
+    if override and override != ".env":
+        return Path(override)
+    cwd_env = Path.cwd() / ".env"
+    if cwd_env.exists():
+        return cwd_env
+    return CONFIG_FILE
+
+
+def init_config() -> Path:
+    """Interactively create configuration file."""
+    print("GraFlag configuration")
+    print(f"Config will be saved to: {CONFIG_FILE}\n")
+
+    values = {}
+    for key, prompt, default in PROMPTS:
+        if default:
+            raw = input(f"  {prompt} [{default}]: ").strip()
+            values[key] = raw if raw else default
+        else:
+            while True:
+                raw = input(f"  {prompt}: ").strip()
+                if raw:
+                    values[key] = raw
+                    break
+                print(f"    {key} is required.")
+
+    CONFIG_DIR.mkdir(parents=True, exist_ok=True)
+    with open(CONFIG_FILE, "w") as f:
+        f.write("# GraFlag Configuration\n")
+        for key, _, _ in PROMPTS:
+            f.write(f"{key}={values[key]}\n")
+
+    print(f"\n[OK] Configuration saved to {CONFIG_FILE}")
+    return CONFIG_FILE
+
+
+class GraflagConfig:
+    """Handle configuration loading and validation for GraFlag."""
+
+    def __init__(self, config_file: str = ".env"):
+        """Initialize configuration from file."""
+        self.config_path = get_config_path(config_file)
+        self.config = self._load_config()
+        self._validate_required_config()
+
+    def _load_config(self) -> Dict[str, str]:
+        """Load configuration from .env file."""
+        config = dict(DEFAULTS)
+
+        if not self.config_path.exists():
+            return config
+
+        with open(self.config_path, "r") as f:
+            for line in f:
+                line = line.strip()
+                if line and not line.startswith("#") and "=" in line:
+                    key, value = line.split("=", 1)
+                    config[key.strip()] = value.strip()
+
+        return config
+
+    def _validate_required_config(self):
+        """Validate that required configuration is present."""
+        required_keys = ["MANAGER_IP"]
+        missing_keys = [key for key in required_keys if not self.get(key)]
+
+        if missing_keys:
+            raise ValueError(
+                f"Missing required configuration: {', '.join(missing_keys)}. "
+                f"Run 'graflag setup' to configure."
+            )
+
+    def get(self, key: str, default: Optional[str] = None) -> Optional[str]:
+        """Get configuration value."""
+        return self.config.get(key, default)
+
+    @property
+    def remote_shared_dir(self) -> str:
+        return self.get("SHARED_DIR", "/shared")
+
+    @property
+    def manager_ip(self) -> str:
+        return self.get("MANAGER_IP")
+
+    @property
+    def ssh_port(self) -> str:
+        return self.get("SSH_PORT", "22")
+
+    @property
+    def ssh_key(self) -> Optional[str]:
+        return self.get("SSH_KEY")
+
+    @property
+    def nfs_port(self) -> str:
+        return self.get("NFS_PORT", "2049")
+
+    @property
+    def hosts_file(self) -> Optional[str]:
+        return self.get("HOSTS_FILE", "hosts.yml")