pyworkflow-engine 0.1.7__py3-none-any.whl → 0.1.10__py3-none-any.whl

pyworkflow/cli/commands/setup.py

@@ -21,7 +21,9 @@ from pyworkflow.cli.utils.config_generator import (
 from pyworkflow.cli.utils.docker_manager import (
     check_docker_available,
     check_service_health,
+    generate_cassandra_docker_compose_content,
     generate_docker_compose_content,
+    generate_mysql_docker_compose_content,
     generate_postgres_docker_compose_content,
     run_docker_command,
     write_docker_compose,
@@ -87,7 +89,10 @@ def _flatten_yaml_config(nested_config: dict) -> dict:
 )
 @click.option(
     "--storage",
-    type=click.Choice(["file", "memory", "sqlite", "postgres", "dynamodb"], case_sensitive=False),
+    type=click.Choice(
+        ["file", "memory", "sqlite", "postgres", "mysql", "dynamodb", "cassandra"],
+        case_sensitive=False,
+    ),
     help="Storage backend type",
 )
 @click.option(
@@ -140,7 +145,7 @@ def setup(
         sys.exit(1)
     except Exception as e:
         print_error(f"\nSetup failed: {str(e)}")
-        if ctx.obj.get("verbose"):
+        if ctx.obj and ctx.obj.get("verbose"):
             raise
         sys.exit(1)
 
@@ -239,6 +244,13 @@ def _run_setup(
 
     # 6. Write configuration file
     print_info("\nGenerating configuration...")
+
+    # Parse Cassandra contact points from comma-separated string to list
+    cassandra_contact_points = None
+    if config_data.get("cassandra_contact_points"):
+        contact_points_str = config_data["cassandra_contact_points"]
+        cassandra_contact_points = [cp.strip() for cp in contact_points_str.split(",")]
+
     yaml_content = generate_yaml_config(
         module=config_data.get("module"),
         runtime=config_data["runtime"],
@@ -251,9 +263,19 @@ def _run_setup(
         postgres_user=config_data.get("postgres_user"),
         postgres_password=config_data.get("postgres_password"),
         postgres_database=config_data.get("postgres_database"),
+        mysql_host=config_data.get("mysql_host"),
+        mysql_port=config_data.get("mysql_port"),
+        mysql_user=config_data.get("mysql_user"),
+        mysql_password=config_data.get("mysql_password"),
+        mysql_database=config_data.get("mysql_database"),
         dynamodb_table_name=config_data.get("dynamodb_table_name"),
         dynamodb_region=config_data.get("dynamodb_region"),
         dynamodb_endpoint_url=config_data.get("dynamodb_endpoint_url"),
+        cassandra_contact_points=cassandra_contact_points,
+        cassandra_port=config_data.get("cassandra_port"),
+        cassandra_keyspace=config_data.get("cassandra_keyspace"),
+        cassandra_user=config_data.get("cassandra_user"),
+        cassandra_password=config_data.get("cassandra_password"),
     )
 
     config_file_path = write_yaml_config(yaml_content, config_path, backup=True)
@@ -313,6 +335,36 @@ def _check_postgres_available() -> bool:
         return False
 
 
+def _check_cassandra_available() -> bool:
+    """
+    Check if cassandra-driver is installed for Cassandra support.
+
+    Returns:
+        True if cassandra-driver is available, False otherwise
+    """
+    try:
+        from cassandra.cluster import Cluster  # noqa: F401
+
+        return True
+    except ImportError:
+        return False
+
+
+def _check_mysql_available() -> bool:
+    """
+    Check if aiomysql is installed for MySQL support.
+
+    Returns:
+        True if aiomysql is available, False otherwise
+    """
+    try:
+        import aiomysql  # noqa: F401
+
+        return True
+    except ImportError:
+        return False
+
+
 def _run_interactive_configuration(
     non_interactive: bool,
     module_override: str | None,
@@ -346,9 +398,11 @@ def _run_interactive_configuration(
     config_data["result_backend"] = "redis://localhost:6379/1"
     print_info("✓ Broker: Redis (will be started via Docker)")
 
-    # Check if SQLite and PostgreSQL are available
+    # Check if SQLite, PostgreSQL, Cassandra and MySQL are available
     sqlite_available = _check_sqlite_available()
     postgres_available = _check_postgres_available()
+    cassandra_available = _check_cassandra_available()
+    mysql_available = _check_mysql_available()
 
     # Storage backend
     if storage_override:
@@ -378,6 +432,27 @@ def _run_interactive_configuration(
             print_info("")
             print_info("Or choose a different storage backend: --storage sqlite")
             raise click.Abort()
+        # Validate if cassandra was requested but not available
+        if storage_type == "cassandra" and not cassandra_available:
+            print_error("\nCassandra storage backend is not available!")
+            print_info("\ncassandra-driver package is not installed.")
+            print_info("To fix this, install cassandra-driver:")
+            print_info("")
+            print_info("  pip install cassandra-driver")
+            print_info("")
+            print_info("Or choose a different storage backend: --storage sqlite")
+            raise click.Abort()
+        # Validate if mysql was requested but not available
+        mysql_available = _check_mysql_available()
+        if storage_type == "mysql" and not mysql_available:
+            print_error("\nMySQL storage backend is not available!")
+            print_info("\naiomysql package is not installed.")
+            print_info("To fix this, install aiomysql:")
+            print_info("")
+            print_info("  pip install aiomysql")
+            print_info("")
+            print_info("Or choose a different storage backend: --storage sqlite")
+            raise click.Abort()
     elif non_interactive:
         if sqlite_available:
             storage_type = "sqlite"
@@ -407,6 +482,14 @@ def _run_interactive_configuration(
             choices.append(
                 {"name": "PostgreSQL - Scalable production database", "value": "postgres"}
             )
+        if cassandra_available:
+            choices.append(
+                {"name": "Cassandra - Distributed NoSQL database (scalable)", "value": "cassandra"}
+            )
+        if mysql_available:
+            choices.append(
+                {"name": "MySQL - Popular open-source relational database", "value": "mysql"}
+            )
         choices.extend(
             [
                 {
@@ -428,6 +511,14 @@ def _run_interactive_configuration(
             print_info("Note: PostgreSQL backend available after: pip install asyncpg")
             print_info("")
 
+        if not cassandra_available:
+            print_info("Note: Cassandra backend available after: pip install cassandra-driver")
+            print_info("")
+
+        if not mysql_available:
+            print_info("Note: MySQL backend available after: pip install aiomysql")
+            print_info("")
+
         storage_type = select(
             "Choose storage backend:",
             choices=choices,
@@ -491,6 +582,38 @@ def _run_interactive_configuration(
                 default="pyworkflow",
             )
 
+    # MySQL connection (for mysql backend)
+    elif storage_type == "mysql":
+        if non_interactive:
+            # Use default connection settings for non-interactive mode
+            config_data["mysql_host"] = "localhost"
+            config_data["mysql_port"] = "3306"
+            config_data["mysql_user"] = "pyworkflow"
+            config_data["mysql_password"] = "pyworkflow"
+            config_data["mysql_database"] = "pyworkflow"
+        else:
+            print_info("\nConfigure MySQL connection:")
+            config_data["mysql_host"] = input_text(
+                "MySQL host:",
+                default="localhost",
+            )
+            config_data["mysql_port"] = input_text(
+                "MySQL port:",
+                default="3306",
+            )
+            config_data["mysql_database"] = input_text(
+                "Database name:",
+                default="pyworkflow",
+            )
+            config_data["mysql_user"] = input_text(
+                "Database user:",
+                default="pyworkflow",
+            )
+            config_data["mysql_password"] = input_text(
+                "Database password:",
+                default="pyworkflow",
+            )
+
     # DynamoDB configuration
     elif storage_type == "dynamodb":
         if non_interactive:
@@ -517,6 +640,40 @@ def _run_interactive_configuration(
                 )
                 config_data["dynamodb_endpoint_url"] = endpoint
 
+    # Cassandra configuration
+    elif storage_type == "cassandra":
+        if non_interactive:
+            config_data["cassandra_contact_points"] = "localhost"
+            config_data["cassandra_port"] = "9042"
+            config_data["cassandra_keyspace"] = "pyworkflow"
+        else:
+            print_info("\nConfigure Cassandra connection:")
+            contact_points = input_text(
+                "Cassandra contact points (comma-separated):",
+                default="localhost",
+            )
+            config_data["cassandra_contact_points"] = contact_points
+
+            config_data["cassandra_port"] = input_text(
+                "Cassandra port:",
+                default="9042",
+            )
+            config_data["cassandra_keyspace"] = input_text(
+                "Keyspace name:",
+                default="pyworkflow",
+            )
+
+            # Optional authentication
+            if confirm("Use Cassandra authentication?", default=False):
+                config_data["cassandra_user"] = input_text(
+                    "Cassandra user:",
+                    default="cassandra",
+                )
+                config_data["cassandra_password"] = input_text(
+                    "Cassandra password:",
+                    default="cassandra",
+                )
+
     return config_data
 
 
@@ -543,6 +700,22 @@ def _setup_docker_infrastructure(
             postgres_password=config_data.get("postgres_password", "pyworkflow"),
             postgres_database=config_data.get("postgres_database", "pyworkflow"),
         )
+    elif storage_type == "cassandra":
+        compose_content = generate_cassandra_docker_compose_content(
+            cassandra_host="cassandra",
+            cassandra_port=int(config_data.get("cassandra_port", "9042")),
+            cassandra_keyspace=config_data.get("cassandra_keyspace", "pyworkflow"),
+            cassandra_user=config_data.get("cassandra_user"),
+            cassandra_password=config_data.get("cassandra_password"),
+        )
+    elif storage_type == "mysql":
+        compose_content = generate_mysql_docker_compose_content(
+            mysql_host="mysql",
+            mysql_port=int(config_data.get("mysql_port", "3306")),
+            mysql_user=config_data.get("mysql_user", "pyworkflow"),
+            mysql_password=config_data.get("mysql_password", "pyworkflow"),
+            mysql_database=config_data.get("mysql_database", "pyworkflow"),
+        )
     else:
         compose_content = generate_docker_compose_content(
             storage_type=storage_type,
@@ -574,10 +747,14 @@ def _setup_docker_infrastructure(
     print_info("\n  Starting services...")
     print_info("")
 
-    # Include postgres in services to start if using postgres storage
+    # Include postgres/cassandra/mysql in services to start if using those storage types
     services_to_start = ["redis"]
     if storage_type == "postgres":
         services_to_start.insert(0, "postgres")
+    elif storage_type == "cassandra":
+        services_to_start.insert(0, "cassandra")
+    elif storage_type == "mysql":
+        services_to_start.insert(0, "mysql")
     if dashboard_available:
         services_to_start.extend(["dashboard-backend", "dashboard-frontend"])
 
@@ -594,6 +771,12 @@ def _setup_docker_infrastructure(
         if storage_type == "postgres":
             postgres_port = config_data.get("postgres_port", "5432")
             ports_in_use = f"{postgres_port}, {ports_in_use}"
+        elif storage_type == "cassandra":
+            cassandra_port = config_data.get("cassandra_port", "9042")
+            ports_in_use = f"{cassandra_port}, {ports_in_use}"
+        elif storage_type == "mysql":
+            mysql_port = config_data.get("mysql_port", "3306")
+            ports_in_use = f"{mysql_port}, {ports_in_use}"
         print_info(f"    • Check if ports {ports_in_use} are already in use")
         print_info("    • View logs: docker compose logs")
         print_info("    • Try: docker compose down && docker compose up -d")
@@ -612,6 +795,16 @@ def _setup_docker_infrastructure(
         pg_port = int(config_data.get("postgres_port", "5432"))
         health_checks["PostgreSQL"] = {"type": "tcp", "host": "localhost", "port": pg_port}
 
+    # Add Cassandra health check if using cassandra storage
+    if storage_type == "cassandra":
+        cass_port = int(config_data.get("cassandra_port", "9042"))
+        health_checks["Cassandra"] = {"type": "tcp", "host": "localhost", "port": cass_port}
+
+    # Add MySQL health check if using mysql storage
+    if storage_type == "mysql":
+        mysql_port_num = int(config_data.get("mysql_port", "3306"))
+        health_checks["MySQL"] = {"type": "tcp", "host": "localhost", "port": mysql_port_num}
+
     # Only check dashboard health if it was started
     if dashboard_available:
         health_checks["Dashboard Backend"] = {
@@ -672,6 +865,12 @@ def _show_next_steps(
         if config_data.get("storage_type") == "postgres":
             postgres_port = config_data.get("postgres_port", "5432")
             print_info(f"  • PostgreSQL:         localhost:{postgres_port}")
+        elif config_data.get("storage_type") == "cassandra":
+            cassandra_port = config_data.get("cassandra_port", "9042")
+            print_info(f"  • Cassandra:          localhost:{cassandra_port}")
+        elif config_data.get("storage_type") == "mysql":
+            mysql_port = config_data.get("mysql_port", "3306")
+            print_info(f"  • MySQL:              localhost:{mysql_port}")
         print_info("  • Redis:              redis://localhost:6379")
         if dashboard_available:
             print_info("  • Dashboard:          http://localhost:5173")

pyworkflow/cli/utils/config_generator.py

@@ -24,9 +24,19 @@ def generate_yaml_config(
     postgres_user: str | None = None,
     postgres_password: str | None = None,
     postgres_database: str | None = None,
+    mysql_host: str | None = None,
+    mysql_port: str | None = None,
+    mysql_user: str | None = None,
+    mysql_password: str | None = None,
+    mysql_database: str | None = None,
     dynamodb_table_name: str | None = None,
     dynamodb_region: str | None = None,
     dynamodb_endpoint_url: str | None = None,
+    cassandra_contact_points: list[str] | None = None,
+    cassandra_port: str | None = None,
+    cassandra_keyspace: str | None = None,
+    cassandra_user: str | None = None,
+    cassandra_password: str | None = None,
 ) -> str:
     """
     Generate YAML configuration content.
@@ -34,7 +44,8 @@ def generate_yaml_config(
     Args:
         module: Optional workflow module path (e.g., "myapp.workflows")
         runtime: Runtime type (e.g., "celery", "local")
-        storage_type: Storage backend type (e.g., "sqlite", "file", "memory", "postgres")
+        storage_type: Storage backend type (e.g., "sqlite", "file", "memory", "postgres",
+            "cassandra")
         storage_path: Optional storage path for file/sqlite backends
         broker_url: Celery broker URL
         result_backend: Celery result backend URL
@@ -46,6 +57,11 @@ def generate_yaml_config(
         dynamodb_table_name: Optional DynamoDB table name
         dynamodb_region: Optional AWS region for DynamoDB
         dynamodb_endpoint_url: Optional local DynamoDB endpoint URL
+        cassandra_contact_points: List of Cassandra contact points (for cassandra backend)
+        cassandra_port: Cassandra CQL native transport port (for cassandra backend)
+        cassandra_keyspace: Cassandra keyspace name (for cassandra backend)
+        cassandra_user: Optional Cassandra user (for cassandra backend)
+        cassandra_password: Optional Cassandra password (for cassandra backend)
 
     Returns:
         YAML configuration as string
@@ -84,6 +100,17 @@ def generate_yaml_config(
             storage_config["password"] = postgres_password
         if postgres_database:
             storage_config["database"] = postgres_database
+    elif storage_type == "mysql":
+        if mysql_host:
+            storage_config["host"] = mysql_host
+        if mysql_port:
+            storage_config["port"] = int(mysql_port)
+        if mysql_user:
+            storage_config["user"] = mysql_user
+        if mysql_password:
+            storage_config["password"] = mysql_password
+        if mysql_database:
+            storage_config["database"] = mysql_database
     elif storage_type == "dynamodb":
         if dynamodb_table_name:
             storage_config["table_name"] = dynamodb_table_name
@@ -91,6 +118,17 @@ def generate_yaml_config(
             storage_config["region"] = dynamodb_region
         if dynamodb_endpoint_url:
             storage_config["endpoint_url"] = dynamodb_endpoint_url
+    elif storage_type == "cassandra":
+        if cassandra_contact_points:
+            storage_config["contact_points"] = cassandra_contact_points
+        if cassandra_port:
+            storage_config["port"] = int(cassandra_port)
+        if cassandra_keyspace:
+            storage_config["keyspace"] = cassandra_keyspace
+        if cassandra_user:
+            storage_config["username"] = cassandra_user
+        if cassandra_password:
+            storage_config["password"] = cassandra_password
     config["storage"] = storage_config
 
     # Celery configuration (only for celery runtime)
@@ -180,6 +218,13 @@ def load_yaml_config(path: Path | None = None) -> dict[str, Any]:
         with open(path) as f:
             config = yaml.safe_load(f)
 
+        # Handle empty config file (yaml.safe_load returns None for empty files)
+        if config is None:
+            raise ValueError(
+                "Config file is empty or contains only comments. "
+                "Please add valid configuration or delete the file to create a new one."
+            )
+
         # Ensure it's a dict
         if not isinstance(config, dict):
             raise ValueError("Config file must contain a YAML dictionary")
@@ -258,6 +303,14 @@ def display_config_summary(config: dict[str, Any]) -> list[str]:
         user = storage.get("user", "pyworkflow")
         lines.append(f"  PostgreSQL: {user}@{host}:{port}/{database}")
 
+    # MySQL-specific config
+    if storage_type == "mysql":
+        host = storage.get("host", "localhost")
+        port = storage.get("port", 3306)
+        database = storage.get("database", "pyworkflow")
+        user = storage.get("user", "pyworkflow")
+        lines.append(f"  MySQL: {user}@{host}:{port}/{database}")
+
     # DynamoDB-specific config
     if storage_type == "dynamodb":
         if "table_name" in storage:
@@ -267,6 +320,16 @@ def display_config_summary(config: dict[str, Any]) -> list[str]:
         if "endpoint_url" in storage:
             lines.append(f"  Endpoint URL: {storage['endpoint_url']}")
 
+    # Cassandra-specific config
+    if storage_type == "cassandra":
+        contact_points = storage.get("contact_points", ["localhost"])
+        port = storage.get("port", 9042)
+        keyspace = storage.get("keyspace", "pyworkflow")
+        hosts = ", ".join(contact_points) if isinstance(contact_points, list) else contact_points
+        lines.append(f"  Cassandra: {hosts}:{port}/{keyspace}")
+        if "username" in storage:
+            lines.append(f"  Cassandra User: {storage['username']}")
+
     # Celery (if applicable)
     if runtime == "celery" and "celery" in config:
         celery = config["celery"]
@@ -314,10 +377,20 @@ def validate_config(config: dict[str, Any]) -> tuple[bool, list[str]]:
         storage_type = storage.get("type")
         if not storage_type:
             errors.append("Missing storage 'type'")
-        elif storage_type not in ["file", "memory", "sqlite", "redis", "postgres", "dynamodb"]:
+        elif storage_type not in [
+            "file",
+            "memory",
+            "sqlite",
+            "redis",
+            "postgres",
+            "mysql",
+            "dynamodb",
+            "cassandra",
+        ]:
             errors.append(
                 f"Invalid storage type: {storage_type}. "
-                "Must be 'file', 'memory', 'sqlite', 'redis', 'postgres' or 'dynamodb'"
+                "Must be 'file', 'memory', 'sqlite', 'redis', 'postgres', 'mysql', 'dynamodb' "
+                "or 'cassandra'"
             )
 
     # Check Celery config if using celery runtime