FlowerPower 0.9.13.1__py3-none-any.whl → 1.0.0b1__py3-none-any.whl

flowerpower/_web/routes/pipelines.py

@@ -1,173 +0,0 @@
-import os
-from pathlib import Path
-from typing import Dict, List, Optional, Any
-
-from fastapi import APIRouter, Request, Form, HTTPException, Depends
-from fastapi.responses import HTMLResponse, RedirectResponse
-from pydantic import BaseModel
-
-from ...pipeline import PipelineManager, Pipeline
-from .. import templates
-
-router = APIRouter(prefix="/pipelines", tags=["pipelines"])
-
-base_dir = os.environ.get("FLOWERPOWER_BASE_DIR", str(Path.cwd()))
-
-class PipelineRunRequest(BaseModel):
-    name: str
-    executor: Optional[str] = None
-    inputs: Optional[Dict[str, Any]] = None
-    final_vars: Optional[List[str]] = None
-    config: Optional[Dict[str, Any]] = None
-    with_tracker: bool = False
-    with_opentelemetry: bool = False
-    with_progressbar: bool = False
-
-class PipelineScheduleRequest(BaseModel):
-    name: str
-    trigger_type: str = "cron"
-    executor: Optional[str] = None
-    inputs: Optional[Dict[str, Any]] = None
-    final_vars: Optional[List[str]] = None
-    config: Optional[Dict[str, Any]] = None
-    with_tracker: bool = False
-    with_opentelemetry: bool = False
-    with_progressbar: bool = False
-    paused: bool = False
-    crontab: Optional[str] = None
-    overwrite: bool = False
-
-@router.get("/", response_class=HTMLResponse)
-async def list_pipelines(request: Request):
-    """List all available pipelines"""
-    pipeline_manager = PipelineManager(base_dir=base_dir)
-    pipelines = pipeline_manager.list_pipelines() or []  # Ensure we have a list
-    
-    return templates.TemplateResponse(
-        "pipelines/list.html",
-        {
-            "request": request,
-            "pipelines": pipelines,
-            "base_dir": base_dir
-        }
-    )
-
-@router.get("/{name}", response_class=HTMLResponse)
-async def get_pipeline(request: Request, name: str):
-    """Get detailed pipeline view"""
-    pipeline = Pipeline(name=name, base_dir=base_dir)
-    summary = pipeline.get_summary()
-    
-    # Get pipeline code
-    code = None
-    pipeline_path = Path(base_dir) / "pipelines" / f"{name}.py"
-    if pipeline_path.exists():
-        with open(pipeline_path, "r") as f:
-            code = f.read()
-    
-    return templates.TemplateResponse(
-        "pipelines/detail.html",
-        {
-            "request": request,
-            "pipeline": name,
-            "summary": summary,
-            "code": code,
-            "base_dir": base_dir
-        }
-    )
-
-@router.post("/{name}/run")
-async def run_pipeline(request: Request, name: str, run_data: PipelineRunRequest):
-    """Run a pipeline"""
-    try:
-        pipeline = Pipeline(name=name, base_dir=base_dir)
-        result = pipeline.run(
-            inputs=run_data.inputs,
-            final_vars=run_data.final_vars,
-            config=run_data.config,
-            executor=run_data.executor,
-            with_tracker=run_data.with_tracker,
-            with_opentelemetry=run_data.with_opentelemetry,
-            with_progressbar=run_data.with_progressbar
-        )
-        
-        return {"success": True, "message": f"Pipeline {name} executed successfully", "result": result}
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
-
-@router.post("/{name}/schedule")
-async def schedule_pipeline(request: Request, name: str, schedule_data: PipelineScheduleRequest):
-    """Schedule a pipeline"""
-    try:
-        pipeline = Pipeline(name=name, base_dir=base_dir)
-        kwargs = {}
-        if schedule_data.crontab:
-            kwargs["crontab"] = schedule_data.crontab
-            
-        schedule_id = pipeline.schedule(
-            trigger_type=schedule_data.trigger_type,
-            inputs=schedule_data.inputs,
-            final_vars=schedule_data.final_vars,
-            config=schedule_data.config,
-            executor=schedule_data.executor,
-            with_tracker=schedule_data.with_tracker,
-            with_opentelemetry=schedule_data.with_opentelemetry,
-            with_progressbar=schedule_data.with_progressbar,
-            paused=schedule_data.paused,
-            overwrite=schedule_data.overwrite,
-            **kwargs
-        )
-        
-        return {"success": True, "message": f"Pipeline {name} scheduled successfully", "schedule_id": schedule_id}
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
-
-@router.post("/{name}/update-code")
-async def update_pipeline_code(
-    request: Request, 
-    name: str, 
-    code: str = Form(...),
-):
-    """Update pipeline code"""
-    try:
-        pipeline_path = Path(base_dir) / "pipelines" / f"{name}.py"
-        with open(pipeline_path, "w") as f:
-            f.write(code)
-        
-        return {"success": True, "message": f"Pipeline code updated successfully"}
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
-
-@router.get("/{name}/edit", response_class=HTMLResponse)
-async def edit_pipeline(request: Request, name: str):
-    """Edit pipeline code and configuration"""
-    pipeline = Pipeline(name=name, base_dir=base_dir)
-    summary = pipeline.get_summary()
-    
-    # Get pipeline code
-    code = None
-    pipeline_path = Path(base_dir) / "pipelines" / f"{name}.py"
-    if pipeline_path.exists():
-        with open(pipeline_path, "r") as f:
-            code = f.read()
-    
-    return templates.TemplateResponse(
-        "pipelines/edit.html",
-        {
-            "request": request,
-            "pipeline": name,
-            "summary": summary,
-            "code": code,
-            "base_dir": base_dir
-        }
-    )
-
-@router.post("/new")
-async def create_pipeline(name: str = Form(...)):
-    """Create a new pipeline"""
-    try:
-        pipeline_manager = PipelineManager(base_dir=base_dir)
-        pipeline_manager.new(name=name)
-        return RedirectResponse(url=f"/pipelines/{name}/edit", status_code=303)
-    except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))

flowerpower/_web/routes/scheduler.py

@@ -1,136 +0,0 @@
-import os
-import importlib.util
-from pathlib import Path
-
-from fastapi import APIRouter, Request, HTTPException, BackgroundTasks
-from fastapi.responses import HTMLResponse
-
-from ...pipeline import PipelineManager
-from .. import templates
-
-router = APIRouter(prefix="/scheduler", tags=["scheduler"])
-
-base_dir = os.environ.get("FLOWERPOWER_BASE_DIR", str(Path.cwd()))
-
-# Check if scheduler is available
-has_scheduler = importlib.util.find_spec("apscheduler") is not None
-if not has_scheduler:
-    # Create minimal router if scheduler not available
-    @router.get("/", response_class=HTMLResponse)
-    async def scheduler_not_available(request: Request):
-        return templates.TemplateResponse(
-            "scheduler/not_available.html",
-            {"request": request}
-        )
-else:
-    from ...scheduler import SchedulerManager
-
-    # Worker process handler
-    worker_process = None
-
-    def start_worker_process(base_dir):
-        global worker_process
-        import subprocess
-        import sys
-
-        cmd = [
-            sys.executable, 
-            "-m", "flowerpower", 
-            "scheduler", 
-            "start-worker",
-            "--base-dir", base_dir
-        ]
-        worker_process = subprocess.Popen(cmd)
-
-    def stop_worker_process():
-        global worker_process
-        if worker_process:
-            worker_process.terminate()
-            worker_process = None
-
-    @router.get("/", response_class=HTMLResponse)
-    async def scheduler_dashboard(request: Request):
-        """Scheduler dashboard with schedules and jobs"""
-        pipeline_manager = PipelineManager(base_dir=base_dir)
-        
-        with SchedulerManager(fs=pipeline_manager._fs, role="scheduler") as sm:
-            schedules = sm.get_schedules(as_dict=True)
-            jobs = sm.get_jobs(as_dict=True)
-            
-        return templates.TemplateResponse(
-            "scheduler/dashboard.html",
-            {
-                "request": request,
-                "schedules": schedules,
-                "jobs": jobs,
-                "worker_running": worker_process is not None,
-                "base_dir": base_dir
-            }
-        )
-
-    @router.post("/start-worker")
-    async def start_worker(background_tasks: BackgroundTasks):
-        """Start scheduler worker"""
-        global worker_process
-        
-        if worker_process is not None:
-            return {"success": False, "message": "Worker is already running"}
-            
-        try:
-            background_tasks.add_task(start_worker_process, base_dir)
-            return {"success": True, "message": "Worker started successfully"}
-        except Exception as e:
-            raise HTTPException(status_code=500, detail=str(e))
-
-    @router.post("/stop-worker")
-    async def stop_worker():
-        """Stop scheduler worker"""
-        global worker_process
-        
-        if worker_process is None:
-            return {"success": False, "message": "No worker is running"}
-            
-        try:
-            stop_worker_process()
-            return {"success": True, "message": "Worker stopped successfully"}
-        except Exception as e:
-            raise HTTPException(status_code=500, detail=str(e))
-
-    @router.post("/pause-schedule/{schedule_id}")
-    async def pause_schedule(schedule_id: str):
-        """Pause a scheduled pipeline"""
-        try:
-            pipeline_manager = PipelineManager(base_dir=base_dir)
-            
-            with SchedulerManager(fs=pipeline_manager._fs, role="scheduler") as sm:
-                sm.pause_schedule(schedule_id)
-                
-            return {"success": True, "message": f"Schedule {schedule_id} paused successfully"}
-        except Exception as e:
-            raise HTTPException(status_code=500, detail=str(e))
-
-    @router.post("/resume-schedule/{schedule_id}")
-    async def resume_schedule(schedule_id: str):
-        """Resume a paused schedule"""
-        try:
-            pipeline_manager = PipelineManager(base_dir=base_dir)
-            
-            with SchedulerManager(fs=pipeline_manager._fs, role="scheduler") as sm:
-                sm.resume_schedule(schedule_id)
-                
-            return {"success": True, "message": f"Schedule {schedule_id} resumed successfully"}
-        except Exception as e:
-            raise HTTPException(status_code=500, detail=str(e))
-
-    @router.delete("/remove-schedule/{schedule_id}")
-    async def remove_schedule(schedule_id: str):
-        """Remove a schedule"""
-        try:
-            pipeline_manager = PipelineManager(base_dir=base_dir)
-            
-            with SchedulerManager(fs=pipeline_manager._fs, role="scheduler") as sm:
-                sm.remove_schedule(schedule_id)
-                
-            return {"success": True, "message": f"Schedule {schedule_id} removed successfully"}
-        except Exception as e:
-            raise HTTPException(status_code=500, detail=str(e))

flowerpower/cfg/pipeline/tracker.py

@@ -1,14 +0,0 @@
-from munch import Munch, munchify
-from pydantic import Field
-
-from ..base import BaseConfig
-
-
-class PipelineTrackerConfig(BaseConfig):
-    project_id: int | None = None
-    # version: str | None = None
-    dag_name: str | None = None
-    tags: dict | Munch = Field(default_factory=dict)
-
-    def model_post_init(self, __context):
-        self.tags = munchify(self.tags)

flowerpower/cfg/project/open_telemetry.py

@@ -1,8 +0,0 @@
-from pydantic import Field
-
-from ..base import BaseConfig
-
-
-class ProjectOpenTelemetryConfig(BaseConfig):
-    host: str = Field(default="localhost")
-    port: int = Field(default=6831)

flowerpower/cfg/project/tracker.py

@@ -1,11 +0,0 @@
-from pydantic import Field
-
-from ..base import BaseConfig
-
-
-class ProjectTrackerConfig(BaseConfig):
-    username: str | None = Field(default=None)
-    api_url: str = "http://localhost:8241"
-    ui_url: str = "http://localhost:8242"
-    api_key: str | None = Field(default=None)
-    verify: bool = False

flowerpower/cfg/project/worker.py

@@ -1,19 +0,0 @@
-import datetime as dt
-
-from munch import Munch, munchify
-from pydantic import Field
-
-from ..base import BaseConfig
-
-
-class ProjectWorkerConfig(BaseConfig):
-    data_store: dict | Munch = Field(default_factory=dict)
-    event_broker: dict | Munch = Field(default_factory=dict)
-    cleanup_interval: int | float | dt.timedelta = Field(default=300)  # int in secods
-    max_concurrent_jobs: int = Field(default=10)
-
-    def model_post_init(self, __context):
-        if isinstance(self.data_store, dict):
-            self.data_store = munchify(self.data_store)
-        if isinstance(self.event_broker, dict):
-            self.event_broker = munchify(self.event_broker)

flowerpower/cli/scheduler.py

@@ -1,309 +0,0 @@
-import typer
-
-from ..scheduler import SchedulerManager  # Adjust import as needed
-from .utils import \
-    parse_dict_or_list_param  # Assuming you have this utility function
-
-# Create a Typer app for scheduler commands
-app = typer.Typer(help="Scheduler management commands")
-
-
-@app.command()
-def start_worker(
-    name: str | None = None,
-    base_dir: str | None = None,
-    background: bool = False,
-    storage_options: str | None = None,
-):
-    """
-    Start a scheduler worker.
-
-    Args:
-        name: Name of the scheduler
-        base_dir: Base directory for the scheduler
-        background: Run in background
-        storage_options: Storage options as JSON or key=value pairs
-    """
-    parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
-
-    with SchedulerManager(
-        name, base_dir, storage_options=parsed_storage_options
-    ) as manager:
-        manager.start_worker(background=background)
-
-
-@app.command()
-def remove_all_schedules(
-    name: str | None = None,
-    base_dir: str | None = None,
-    storage_options: str | None = None,
-):
-    """
-    Remove all schedules from the scheduler.
-
-    Args:
-        name: Name of the scheduler
-        base_dir: Base directory for the scheduler
-        storage_options: Storage options as JSON or key=value pairs
-    """
-    parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
-
-    with SchedulerManager(
-        name, base_dir, role="scheduler", storage_options=parsed_storage_options
-    ) as manager:
-        manager.remove_all_schedules()
-
-
-# @app.command()
-# def add_schedule(
-#     name: str | None = None,
-#     base_dir: str | None = None,
-#     storage_options: str | None = None,
-#     trigger_type: str = "cron",
-#     **kwargs,
-# ) -> str:
-#     """
-#     Add a schedule to the scheduler.
-
-#     Args:
-#         name: Name of the scheduler
-#         base_dir: Base directory for the scheduler
-#         storage_options: Storage options as JSON or key=value pairs
-#         trigger_type: Type of trigger (cron, interval, etc.)
-#         **kwargs: Additional schedule parameters
-#     """
-#     parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
-
-#     with SchedulerManager(name, base_dir, role="scheduler") as manager:
-#         schedule_id = manager.add_schedule(
-#             storage_options=parsed_storage_options, type=trigger_type, **kwargs
-#         )
-
-#     typer.echo(f"Schedule added with ID: {schedule_id}")
-#     return schedule_id
-
-
-# @app.command()
-# def add_job(
-#     name: str | None = None,
-#     base_dir: str | None = None,
-#     storage_options: str | None = None,
-#     **kwargs,
-# ) -> str:
-#     """
-#     Add a job to the scheduler.
-
-#     Args:
-#         name: Name of the scheduler
-#         base_dir: Base directory for the scheduler
-#         storage_options: Storage options as JSON or key=value pairs
-#         **kwargs: Job parameters
-#     """
-#     parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
-
-#     with SchedulerManager(name, base_dir, role="scheduler") as manager:
-#         job_id = manager.add_job(storage_options=parsed_storage_options, **kwargs)
-
-#     typer.echo(f"Job added with ID: {job_id}")
-#     return str(job_id)
-
-
-# @app.command()
-# def run_job(
-#     name: str | None = None,
-#     base_dir: str | None = None,
-#     storage_options: str | None = None,
-#     **kwargs,
-# ):
-#     """
-#     Run a job and return its result.
-
-#     Args:
-#         name: Name of the scheduler
-#         base_dir: Base directory for the scheduler
-#         storage_options: Storage options as JSON or key=value pairs
-#         **kwargs: Job parameters
-#     """
-#     parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
-
-#     with SchedulerManager(name, base_dir, role="scheduler") as manager:
-#         result = manager.run_job(storage_options=parsed_storage_options, **kwargs)
-
-#     typer.echo("Job result:")
-#     typer.echo(result)
-#     return result
-
-
-@app.command()
-def get_schedules(
-    name: str | None = None,
-    base_dir: str | None = None,
-    storage_options: str | None = None,
-):
-    """
-    Get all schedules from the scheduler.
-
-    Args:
-        name: Name of the scheduler
-        base_dir: Base directory for the scheduler
-        storage_options: Storage options as JSON or key=value pairs
-    """
-    parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
-
-    with SchedulerManager(
-        name, base_dir, role="scheduler", storage_options=parsed_storage_options
-    ) as manager:
-        schedules = manager.get_schedules()
-
-    typer.echo("Schedules:")
-    for schedule in schedules:
-        typer.echo(schedule)
-    return schedules
-
-
-@app.command()
-def get_tasks(
-    name: str | None = None,
-    base_dir: str | None = None,
-    storage_options: str | None = None,
-):
-    """
-    Get all tasks from the scheduler.
-
-    Args:
-        name: Name of the scheduler
-        base_dir: Base directory for the scheduler
-        storage_options: Storage options as JSON or key=value pairs
-    """
-    parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
-
-    with SchedulerManager(
-        name, base_dir, role="scheduler", storage_options=parsed_storage_options
-    ) as manager:
-        tasks = manager.get_tasks()
-
-    typer.echo("Tasks:")
-    for task in tasks:
-        typer.echo(task)
-    return tasks
-
-
-@app.command()
-def get_jobs(
-    name: str | None = None,
-    base_dir: str | None = None,
-    storage_options: str | None = None,
-):
-    """
-    Get all jobs from the scheduler.
-
-    Args:
-        name: Name of the scheduler
-        base_dir: Base directory for the scheduler
-        storage_options: Storage options as JSON or key=value pairs
-    """
-    parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
-
-    with SchedulerManager(
-        name, base_dir, role="scheduler", storage_options=parsed_storage_options
-    ) as manager:
-        jobs = manager.get_jobs()
-
-    typer.echo("Jobs:")
-    for job in jobs:
-        typer.echo(job)
-    return jobs
-
-
-@app.command()
-def get_job_result(
-    job_id: str,
-    name: str | None = None,
-    base_dir: str | None = None,
-    storage_options: str | None = None,
-):
-    """
-    Get the result of a specific job.
-
-    Args:
-        job_id: ID of the job
-        name: Name of the scheduler
-        base_dir: Base directory for the scheduler
-        storage_options: Storage options as JSON or key=value pairs
-    """
-    parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
-
-    with SchedulerManager(
-        name, base_dir, role="scheduler", storage_options=parsed_storage_options
-    ) as manager:
-        result = manager.get_job_result(job_id)
-
-    typer.echo("Job Result:")
-    typer.echo(result)
-    return result
-
-
-@app.command()
-def show_schedules(
-    name: str | None = None,
-    base_dir: str | None = None,
-    storage_options: str | None = None,
-):
-    """
-    Show all schedules in the scheduler.
-
-    Args:
-        name: Name of the scheduler
-        base_dir: Base directory for the scheduler
-        storage_options: Storage options as JSON or key=value pairs
-    """
-    parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
-
-    with SchedulerManager(
-        name, base_dir, role="scheduler", storage_options=parsed_storage_options
-    ) as manager:
-        manager.show_schedules()
-
-
-@app.command()
-def show_jobs(
-    name: str | None = None,
-    base_dir: str | None = None,
-    storage_options: str | None = None,
-):
-    """
-    Show all jobs in the scheduler.
-
-    Args:
-        name: Name of the scheduler
-        base_dir: Base directory for the scheduler
-        storage_options: Storage options as JSON or key=value pairs
-    """
-    parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
-
-    with SchedulerManager(
-        name, base_dir, role="scheduler", storage_options=parsed_storage_options
-    ) as manager:
-        manager.show_jobs()
-
-
-@app.command()
-def show_tasks(
-    name: str | None = None,
-    base_dir: str | None = None,
-    storage_options: str | None = None,
-):
-    """
-    Show all tasks in the scheduler.
-
-    Args:
-        name: Name of the scheduler
-        base_dir: Base directory for the scheduler
-        storage_options: Storage options as JSON or key=value pairs
-    """
-    parsed_storage_options = parse_dict_or_list_param(storage_options, "dict") or {}
-
-    with SchedulerManager(
-        name, base_dir, role="scheduler", storage_options=parsed_storage_options
-    ) as manager:
-        manager.show_tasks()