flowyml 1.1.0__py3-none-any.whl

flowyml/ui/backend/routers/schedules.py

@@ -0,0 +1,222 @@
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+from flowyml.core.scheduler import PipelineScheduler
+from flowyml.registry.pipeline_registry import pipeline_registry
+
+router = APIRouter()
+# Note: In a real app, the scheduler instance should be a singleton managed by the app state
+# For now, we instantiate it here, but it might not persist state across reloads if not handled carefully.
+# Ideally, the scheduler is started when the backend starts.
+scheduler = PipelineScheduler()
+scheduler.start()  # Start the scheduler thread
+
+
+class ScheduleRequest(BaseModel):
+    name: str
+    pipeline_name: str
+    schedule_type: str  # 'daily', 'hourly', 'interval', 'cron'
+    hour: int | None = 0
+    minute: int | None = 0
+    interval_seconds: int | None = 0
+    cron_expression: str | None = None
+    timezone: str = "UTC"
+    project_name: str | None = None
+
+
+@router.get("/")
+async def list_schedules():
+    """List all active schedules."""
+    # Convert Schedule objects to dicts for JSON serialization
+    schedules = scheduler.list_schedules()
+    return [
+        {
+            "pipeline_name": s.pipeline_name,
+            "schedule_type": s.schedule_type,
+            "schedule_value": s.schedule_value,
+            "enabled": s.enabled,
+            "last_run": s.last_run,
+            "next_run": s.next_run,
+            "timezone": s.timezone,
+        }
+        for s in schedules
+    ]
+
+
+@router.get("/health")
+async def get_scheduler_health():
+    """Get scheduler health metrics."""
+    return scheduler.health_check()
+
+
+@router.post("/")
+async def create_schedule(schedule: ScheduleRequest):
+    """Create a new schedule."""
+    # 1. Look up pipeline factory
+    pipeline_factory = pipeline_registry.get(schedule.pipeline_name)
+
+    if not pipeline_factory:
+        # Try to see if it's a template
+        from flowyml.core.templates import TEMPLATES, create_from_template
+
+        if schedule.pipeline_name in TEMPLATES:
+            # For templates, we need a way to instantiate them with default args or args provided in request
+            # This is a simplification for the demo
+            def template_wrapper():
+                # Create and run template pipeline
+                # Note: This assumes default args work. In reality, we'd need more config.
+                p = create_from_template(schedule.pipeline_name)
+                return p.run()
+
+            pipeline_func = template_wrapper
+        else:
+            # Check if it's a historical pipeline (in metadata but not registry)
+            # This means we can't run it because we don't have the code loaded
+            from flowyml.storage.metadata import SQLiteMetadataStore
+
+            store = SQLiteMetadataStore()
+            pipelines = store.list_pipelines()
+
+            if schedule.pipeline_name in pipelines:
+                # Try to load pipeline definition
+                from flowyml.storage.metadata import SQLiteMetadataStore
+
+                store = SQLiteMetadataStore()
+                pipeline_def = store.get_pipeline_definition(schedule.pipeline_name)
+
+                if pipeline_def:
+                    # Reconstruct pipeline from definition
+                    from flowyml.core.pipeline import Pipeline
+                    from flowyml.core.context import Context
+
+                    def pipeline_wrapper():
+                        # Reconstruct pipeline each time
+                        p = Pipeline.from_definition(pipeline_def, Context())
+                        return p.run()
+
+                    pipeline_func = pipeline_wrapper
+                else:
+                    raise HTTPException(
+                        status_code=400,
+                        detail=f"Pipeline '{schedule.pipeline_name}' found in history but no definition stored. Please run the pipeline again to enable scheduling.",
+                    )
+            else:
+                raise HTTPException(
+                    status_code=404,
+                    detail=f"Pipeline '{schedule.pipeline_name}' not found in registry. Please register it using @register_pipeline.",
+                )
+    else:
+        # Create a wrapper to instantiate and run the pipeline
+        def pipeline_wrapper():
+            # Instantiate fresh pipeline for each run
+            p = pipeline_factory()
+            return p.run()
+
+        pipeline_func = pipeline_wrapper
+
+    # 2. Schedule it
+    try:
+        if schedule.schedule_type == "daily":
+            scheduler.schedule_daily(
+                name=schedule.name,
+                pipeline_func=pipeline_func,
+                hour=schedule.hour,
+                minute=schedule.minute,
+                timezone=schedule.timezone,
+            )
+        elif schedule.schedule_type == "hourly":
+            scheduler.schedule_hourly(
+                name=schedule.name,
+                pipeline_func=pipeline_func,
+                minute=schedule.minute,
+                timezone=schedule.timezone,
+            )
+        elif schedule.schedule_type == "interval":
+            scheduler.schedule_interval(
+                name=schedule.name,
+                pipeline_func=pipeline_func,
+                seconds=schedule.interval_seconds,
+                timezone=schedule.timezone,
+            )
+        elif schedule.schedule_type == "cron":
+            if not schedule.cron_expression:
+                raise HTTPException(status_code=400, detail="Cron expression required for cron schedule")
+            scheduler.schedule_cron(
+                name=schedule.name,
+                pipeline_func=pipeline_func,
+                cron_expression=schedule.cron_expression,
+                timezone=schedule.timezone,
+            )
+        else:
+            raise HTTPException(status_code=400, detail="Invalid schedule type")
+
+        return {"status": "success", "message": f"Scheduled '{schedule.name}'"}
+
+    except ImportError as e:
+        raise HTTPException(status_code=501, detail=str(e))
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.delete("/{schedule_name}")
+async def delete_schedule(schedule_name: str):
+    """Remove a schedule."""
+    scheduler.unschedule(schedule_name)
+    return {"status": "success", "message": f"Schedule {schedule_name} removed"}
+
+
+@router.post("/{schedule_name}/enable")
+async def enable_schedule(schedule_name: str):
+    """Enable a schedule."""
+    scheduler.enable(schedule_name)
+    return {"status": "success"}
+
+
+@router.post("/{schedule_name}/disable")
+async def disable_schedule(schedule_name: str):
+    """Disable a schedule."""
+    scheduler.disable(schedule_name)
+    return {"status": "success"}
+
+
+@router.get("/{schedule_name}/history")
+async def get_schedule_history(schedule_name: str, limit: int = 50):
+    """Get execution history for a schedule."""
+    return scheduler.get_history(schedule_name, limit)
+
+
+@router.get("/registered-pipelines")
+async def list_registered_pipelines(project: str = None):
+    """List all pipelines available for scheduling."""
+    from flowyml.core.templates import list_templates
+    from flowyml.storage.metadata import SQLiteMetadataStore
+
+    registered = pipeline_registry.list_pipelines()
+    templates = list_templates()
+
+    # Also get pipelines from metadata store (historical runs)
+    metadata_pipelines = []
+    try:
+        store = SQLiteMetadataStore()
+        import sqlite3
+
+        conn = sqlite3.connect(store.db_path)
+        cursor = conn.cursor()
+
+        if project:
+            cursor.execute(
+                "SELECT DISTINCT pipeline_name FROM runs WHERE project = ? ORDER BY pipeline_name",
+                (project,),
+            )
+        else:
+            cursor.execute("SELECT DISTINCT pipeline_name FROM runs ORDER BY pipeline_name")
+
+        metadata_pipelines = [row[0] for row in cursor.fetchall()]
+        conn.close()
+    except Exception as e:
+        print(f"Failed to fetch pipelines from metadata store: {e}")
+
+    return {
+        "registered": list(registered.keys()),
+        "templates": templates,
+        "metadata": metadata_pipelines,
+    }

flowyml/ui/backend/routers/traces.py

@@ -0,0 +1,84 @@
+from fastapi import APIRouter, HTTPException
+from flowyml.storage.metadata import SQLiteMetadataStore
+import contextlib
+import builtins
+
+router = APIRouter()
+
+
+@router.get("/")
+async def list_traces(
+    limit: int = 50,
+    trace_id: str | None = None,
+    event_type: str | None = None,
+    project: str | None = None,
+):
+    """List traces, optionally filtered by project."""
+    store = SQLiteMetadataStore()
+
+    # We need to implement list_traces in metadata store or query manually
+    # For now, let's query manually via sqlite
+    import sqlite3
+
+    conn = sqlite3.connect(store.db_path)
+    cursor = conn.cursor()
+
+    query = "SELECT * FROM traces"
+    params = []
+    conditions = []
+
+    if trace_id:
+        conditions.append("trace_id = ?")
+        params.append(trace_id)
+
+    if event_type:
+        conditions.append("event_type = ?")
+        params.append(event_type)
+
+    if project:
+        conditions.append("project = ?")
+        params.append(project)
+
+    if conditions:
+        query += " WHERE " + " AND ".join(conditions)
+
+    query += " ORDER BY start_time DESC LIMIT ?"
+    params.append(limit)
+
+    cursor.execute(query, params)
+    columns = [description[0] for description in cursor.description]
+    rows = cursor.fetchall()
+
+    traces = []
+    import json
+
+    for row in rows:
+        trace = dict(zip(columns, row, strict=False))
+        # Parse JSON fields
+        for field in ["inputs", "outputs", "metadata"]:
+            if trace[field]:
+                with contextlib.suppress(builtins.BaseException):
+                    trace[field] = json.loads(trace[field])
+        traces.append(trace)
+
+    conn.close()
+    return traces
+
+
+@router.get("/{trace_id}")
+async def get_trace(trace_id: str):
+    """Get a specific trace tree."""
+    store = SQLiteMetadataStore()
+    events = store.get_trace(trace_id)
+    if not events:
+        raise HTTPException(status_code=404, detail="Trace not found")
+
+    # Reconstruct tree
+    root_events = [e for e in events if not e["parent_id"]]
+
+    def build_tree(event):
+        children = [e for e in events if e["parent_id"] == event["event_id"]]
+        event["children"] = [build_tree(child) for child in children]
+        return event
+
+    return [build_tree(root) for root in root_events]

flowyml/ui/frontend/Dockerfile

@@ -0,0 +1,20 @@
+# Build stage
+FROM node:18-alpine as build
+
+WORKDIR /app
+
+COPY package.json package-lock.json ./
+RUN npm ci
+
+COPY . .
+RUN npm run build
+
+# Serve stage
+FROM nginx:alpine
+
+COPY --from=build /app/dist /usr/share/nginx/html
+COPY nginx.conf /etc/nginx/conf.d/default.conf
+
+EXPOSE 80
+
+CMD ["nginx", "-g", "daemon off;"]

flowyml/ui/frontend/README.md

@@ -0,0 +1,315 @@
+# flowyml UI Frontend
+
+Modern React-based frontend for the flowyml ML pipeline orchestration framework.
+
+## 🚀 Quick Start
+
+### Prerequisites
+
+- **Node.js**: >= 16.x (recommended: 18.x or 20.x)
+- **npm**: >= 8.x (comes with Node.js)
+
+### Installation
+
+```bash
+# Install dependencies
+npm install
+```
+
+### Development
+
+```bash
+# Start development server with hot reload
+npm run dev
+```
+
+This will start the Vite dev server on `http://localhost:5173`. The dev server will:
+- Hot reload on file changes
+- Proxy API requests to `http://localhost:8080` (backend)
+- Provide fast HMR (Hot Module Replacement)
+
+**Note**: Make sure the backend is running before starting the frontend:
+
+```bash
+# In another terminal, start the backend
+cd ../../..  # Back to repo root
+flowyml ui start --port 8080
+```
+
+### Production Build
+
+```bash
+# Build for production
+npm run build
+```
+
+Production artifacts will be in the `dist/` directory. The flowyml Python backend automatically serves these files when available.
+
+## 📁 Project Structure
+
+```
+frontend/
+├── src/
+│   ├── components/      # Reusable UI components
+│   ├── pages/           # Page-level components
+│   ├── hooks/           # Custom React hooks
+│   ├── utils/           # Utility functions
+│   ├── api/             # API client functions
+│   ├── types/           # TypeScript type definitions
+│   ├── styles/          # Global styles and themes
+│   ├── App.tsx          # Main app component
+│   └── main.tsx         # Application entry point
+├── public/              # Static assets
+├── dist/                # Production build output (generated)
+├── index.html           # HTML entry point
+├── package.json         # Dependencies and scripts
+├── vite.config.ts       # Vite configuration
+├── tsconfig.json        # TypeScript configuration
+└── README.md            # This file
+```
+
+## 🛠️ Available Scripts
+
+### `npm run dev`
+
+Starts the development server with hot reload.
+
+- **URL**: http://localhost:5173
+- **Features**: HMR, fast refresh, TypeScript checking
+- **API Proxy**: Proxies `/api/*` to backend at `localhost:8080`
+
+### `npm run build`
+
+Builds the app for production to the `dist/` folder.
+
+- Optimized for performance
+- Minified and bundled
+- TypeScript type checking
+- Ready to be served by the Python backend
+
+### `npm run preview`
+
+Preview the production build locally:
+
+```bash
+npm run build
+npm run preview
+```
+
+Serves the production build at `http://localhost:4173`.
+
+### `npm run lint`
+
+Run ESLint to check code quality:
+
+```bash
+npm run lint
+```
+
+### `npm run type-check`
+
+Run TypeScript type checking without building:
+
+```bash
+npm run type-check
+```
+
+## 🎨 Tech Stack
+
+- **Framework**: React 18
+- **Build Tool**: Vite
+- **Language**: TypeScript
+- **Styling**: CSS Modules / Tailwind CSS (if configured)
+- **State Management**: React Context / Zustand (if needed)
+- **Routing**: React Router (if multi-page)
+- **API Client**: Fetch API / Axios
+
+## 🔌 API Integration
+
+The frontend communicates with the flowyml backend via REST API:
+
+### Development Mode
+
+In development, Vite proxies API requests to avoid CORS issues:
+
+```typescript
+// vite.config.ts
+export default defineConfig({
+  server: {
+    proxy: {
+      '/api': {
+        target: 'http://localhost:8080',
+        changeOrigin: true
+      }
+    }
+  }
+})
+```
+
+### Production Mode
+
+In production, the frontend is served by the Python backend from the `dist/` folder, so all API requests naturally go to the same origin.
+
+### Example API Usage
+
+```typescript
+// src/api/pipelines.ts
+export async function getPipelines() {
+  const response = await fetch('/api/pipelines');
+  return response.json();
+}
+
+export async function getPipeline(id: string) {
+  const response = await fetch(`/api/pipelines/${id}`);
+  return response.json();
+}
+```
+
+## 🏗️ Development Workflow
+
+### 1. Setup (First Time)
+
+```bash
+# Clone the repo and navigate to frontend
+cd flowyml/ui/frontend
+
+# Install dependencies
+npm install
+```
+
+### 2. Development
+
+```bash
+# Terminal 1: Start backend
+cd ../../..
+flowyml ui start --dev
+
+# Terminal 2: Start frontend
+cd flowyml/ui/frontend
+npm run dev
+```
+
+### 3. Making Changes
+
+1. Edit files in `src/`
+2. Hot reload will update the browser automatically
+3. Check console for errors
+4. Test API integration with backend running
+
+### 4. Building for Production
+
+```bash
+# Build frontend
+npm run build
+
+# Test the backend serves it correctly
+cd ../../..
+flowyml ui start
+# Visit http://localhost:8080
+```
+
+## 🐛 Troubleshooting
+
+### Issue: `npm install` fails
+
+**Solution**: Make sure you have Node.js 16+ installed:
+
+```bash
+node --version  # Should be >= 16.x
+npm --version   # Should be >= 8.x
+```
+
+### Issue: API calls return 404
+
+**Problem**: Backend not running or wrong port.
+
+**Solution**:
+1. Start the backend: `flowyml ui start --port 8080`
+2. Check proxy config in `vite.config.ts`
+3. Verify backend is on `http://localhost:8080/api/health`
+
+### Issue: Hot reload not working
+
+**Solution**:
+1. Make sure you're running `npm run dev` (not `npm run build`)
+2. Check for errors in the terminal
+3. Try clearing Vite cache: `rm -rf node_modules/.vite`
+
+### Issue: Build succeeds but backend shows "Frontend not built"
+
+**Cause**: Backend looking in wrong directory.
+
+**Solution**: Ensure the `dist/` folder exists in `flowyml/ui/frontend/dist/`
+
+```bash
+# Check if dist exists
+ls -la dist/
+
+# If not, rebuild
+npm run build
+```
+
+## 🎯 Key Features to Implement
+
+The frontend should provide:
+
+1. **Dashboard**
+   - Overview of recent pipeline runs
+   - Active/running pipelines
+   - Quick stats (success rate, avg duration)
+
+2. **Pipeline List**
+   - All registered pipelines
+   - Search and filter
+   - Click to view details
+
+3. **Pipeline Details**
+   - DAG visualization
+   - Step-by-step execution status
+   - Logs per step
+   - Artifacts produced
+
+4. **Run History**
+   - List of all runs
+   - Filter by pipeline, status, date
+   - Compare runs side-by-side
+
+5. **Asset Explorer**
+   - Browse datasets, models, metrics
+   - View lineage graph
+   - Download artifacts
+
+6. **Experiment Tracking**
+   - Compare experiments
+   - Metrics visualization (charts)
+   - Parameter comparison
+
+7. **Real-Time Updates** (Future)
+   - WebSocket connection for live updates
+   - Live metric streaming
+   - Real-time log tailing
+
+## 📚 Resources
+
+- [Vite Documentation](https://vitejs.dev/)
+- [React Documentation](https://react.dev/)
+- [TypeScript Handbook](https://www.typescriptlang.org/docs/)
+- [flowyml UI Guide](../../../UI_GUIDE.md)
+
+## 🤝 Contributing
+
+When contributing to the frontend:
+
+1. Follow the existing code style
+2. Use TypeScript for type safety
+3. Write meaningful component names
+4. Add comments for complex logic
+5. Test API integration thoroughly
+6. Ensure production build works
+
+## 📝 Notes
+
+- The Python backend serves the built frontend from `dist/`
+- In production, there's no separate frontend server
+- All state is fetched from the backend API
+- WebSocket support is planned for future releases