beamflow-cli 0.3.4__tar.gz → 0.3.6__tar.gz

{beamflow_cli-0.3.4 → beamflow_cli-0.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: beamflow-cli
-Version: 0.3.4
+Version: 0.3.6
 Summary: CLI for the Beamflow Managed Platform
 Author: juraj.bezdek@gmail.com
 Author-email: juraj.bezdek@gmail.com

{beamflow_cli-0.3.4 → beamflow_cli-0.3.6}/beamflow/commands/deploy.py

@@ -1,6 +1,6 @@
 from typing import Optional
-import typer
 import asyncio
+import typer
 from pathlib import Path
 from rich.console import Console
 from rich.progress import Progress, SpinnerColumn, TextColumn
@@ -11,12 +11,26 @@ from .build import run_build_flow
 app = typer.Typer()
 console = Console()
 
+# Status labels shown to the user while polling
+_STATUS_LABELS = {
+    "QUEUED":         "Queued, waiting for deployment to start...",
+    "DEPLOYING":      "Deploying services to Cloud Run...",
+    "INIT_PENDING":   "Waiting for worker to register scheduled tasks...",
+    "INIT_RECEIVED":  "Tasks registered, running health checks...",
+    "HEALTH_CHECK":   "Health-checking API and Worker...",
+    "DEPLOYED":       "Deployed successfully ✓",
+    "FAILED":         "Deployment failed.",
+}
+
+_TERMINAL_STATUSES = {"DEPLOYED", "FAILED"}
+
+
 async def run_deploy_flow(artifact_id: Optional[str] = None):
     project_config = load_project_config()
     if not project_config:
         console.print("[red]No .beamflow found. Please run 'beamflow init' first.[/red]")
         raise typer.Exit(code=1)
-    
+
     project_id = project_config.project_id
     if not project_id:
         console.print("[red]project_id not found in .beamflow[/red]")
@@ -41,44 +55,39 @@ async def run_deploy_flow(artifact_id: Optional[str] = None):
         deploy_data = await api.post("/v1/deploy", json={
             "project_id": project_id,
             "artifact_id": artifact_id,
-            "env_vars": {} # TODO: load from .beamflow.yaml or env
+            "env_vars": {}
         })
         deploy_id = deploy_data["deploy_id"]
         progress.update(task, description=f"Deployment triggered (ID: {deploy_id}).")
-        
-        # 3. Poll status
-        task = progress.add_task(description="Deploying...", total=None)
+
+        # 3. Poll status until terminal
+        last_status = None
         while True:
             status_data = await api.get(f"/v1/deploy/{deploy_id}/status")
-            status = status_data["status"]
-            
-            if status == "SUCCESS":
-                progress.update(task, description="Deployment successful!")
-                return status_data
-            elif status in ["FAILURE", "INTERNAL_ERROR", "TIMEOUT", "CANCELLED"]:
-                progress.update(task, description=f"Deployment failed with status: {status}")
-                console.print(f"[red]Deployment failed: {status}[/red]")
+            current_status = status_data["status"]
+
+            if current_status != last_status:
+                label = _STATUS_LABELS.get(current_status, f"Status: {current_status}")
+                progress.update(task, description=label)
+                last_status = current_status
+
+            if current_status == "DEPLOYED":
+                break
+            elif current_status == "FAILED":
+                error = status_data.get("error_message", "Unknown error")
+                console.print(f"[red]Deployment failed: {error}[/red]")
                 raise typer.Exit(code=1)
-            
-            # Since deployment is mostly a placeholder in API for now, 
-            # we might want to just return if it's QUEUED or something
-            # but to be correct we should poll.
-            progress.update(task, description=f"Deploying... ({status})")
-            
-            # NOTE: For now, the API just returns QUEUED and doesn't update.
-            # I'll add a safety break if it stays QUEUED for too long or just return Success for demo if it's QUEUED.
-            # Actually, I'll just return it so the user sees it.
-            if status == "QUEUED":
-                # For demo purposes, we can assume it will eventually succeed or just stop here.
-                # But let's follow the polling pattern.
-                pass
-                
+
             await asyncio.sleep(5)
 
+    return status_data
+
+
 @app.command()
 def deploy(
     env: str = typer.Argument(..., help="Environment to deploy to"),
-    artifact: Optional[str] = typer.Option(None, "--artifact", "-a", help="Artifact ID to deploy")
+    artifact: Optional[str] = typer.Option(None, "--artifact", "-a", help="Artifact ID to deploy"),
+    non_interactive: bool = typer.Option(False, "--non-interactive", help="Do not ask for confirmation")
 ):
     """Deploy an artifact to the managed platform for a specified environment."""
     project_config = load_project_config()
@@ -86,19 +95,45 @@ def deploy(
         console.print("[red]No .beamflow found. Please run 'beamflow init' first.[/red]")
         raise typer.Exit(code=1)
 
+    # Ask for confirmation unless non-interactive is provided
+    if not non_interactive:
+        from rich.prompt import Confirm
+        if not Confirm.ask(f"Are you sure you want to deploy to '{env}'?"):
+            console.print("Deployment cancelled.")
+            raise typer.Exit(code=0)
+
     # Check if env is managed
     is_managed = False
     for em in project_config.environments:
         if em.name == env:
             is_managed = em.managed
             break
-    
+
     if not is_managed:
         console.print(f"[red]Environment '{env}' is not managed. Deployment is only supported for managed environments.[/red]")
         console.print("[yellow]Update your .beamflow environments if this is incorrect.[/yellow]")
         raise typer.Exit(code=1)
 
+    project_id = project_config.project_id
+    if not project_id:
+        if not non_interactive:
+            from rich.prompt import Confirm
+            if Confirm.ask("Project ID not found. Would you like to link to a managed project now?"):
+                from .project import _link_project
+                _link_project(project_config)
+                project_id = project_config.project_id
+
+        if not project_id:
+            console.print("[red]project_id not found in .beamflow. Use 'beamflow init' or link to a project.[/red]")
+            raise typer.Exit(code=1)
+
     result = asyncio.run(run_deploy_flow(artifact))
-    console.print(f"[green]Deployment finished![/green]")
-    console.print(f"Deployment ID: [bold]{result['deploy_id']}[/bold]")
-    console.print(f"Status: [bold]{result['status']}[/bold]")
+
+    api_url = result.get("api_url", "")
+    worker_url = result.get("worker_url", "")
+    console.print(f"[green]✓ Deployment complete![/green]")
+    console.print(f"  Deployment ID : [bold]{result['deploy_id']}[/bold]")
+    if api_url:
+        console.print(f"  API URL       : [bold]{api_url}[/bold]")
+    if worker_url:
+        console.print(f"  Worker URL    : [bold]{worker_url}[/bold]")

{beamflow_cli-0.3.4 → beamflow_cli-0.3.6}/beamflow/main.py

@@ -73,6 +73,21 @@ def check():
     """Inspect the current project structure."""
     project_cmds.check()
 
+@app.command()
+def link():
+    """
+    [bold cyan]Link[/bold cyan] your current project to a Beamflow Managed Project.
+    """
+    from .core.config import load_project_config
+    from .commands.project import _link_project
+    
+    project_config = load_project_config()
+    if not project_config:
+        console.print("[red]No .beamflow found. Please run 'beamflow init' first.[/red]")
+        raise typer.Exit(code=1)
+        
+    _link_project(project_config)
+
 @env_app.command("add")
 def env_add(
     env_name: Optional[str] = typer.Argument(None, help="Environment name to add"),
@@ -115,7 +130,7 @@ def env_init_vscode(
 def run(
     ctx: typer.Context,
     env: str = typer.Argument("local", help="Environment to run (default: local)"),
-    build: bool = typer.Option(False, "--build", help="Build images before starting"),
+    build: bool = typer.Option(True, "--build/--no-build", help="Build images before starting"),
     detach: bool = typer.Option(False, "--detach", "-d", help="Run in background"),
     logs: bool = typer.Option(False, "--logs", help="Follow logs (useful with --detach)")
 ):
@@ -187,7 +202,8 @@ def build(
 def deploy(
     ctx: typer.Context,
     env: str = typer.Argument("prod", help="Environment to deploy to (default: prod)"),
-    artifact: Optional[str] = typer.Option(None, "--artifact", "-a", help="Artifact ID to deploy")
+    artifact: Optional[str] = typer.Option(None, "--artifact", "-a", help="Artifact ID to deploy"),
+    non_interactive: bool = typer.Option(False, "--non-interactive", help="Do not ask for confirmation")
 ):
     """
     [bold cyan]Deploy[/bold cyan] your project to the Beamflow Managed Platform.
@@ -213,7 +229,7 @@ def deploy(
 
     if not env_mode:
         if env == "prod":
-            if Confirm.ask(f"Environment '{env}' not found. Would you like to set it up now?"):
+            if not non_interactive and Confirm.ask(f"Environment '{env}' not found. Would you like to set it up now?"):
                 # We need to pass the actual project_config object to add_environment
                 # But project_cmds is a module, we should be careful about cyclic imports or just use it.
                 project_cmds.add_environment(project_config, env_name=env)
@@ -232,7 +248,7 @@ def deploy(
             console.print(f"To deploy to a specific environment, use: [bold]beamflow deploy <env_name>[/bold]")
             raise typer.Exit(code=1)
 
-    deploy_cmds.deploy(env=env, artifact=artifact)
+    deploy_cmds.deploy(env=env, artifact=artifact, non_interactive=non_interactive)
 
 @app.command()
 def whoami():

beamflow_cli-0.3.6/beamflow/templates/AGENTS.md

@@ -0,0 +1,222 @@
+# Beamflow Agents Guide
+
+This guide explains how to build AI Agents and robust integrations using Beamflow. It covers the core concepts you need to know to bring data in, process it, and interact with external systems.
+
+## Core Concepts
+
+Beamflow integrations revolve around the following core ideas:
+- **Integration Context**: Automatically tracks the current integration, pipeline, run, and context across distributed operations.
+- **Ingress**: How data enters your pipelines (via Webhooks or Polled schedules).
+- **Clients**: How you make authenticated requests to external APIs.
+- **Records Feed**: How you queue and deduplicate records for processing.
+- **Project Structure**: How to organize code between API, Worker, and Shared modules.
+
+## Bringing Data In: Ingress
+
+You can trigger your pipelines through two main ingress decorators provided by `beamflow_lib.pipelines.ingress`:
+
+### 1. Webhooks (`@ingress.webhook`)
+Use webhooks when an external system can push events to your application. This decorator registers the handler as part of the integration but leaves the exact handling logic to you.
+
+```python
+from beamflow_lib.pipelines.ingress import ingress
+
+@router.post("/webhook/slack")
+@ingress.webhook(pipeline="slack.messages", integration="slack")
+async def handle_slack_webhook(request):
+    data = await request.json()
+    # Read the webhook and process the data!
+```
+
+### 2. Polling (`@ingress.poll`)
+Use polling when you need to fetch data on a schedule. This decorator acts as a specialized scheduler entrypoint that injects a durable `state` dictionary into your function. This is perfect for remembering "watermarks" (like the last processed token or timestamp) to paginate stateful APIs.
+
+The state is automatically saved for you as long as the function executes without exceptions.
+
+```python
+from beamflow_lib.pipelines.ingress import ingress
+from datetime import datetime
+
+@ingress.poll(pipeline="slack.messages", integration="slack", schedule="*/5 * * * *")
+async def poll_slack(state: dict):
+    # Retrieve the watermark from the previous run
+    since = state.get("since")
+    
+    # Fetch new data using the watermark...
+    page = await fetch_messages_from_api(since=since)
+    
+    # Process or publish data
+    
+    # Update the watermark; it will be automatically saved!
+    state["since"] = datetime.now()
+```
+
+## Making External Requests: Clients
+
+Beamflow makes it easy to interact with external APIs via Clients. They handle setting Base URLs, automatic Authentication headers, observability tracing, and defaults out of the box.
+
+### Adding a Client
+To configure a new REST client, you place a configuration file (like `slackClient.yaml`) in your shared clients folder. Beamflow dynamically loads these clients if your configuration registry points to the folder. For example, if you have `_config/shared/clients.yaml` configured as follows:
+```yaml
+path: clients
+pattern: "*.yaml"
+```
+You can simply define `clients/slackClient.yaml` and the system will expose it to your tasks.
+
+### Using a Client
+Once configured, you can retrieve the standard HTTP client (from the `beamflow_clients` package) anywhere.
+
+```python
+from beamflow_clients import get_client
+from beamflow_lib.decorators import integration_task
+
+# Load the client configuration
+slack_client = get_client("slackClient")
+
+@integration_task(integration="slack", integration_pipeline="send_message")
+async def send_slack_message_task(message: str):
+    # Authorization and base URL logic are automatically handled here
+    await slack_client.request("POST", "/chat.postMessage", json={"text": message})
+```
+
+### Custom Clients
+If you have an API you interact with heavily, you can define a custom typed client. This encapsulates specific API routes for a better developer experience.
+
+```python
+from beamflow_lib.clients import HttpClient, client
+from typing import List
+
+# Extending HttpClient and registering with the @client decorator
+@client("DemoClient")
+class DemoClient(HttpClient):
+    """Custom client for the Demo API."""
+    
+    async def get_users(self) -> List[dict]:
+        response = await self.request("GET", "/users")
+        return response.json()
+        
+    async def get_user(self, user_id: int) -> dict:
+        response = await self.request("GET", f"/users/{user_id}")
+        return response.json()
+
+# You can then resolve this custom client by name:
+# demo = get_client("DemoClient")
+```
+
+## Processing Data: Records Feed
+
+When you receive payloads via Webhooks or Polling, you usually want to process them robustly and asynchronously on the backend. The `RecordsFeed` module provides an opinionated record queue with built-in deduplication that prevents overwhelming pipelines.
+
+### Publishing Records
+The framework resolves deduplication using a unique combination of `(integration, record_type, record_id)`. If multiple records arrive with the same identity, "latest-wins" semantics are applied (keeping the one with the latest timestamp).
+
+It also seamlessly supports feeding massive data payloads -- large objects (> 5KB) are automatically offloaded to a BlobStore.
+
+```python
+from beamflow_lib.pipelines.records_feed import RecordsFeed
+from beamflow_lib.pipelines.records_model import RecordData
+
+# Retrieve your specific feed
+feed = RecordsFeed.get(feed_id="slack.messages")
+
+# A common pattern is inserting data retrieved via an `@ingress.poll` into a feed
+await feed.publish(RecordData(
+    record_id="msg_123",
+    record_type="message",
+    data={"text": "Hello World", "user": "U123"}
+))
+```
+
+### Consuming Records
+To process the data asynchronously, decorate a function with `@feed_consumer`. This supports robust configurations like automatic batched receiving, delays, rate-limiting, and concurrency control.
+
+```python
+from beamflow_lib import feed_consumer, RecordData
+
+@feed_consumer(
+    feed_id="slack.messages", 
+    batch=True,
+    max_batch_size=50,
+    max_delay_ms=2000 # wait up to 2 seconds for the queue to fill
+)
+async def process_batch_messages(records: list[RecordData]):
+    """
+    Consumes a maximum of 50 records in a batch, waiting 
+    up to 2 seconds for the queue to fill.
+    """
+    print(f"Began processing {len(records)} records.")
+    for record in records:
+        print(f"Record: {record.record_id} of type {record.record_type}")
+        print(f"Data Payload: {record.data}")
+```
+
+```python
+@feed_consumer(
+    feed_id="slack.messages", 
+    batch=False
+)
+async def process_batch_messages(record: RecordData):
+    """
+    Consumes a single record at a time.
+    """
+    print(f"Began processing {record.record_id} of type {record.record_type}")
+    print(f"Data Payload: {record.data}")
+```
+## Project Structure
+
+A typical Beamflow project is organized into three main areas: Configuration, Deployment, and Source Code.
+
+```text
+.
+├── config/                 # Application configuration
+│   ├── shared/             # Base configuration for all environments
+│   │   ├── backend.yaml    # Task backend settings (shared)
+│   │   ├── clients.yaml    # Points to the clients folder
+│   │   ├── clients/        # Shared client configurations
+│   │   │   └── fooClient.yaml
+│   │   └── .env            # Shared environment variables
+│   └── local/              # Environment-specific overrides (e.g., local, dev, prod)
+│       ├── backend.yaml    # Local-specific backend settings
+│       └── .env            # Local-specific environment variables
+├── deployment/             # deployment-related files
+│   ├── shared/             # Base Dockerfiles
+│   │   ├── api.Dockerfile
+│   │   └── worker.Dockerfile
+│   └── local/              # Local deployment configuration
+│       └── docker-compose.yaml
+├── src/                    # Source code for your integration
+│   ├── api/                # API-specific code (Routes, Webhooks)
+│   │   └── routes/
+│   │       └── webhooks.py # typical place for @ingress.webhook ... we can also add the webhooks into purpose specific files ie slack_webhooks.py / stripe_webhooks.py etc
+│   │       └── routes.py   # we can also add standard api routes ... like health check etc
+│   ├── worker/             # Worker-specific code (Tasks, Consumers)
+│   │   └── tasks/
+│   │       └── xyzTask.py  # place for feed consumers, polling tasks, integration tasks etc
+│   └── shared/             # Shared code (Models, Clients, Utils)
+│       ├── clients/        # here should to the custom clients
+│       └── models/         # here should to the custom models
+├── api_main.py             # API Entry point
+└── worker_main.py          # Worker Entry point
+```
+
+### Source Organization (`src/`)
+
+Beamflow separates code based on where it executes to ensure clean isolation and efficient scaling:
+
+1.  **API (`src/api/`)**: Code that runs on the webserver. Its primary role is to handle incoming requests, typically defined using the `@ingress.webhook` decorator.
+2.  **Worker (`src/worker/`)**: Code that runs on the background worker. This is where the "heavy lifting" happens, including `@integration_task`, `@ingress.poll`, and `@feed_consumer` handlers.
+3.  **Shared (`src/shared/`)**: Logic used by both the API and Worker, such as custom typed Clients, Pydantic models, and utility functions.
+
+### Configuration & Environments
+
+Beamflow uses a tiered configuration system that allows for seamless transitions between environments:
+
+- **Inheritance**: Configuration is loaded from `config/shared/` first, and then overridden by environment-specific files in `config/{env}/`.
+- **Backend Setup**: The `backend.yaml` file defines how tasks are executed (e.g., `asyncio` for local dev or `dramatiq` or `managed` for production).
+- **Environment Variables**: `.env` files in both the shared and environment folders are automatically resolved and injected into the application.
+
+### Execution Entry Points
+
+- **`api_main.py`**: The entry point for the web server. It sets up the FastAPI application and automatically imports modules from `src/api` to register routes.
+- **`worker_main.py`**: The entry point for the background process. It connects to the task backend (like Redis) and automatically imports modules from `src/worker` to register task signatures.
+

{beamflow_cli-0.3.4 → beamflow_cli-0.3.6}/beamflow/templates/_.dockerignore

@@ -4,6 +4,5 @@ __pycache__
 .pytest_cache
 *.pyc
 .beamflow
-pyproject.toml
 README.md
 tests

{beamflow_cli-0.3.4 → beamflow_cli-0.3.6}/beamflow/templates/_api_main.py

@@ -3,7 +3,7 @@ from fastapi import Request
 from beamflow_lib.config.env_loader import load_config_dir
 from beamflow_runtime.ingress.app import create_fastapi_app
 import os
-env = os.getenv("ENVIVROMENT", "dev")
+env = os.getenv("ENVIRONMENT", "dev")
 # Load configuration relative to this file
 config = load_config_dir("config", environment=env)
 

{beamflow_cli-0.3.4 → beamflow_cli-0.3.6}/beamflow/templates/_config/[env]/(backend-dramatiq)/backend.yaml

@@ -1,7 +1,7 @@
 backend:
   type: dramatiq
-  dramatiq:
-    redis_url: redis://redis:6380/0
+
+
 webhooks:
   prefix: /webhooks
 

{beamflow_cli-0.3.4 → beamflow_cli-0.3.6}/beamflow/templates/_config/[env]/_backend.yaml

@@ -1,4 +1,4 @@
 # Environment-specific overrides for backend
 backend:
-  # The type of backend can be 'dramatiq', 'async_backend', or 'managed'
+  # The type of backend can be 'dramatiq', 'asyncio', or 'managed'
   type: managed | dramatiq | asyncio

beamflow_cli-0.3.6/beamflow/templates/_config/shared/.env

@@ -0,0 +1 @@
+BEAMFLOW_API_URL="http://localhost:8080"

{beamflow_cli-0.3.4/beamflow/templates/_config → beamflow_cli-0.3.6/beamflow/templates/_config/shared}/clients/demoClient.yaml

@@ -13,7 +13,7 @@ auth:
   extra_params: {}
 
 # Authentication Alternatives (Commented out)
-# ------------------------------------------
+# ------------------------------------------s
 # Basic Auth:
 # auth:
 #   type: basic

beamflow_cli-0.3.6/beamflow/templates/_config/shared/clients.yaml

@@ -0,0 +1,3 @@
+path: clients
+pattern: "*.yaml"
+recursive: false

beamflow_cli-0.3.6/beamflow/templates/_deployment/shared/api.Dockerfile

@@ -0,0 +1,18 @@
+# API Dockerfile for {project_name}
+FROM beamflow/beamflow-base:latest
+
+WORKDIR /app
+COPY . .
+
+# Build arg that chooses which extra(s) to install
+# e.g. SERVICE_EXTRAS="api" or "worker" or "api,worker"
+ARG SERVICE_EXTRAS="api"
+
+# Install base deps + extras into system site-packages
+# Base deps:
+RUN uv pip install --system -r pyproject.toml
+RUN uv pip install --system -r pyproject.toml --extra api || echo "no worker extra; skipping"
+RUN rm -rf src/worker
+EXPOSE 8000
+ENV PYTHONPATH=/app/src:/app
+CMD ["python", "api_main.py"]

beamflow_cli-0.3.6/beamflow/templates/_deployment/shared/worker.Dockerfile

@@ -0,0 +1,18 @@
+# Worker Dockerfile for {project_name}
+FROM beamflow/beamflow-base:latest
+
+WORKDIR /app
+COPY . .
+
+# Build arg that chooses which extra(s) to install
+# e.g. SERVICE_EXTRAS="api" or "worker" or "api,worker"
+ARG SERVICE_EXTRAS="worker"
+
+# Install base deps + extras into system site-packages
+# Base deps:
+RUN uv pip install --system -r pyproject.toml
+RUN uv pip install --system -r pyproject.toml --extra worker || echo "no worker extra; skipping"
+
+RUN rm -rf src/api
+ENV PYTHONPATH=/app/src:/app
+CMD ["python", "-u", "worker_main.py"]

{beamflow_cli-0.3.4 → beamflow_cli-0.3.6}/beamflow/templates/_worker_main.py

@@ -8,8 +8,8 @@ from beamflow_runtime import initialize_runtime, run_worker
 logging.basicConfig(level=logging.INFO)
 
 # Load config
-environment = os.getenv("ENVIRONMENT", "dev")
-config = load_config_dir("config", environment=environment)
+env = os.getenv("ENVIRONMENT", "dev")
+config = load_config_dir("config", environment=env)
 
 # Initialize runtime with auto-import of the tasks directory
 runtime = initialize_runtime(

{beamflow_cli-0.3.4 → beamflow_cli-0.3.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "beamflow-cli"
-version = "0.3.4"
+version = "0.3.6"
 description = "CLI for the Beamflow Managed Platform"
 authors = [{name = "juraj.bezdek@gmail.com", email = "juraj.bezdek@gmail.com"}]
 requires-python = ">=3.11"

beamflow_cli-0.3.4/beamflow/templates/_deployment/shared/api.Dockerfile

@@ -1,10 +0,0 @@
-# API Dockerfile for {project_name}
-FROM beamflow/beamflow-base:latest
-
-WORKDIR /app
-COPY . .
-
-RUN rm -rf src/worker
-EXPOSE 8000
-ENV PYTHONPATH=/app/src:/app
-CMD ["python", "api_main.py"]

beamflow_cli-0.3.4/beamflow/templates/_deployment/shared/worker.Dockerfile

@@ -1,9 +0,0 @@
-# Worker Dockerfile for {project_name}
-FROM beamflow/beamflow-base:latest
-
-WORKDIR /app
-COPY . .
-
-RUN rm -rf src/api
-ENV PYTHONPATH=/app/src:/app
-CMD ["python", "-u", "worker_main.py"]