evalforge-runtime 0.1.0__tar.gz

evalforge_runtime-0.1.0/.github/workflows/workflow.yml

@@ -0,0 +1,46 @@
+name: Publish to PyPI
+
+on:
+  push:
+    branches: [main]
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build dependencies
+        run: pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Upload distribution artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    steps:
+      - name: Download distribution artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: python-package-distributions
+          path: dist/
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

evalforge_runtime-0.1.0/.gitignore

@@ -0,0 +1,41 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.egg-info/
+*.egg
+dist/
+build/
+*.whl
+
+# Virtual environments
+.venv/
+venv/
+ENV/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+
+# Runtime data
+data/
+*.db
+*.sqlite
+
+# Environment
+.env
+.env.local
+
+# OS
+.DS_Store
+Thumbs.db
+
+# UV
+.python-version

evalforge_runtime-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Jannis Conen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

evalforge_runtime-0.1.0/PKG-INFO

@@ -0,0 +1,259 @@
+Metadata-Version: 2.4
+Name: evalforge-runtime
+Version: 0.1.0
+Summary: Runtime engine for EvalForge generated applications
+Project-URL: Homepage, https://github.com/JannisConen/evalforge-runtime
+Project-URL: Repository, https://github.com/JannisConen/evalforge-runtime
+Project-URL: Issues, https://github.com/JannisConen/evalforge-runtime/issues
+Author: Jannis Conen
+License-Expression: MIT
+License-File: LICENSE
+Keywords: ai,evalforge,evaluation,fastapi,llm,runtime
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: FastAPI
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Libraries
+Requires-Python: >=3.10
+Requires-Dist: aiosqlite>=0.20.0
+Requires-Dist: fastapi>=0.115.0
+Requires-Dist: httpx>=0.28.0
+Requires-Dist: litellm>=1.60.0
+Requires-Dist: pydantic-settings>=2.7.0
+Requires-Dist: pydantic>=2.10.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: python-multipart>=0.0.18
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: sqlalchemy>=2.0.0
+Requires-Dist: uvicorn[standard]>=0.34.0
+Provides-Extra: all
+Requires-Dist: apscheduler<4.0.0,>=3.10.0; extra == 'all'
+Requires-Dist: azure-identity>=1.15.0; extra == 'all'
+Requires-Dist: azure-keyvault-secrets>=4.8.0; extra == 'all'
+Requires-Dist: boto3>=1.34.0; extra == 'all'
+Requires-Dist: google-auth>=2.0.0; extra == 'all'
+Requires-Dist: langfuse>=2.0.0; extra == 'all'
+Provides-Extra: aws
+Requires-Dist: boto3>=1.34.0; extra == 'aws'
+Provides-Extra: azure
+Requires-Dist: azure-identity>=1.15.0; extra == 'azure'
+Requires-Dist: azure-keyvault-secrets>=4.8.0; extra == 'azure'
+Provides-Extra: dev
+Requires-Dist: pytest-asyncio>=0.24.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.9.0; extra == 'dev'
+Provides-Extra: exchange
+Requires-Dist: httpx>=0.28.0; extra == 'exchange'
+Provides-Extra: gmail
+Requires-Dist: google-auth>=2.0.0; extra == 'gmail'
+Provides-Extra: langfuse
+Requires-Dist: langfuse>=2.0.0; extra == 'langfuse'
+Provides-Extra: scheduling
+Requires-Dist: apscheduler<4.0.0,>=3.10.0; extra == 'scheduling'
+Description-Content-Type: text/markdown
+
+# evalforge-runtime
+
+Runtime engine for [EvalForge](https://github.com/JannisConen/evalforge-runtime) generated applications. Provides a FastAPI server that executes LLM-powered processes with cost tracking, execution logging, secret management, and file handling.
+
+> Completely vibe-coded in 1 hour.
+
+## Installation
+
+```bash
+pip install evalforge-runtime
+```
+
+With optional extras:
+
+```bash
+# Langfuse observability
+pip install evalforge-runtime[langfuse]
+
+# Azure Key Vault secrets
+pip install evalforge-runtime[azure]
+
+# AWS Secrets Manager
+pip install evalforge-runtime[aws]
+
+# Everything
+pip install evalforge-runtime[all]
+```
+
+## Quick start
+
+### 1. Create a config file
+
+Create `evalforge.config.yaml` in your project root:
+
+```yaml
+project:
+  id: my-project
+
+llm:
+  model: anthropic/claude-sonnet-4-6
+
+processes:
+  classify:
+    process_id: classify
+    trigger:
+      type: webhook
+    instructions: |
+      Classify the incoming support ticket into one of: billing, technical, general.
+```
+
+### 2. Set your API keys
+
+```bash
+export EVALFORGE_API_KEYS=my-secret-key
+export ANTHROPIC_API_KEY=sk-ant-...
+```
+
+`EVALFORGE_API_KEYS` is a comma-separated list of keys that authenticate requests to the runtime.
+
+### 3. Start the server
+
+```bash
+evalforge-runtime start --config evalforge.config.yaml
+```
+
+The server starts on `http://localhost:8000` by default.
+
+### 4. Call a process
+
+```bash
+curl -X POST http://localhost:8000/process/classify \
+  -H "Authorization: Bearer my-secret-key" \
+  -H "Content-Type: application/json" \
+  -d '{"ticket_text": "I cannot log in to my account"}'
+```
+
+## API endpoints
+
+| Method | Path | Description |
+|--------|------|-------------|
+| `GET` | `/health` | Health check with version and uptime |
+| `POST` | `/process/<name>` | Execute a process (authenticated) |
+| `GET` | `/executions` | Query execution history |
+| `GET` | `/executions/:id` | Single execution detail |
+| `GET` | `/executions/stats` | Aggregated cost, latency, and volume stats |
+
+## Configuration
+
+The runtime is configured via `evalforge.config.yaml`. Environment variables can be referenced with `${VAR_NAME}` syntax.
+
+### Top-level sections
+
+```yaml
+project:
+  id: my-project              # Required: project identifier
+  evalforge_url: https://...   # Optional: EvalForge server URL
+  version: 1.0.0
+
+llm:
+  model: anthropic/claude-sonnet-4-6  # Default LLM model (LiteLLM format)
+
+database:
+  url: sqlite+aiosqlite:///./data/app.db  # Execution log database
+
+storage:
+  type: local          # local or s3
+  path: ./data/files   # Local file storage path
+
+auth:
+  methods:
+    - type: api_key    # api_key or oauth2
+
+secrets:
+  provider: env        # env, evalforge, azure_keyvault, aws_secrets_manager, sap_credential_store
+
+observability:
+  langfuse:
+    enabled: false
+    host: https://cloud.langfuse.com
+```
+
+### Process configuration
+
+Each process defines an LLM-powered task:
+
+```yaml
+processes:
+  my-process:
+    process_id: my-process
+    trigger:
+      type: webhook           # webhook, schedule, or process
+      cron: "0 9 * * *"       # For schedule triggers
+      after: other-process    # For process triggers (chaining)
+    instructions: |
+      Your LLM instructions here...
+    llm_model: openai/gpt-4o  # Override default model
+    connector: exchange        # Optional: built-in connector
+    review:
+      enabled: true
+      timeout: 24h
+```
+
+### Custom process modules
+
+For advanced logic, provide Python modules for the three-phase pipeline:
+
+```yaml
+processes:
+  my-process:
+    process_id: my-process
+    trigger:
+      type: webhook
+    before_module: processes.my_process.before     # Pre-processing
+    execution_module: processes.my_process.execution  # Custom LLM call
+    after_module: processes.my_process.after        # Post-processing
+```
+
+Each module should define an async function matching the phase signature. See the [EvalForge docs](https://github.com/JannisConen/evalforge-runtime) for details.
+
+## Secret providers
+
+The runtime supports multiple secret backends:
+
+| Provider | Config value | Description |
+|----------|-------------|-------------|
+| Environment | `env` | Read from environment variables (default) |
+| EvalForge | `evalforge` | Fetch from EvalForge server |
+| Azure Key Vault | `azure_keyvault` | Azure Key Vault (`pip install evalforge-runtime[azure]`) |
+| AWS Secrets Manager | `aws_secrets_manager` | AWS Secrets Manager (`pip install evalforge-runtime[aws]`) |
+| SAP Credential Store | `sap_credential_store` | SAP BTP Credential Store |
+
+## CLI reference
+
+```
+evalforge-runtime start [OPTIONS]
+
+Options:
+  --config PATH   Path to evalforge.config.yaml (default: evalforge.config.yaml)
+  --host TEXT      Bind host (default: 0.0.0.0)
+  --port INT       Bind port (default: 8000)
+```
+
+## Development
+
+```bash
+# Clone and install
+git clone https://github.com/JannisConen/evalforge-runtime.git
+cd evalforge-runtime
+pip install -e ".[dev]"
+
+# Run tests
+pytest -v
+
+# Lint
+ruff check .
+```
+
+## License
+
+MIT

evalforge_runtime-0.1.0/README.md

@@ -0,0 +1,200 @@
+# evalforge-runtime
+
+Runtime engine for [EvalForge](https://github.com/JannisConen/evalforge-runtime) generated applications. Provides a FastAPI server that executes LLM-powered processes with cost tracking, execution logging, secret management, and file handling.
+
+> Completely vibe-coded in 1 hour.
+
+## Installation
+
+```bash
+pip install evalforge-runtime
+```
+
+With optional extras:
+
+```bash
+# Langfuse observability
+pip install evalforge-runtime[langfuse]
+
+# Azure Key Vault secrets
+pip install evalforge-runtime[azure]
+
+# AWS Secrets Manager
+pip install evalforge-runtime[aws]
+
+# Everything
+pip install evalforge-runtime[all]
+```
+
+## Quick start
+
+### 1. Create a config file
+
+Create `evalforge.config.yaml` in your project root:
+
+```yaml
+project:
+  id: my-project
+
+llm:
+  model: anthropic/claude-sonnet-4-6
+
+processes:
+  classify:
+    process_id: classify
+    trigger:
+      type: webhook
+    instructions: |
+      Classify the incoming support ticket into one of: billing, technical, general.
+```
+
+### 2. Set your API keys
+
+```bash
+export EVALFORGE_API_KEYS=my-secret-key
+export ANTHROPIC_API_KEY=sk-ant-...
+```
+
+`EVALFORGE_API_KEYS` is a comma-separated list of keys that authenticate requests to the runtime.
+
+### 3. Start the server
+
+```bash
+evalforge-runtime start --config evalforge.config.yaml
+```
+
+The server starts on `http://localhost:8000` by default.
+
+### 4. Call a process
+
+```bash
+curl -X POST http://localhost:8000/process/classify \
+  -H "Authorization: Bearer my-secret-key" \
+  -H "Content-Type: application/json" \
+  -d '{"ticket_text": "I cannot log in to my account"}'
+```
+
+## API endpoints
+
+| Method | Path | Description |
+|--------|------|-------------|
+| `GET` | `/health` | Health check with version and uptime |
+| `POST` | `/process/<name>` | Execute a process (authenticated) |
+| `GET` | `/executions` | Query execution history |
+| `GET` | `/executions/:id` | Single execution detail |
+| `GET` | `/executions/stats` | Aggregated cost, latency, and volume stats |
+
+## Configuration
+
+The runtime is configured via `evalforge.config.yaml`. Environment variables can be referenced with `${VAR_NAME}` syntax.
+
+### Top-level sections
+
+```yaml
+project:
+  id: my-project              # Required: project identifier
+  evalforge_url: https://...   # Optional: EvalForge server URL
+  version: 1.0.0
+
+llm:
+  model: anthropic/claude-sonnet-4-6  # Default LLM model (LiteLLM format)
+
+database:
+  url: sqlite+aiosqlite:///./data/app.db  # Execution log database
+
+storage:
+  type: local          # local or s3
+  path: ./data/files   # Local file storage path
+
+auth:
+  methods:
+    - type: api_key    # api_key or oauth2
+
+secrets:
+  provider: env        # env, evalforge, azure_keyvault, aws_secrets_manager, sap_credential_store
+
+observability:
+  langfuse:
+    enabled: false
+    host: https://cloud.langfuse.com
+```
+
+### Process configuration
+
+Each process defines an LLM-powered task:
+
+```yaml
+processes:
+  my-process:
+    process_id: my-process
+    trigger:
+      type: webhook           # webhook, schedule, or process
+      cron: "0 9 * * *"       # For schedule triggers
+      after: other-process    # For process triggers (chaining)
+    instructions: |
+      Your LLM instructions here...
+    llm_model: openai/gpt-4o  # Override default model
+    connector: exchange        # Optional: built-in connector
+    review:
+      enabled: true
+      timeout: 24h
+```
+
+### Custom process modules
+
+For advanced logic, provide Python modules for the three-phase pipeline:
+
+```yaml
+processes:
+  my-process:
+    process_id: my-process
+    trigger:
+      type: webhook
+    before_module: processes.my_process.before     # Pre-processing
+    execution_module: processes.my_process.execution  # Custom LLM call
+    after_module: processes.my_process.after        # Post-processing
+```
+
+Each module should define an async function matching the phase signature. See the [EvalForge docs](https://github.com/JannisConen/evalforge-runtime) for details.
+
+## Secret providers
+
+The runtime supports multiple secret backends:
+
+| Provider | Config value | Description |
+|----------|-------------|-------------|
+| Environment | `env` | Read from environment variables (default) |
+| EvalForge | `evalforge` | Fetch from EvalForge server |
+| Azure Key Vault | `azure_keyvault` | Azure Key Vault (`pip install evalforge-runtime[azure]`) |
+| AWS Secrets Manager | `aws_secrets_manager` | AWS Secrets Manager (`pip install evalforge-runtime[aws]`) |
+| SAP Credential Store | `sap_credential_store` | SAP BTP Credential Store |
+
+## CLI reference
+
+```
+evalforge-runtime start [OPTIONS]
+
+Options:
+  --config PATH   Path to evalforge.config.yaml (default: evalforge.config.yaml)
+  --host TEXT      Bind host (default: 0.0.0.0)
+  --port INT       Bind port (default: 8000)
+```
+
+## Development
+
+```bash
+# Clone and install
+git clone https://github.com/JannisConen/evalforge-runtime.git
+cd evalforge-runtime
+pip install -e ".[dev]"
+
+# Run tests
+pytest -v
+
+# Lint
+ruff check .
+```
+
+## License
+
+MIT

evalforge_runtime-0.1.0/fixtures/evalforge.config.yaml

@@ -0,0 +1,34 @@
+project:
+  id: "test_proj_001"
+  version: "0.1.0"
+
+secrets:
+  provider: env
+
+auth:
+  methods:
+    - type: api_key
+      header: "X-API-Key"
+
+llm:
+  model: "anthropic/claude-haiku-4-5"
+
+database:
+  url: "sqlite+aiosqlite:///./data/app.db"
+
+storage:
+  type: local
+  path: "./data/files"
+
+processes:
+  email-categorizer:
+    process_id: "proc_001"
+    instructions: |
+      You are an email categorizer. Given an email subject and body,
+      categorize it and assign a priority score.
+
+      Respond with JSON: {"category": "...", "priority": N}
+      Categories: inquiry, complaint, spam, invoice, other
+      Priority: 1-10 (10 = most urgent)
+    trigger:
+      type: webhook

evalforge_runtime-0.1.0/pyproject.toml

@@ -0,0 +1,79 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "evalforge-runtime"
+version = "0.1.0"
+description = "Runtime engine for EvalForge generated applications"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+authors = [{ name = "Jannis Conen" }]
+keywords = ["evalforge", "llm", "ai", "evaluation", "runtime", "fastapi"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Framework :: FastAPI",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Libraries",
+]
+
+dependencies = [
+    "fastapi>=0.115.0",
+    "uvicorn[standard]>=0.34.0",
+    "litellm>=1.60.0",
+    "pydantic>=2.10.0",
+    "pydantic-settings>=2.7.0",
+    "sqlalchemy>=2.0.0",
+    "aiosqlite>=0.20.0",
+    "httpx>=0.28.0",
+    "pyyaml>=6.0",
+    "python-multipart>=0.0.18",
+    "python-dotenv>=1.0.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/JannisConen/evalforge-runtime"
+Repository = "https://github.com/JannisConen/evalforge-runtime"
+Issues = "https://github.com/JannisConen/evalforge-runtime/issues"
+
+[project.optional-dependencies]
+exchange = ["httpx>=0.28.0"]
+gmail = ["google-auth>=2.0.0"]
+scheduling = ["apscheduler>=3.10.0,<4.0.0"]
+azure = ["azure-identity>=1.15.0", "azure-keyvault-secrets>=4.8.0"]
+aws = ["boto3>=1.34.0"]
+langfuse = ["langfuse>=2.0.0"]
+all = [
+    "google-auth>=2.0.0",
+    "apscheduler>=3.10.0,<4.0.0",
+    "azure-identity>=1.15.0",
+    "azure-keyvault-secrets>=4.8.0",
+    "boto3>=1.34.0",
+    "langfuse>=2.0.0",
+]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.24.0",
+    "ruff>=0.9.0",
+]
+
+[project.scripts]
+evalforge-runtime = "evalforge_runtime:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/evalforge_runtime"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]
+
+[tool.ruff]
+target-version = "py310"
+line-length = 100