feedloop 0.1.1__tar.gz → 0.1.2__tar.gz

feedloop-0.1.2/HOWTO.md

@@ -0,0 +1,302 @@
+# feedloop — How-To Guide
+
+> The fastest way to collect human preference data for LLMs.
+
+---
+
+## What is feedloop?
+
+feedloop is a lightweight developer tool that lets you collect **human preference feedback** on LLM outputs directly from your Python code. You submit pairs of model responses, a human reviews them in a browser UI, and feedloop exports the results in **DPO-ready JSONL format** — ready to use for fine-tuning or evaluation.
+
+---
+
+## Installation
+
+```bash
+pip install feedloop
+```
+
+**Requirements:** Python 3.10+
+
+---
+
+## Quickstart (5 minutes)
+
+### 1. Import and start the server
+
+```python
+import feedloop
+
+feedloop.start()
+# feedloop: server running at http://localhost:7777
+# (browser opens automatically)
+```
+
+This launches a local web server in the background and opens the review UI in your browser. Your script keeps running normally.
+
+### 2. Submit a comparison
+
+```python
+comparison_id = feedloop.compare(
+    prompt="Explain quantum entanglement in simple terms.",
+    outputs=[
+        "Quantum entanglement is when two particles become linked...",  # Output A
+        "Imagine two magic coins that always land on opposite sides...", # Output B
+    ]
+)
+```
+
+The comparison immediately appears in the browser UI for a human to review.
+
+### 3. (Optional) Wait for feedback
+
+```python
+# Block until this specific comparison is rated
+result = feedloop.wait(comparison_id)
+
+print(result["chosen"])    # the preferred output
+print(result["rejected"])  # the other output
+```
+
+Or wait for **all** pending comparisons to be rated:
+
+```python
+feedloop.wait()  # blocks until inbox is empty
+```
+
+### 4. Export your data
+
+```python
+count = feedloop.export("my_preferences.jsonl")
+# feedloop: exported 42 preferences to my_preferences.jsonl
+```
+
+---
+
+## The Review UI
+
+When you call `feedloop.start()`, a browser tab opens at `http://localhost:7777`.
+
+- **Left / Right buttons** — click to choose the preferred output
+- **Skip** — skip a comparison if neither output is clearly better
+- **Progress bar** — shows how many comparisons remain in the session
+
+The UI polls automatically — new comparisons appear in real time as your script submits them.
+
+---
+
+## Exported Data Format
+
+feedloop exports in **DPO (Direct Preference Optimization)** format — one JSON object per line:
+
+```jsonl
+{"prompt": "Explain quantum entanglement...", "chosen": "Imagine two magic coins...", "rejected": "Quantum entanglement is when two particles..."}
+{"prompt": "Write a haiku about autumn.", "chosen": "Leaves fall silently...", "rejected": "Autumn brings cool air..."}
+```
+
+This format is directly compatible with:
+- [TRL](https://github.com/huggingface/trl) `DPOTrainer`
+- [OpenRLHF](https://github.com/OpenRLHF/OpenRLHF)
+- Any custom fine-tuning pipeline that accepts preference pairs
+
+---
+
+## Full API Reference
+
+### `feedloop.start(port, db_path, open_browser)`
+
+Launches the feedback server. Safe to call multiple times — idempotent.
+
+| Parameter | Type | Default | Description |
+|---|---|---|---|
+| `port` | `int` | `7777` | Port to run the server on |
+| `db_path` | `str \| None` | `None` | Path to SQLite DB file. Uses a temp file if not set |
+| `open_browser` | `bool` | `True` | Auto-open browser on start |
+
+```python
+feedloop.start(port=8080, db_path="./feedback.db", open_browser=False)
+```
+
+---
+
+### `feedloop.compare(prompt, outputs, metadata)`
+
+Submits a pair of outputs for human review. Returns a `comparison_id` string.
+
+| Parameter | Type | Required | Description |
+|---|---|---|---|
+| `prompt` | `str` | ✅ | The input prompt shown to the reviewer |
+| `outputs` | `list[str]` | ✅ | Exactly 2 model outputs to compare |
+| `metadata` | `dict \| None` | ❌ | Optional key/value data attached to the record |
+
+```python
+cid = feedloop.compare(
+    prompt="Summarise this article in one sentence.",
+    outputs=[response_from_model_a, response_from_model_b],
+    metadata={"model_a": "gpt-4o", "model_b": "claude-3-5-sonnet"},
+)
+```
+
+---
+
+### `feedloop.wait(comparison_id, timeout)`
+
+Blocks until a comparison is rated (or all pending comparisons are done).
+
+| Parameter | Type | Default | Description |
+|---|---|---|---|
+| `comparison_id` | `str \| None` | `None` | Wait for a specific comparison. If `None`, waits for all pending |
+| `timeout` | `float \| None` | `None` | Max seconds to wait. Returns `None` on timeout |
+
+**Return value (single comparison):**
+```python
+{"prompt": "...", "chosen": "...", "rejected": "..."}
+```
+
+**Return value (wait for all):**
+```python
+{"completed": 42, "total": 42}
+```
+
+---
+
+### `feedloop.status()`
+
+Returns a live count of comparisons in the current session.
+
+```python
+feedloop.status()
+# {"pending": 3, "completed": 10, "skipped": 1, "total": 14}
+```
+
+---
+
+### `feedloop.export(path, format)`
+
+Exports all completed comparisons to a JSONL file.
+
+| Parameter | Type | Default | Description |
+|---|---|---|---|
+| `path` | `str` | `"preferences.jsonl"` | Output file path |
+| `format` | `str` | `"dpo"` | Export format. Currently `"dpo"` only |
+
+```python
+feedloop.export("data/run_1_preferences.jsonl")
+```
+
+---
+
+### `feedloop.stop()`
+
+Shuts down the server and closes the database. Called automatically on script exit.
+
+```python
+feedloop.stop()
+```
+
+---
+
+## Running the Server Standalone (CLI)
+
+You can run feedloop as a standalone server without embedding it in a script:
+
+```bash
+feedloop
+# feedloop: server running at http://localhost:7777
+# feedloop: Press Ctrl+C to stop
+```
+
+**CLI options:**
+
+```bash
+feedloop --port 8080              # use a different port
+feedloop --db ./feedback.db       # persist data to a specific file
+feedloop --no-browser             # don't auto-open the browser
+```
+
+This is useful when you want to keep the UI open across multiple script runs, or when running feedloop on a remote machine.
+
+---
+
+## Complete Example: Comparing Two Models
+
+```python
+import feedloop
+from openai import OpenAI
+
+client = OpenAI()
+
+prompts = [
+    "What is the capital of France?",
+    "Explain recursion to a 10-year-old.",
+    "Write a one-line poem about the ocean.",
+]
+
+feedloop.start(db_path="session.db")
+
+for prompt in prompts:
+    response_a = client.chat.completions.create(
+        model="gpt-4o-mini",
+        messages=[{"role": "user", "content": prompt}],
+    ).choices[0].message.content
+
+    response_b = client.chat.completions.create(
+        model="gpt-4o",
+        messages=[{"role": "user", "content": prompt}],
+    ).choices[0].message.content
+
+    feedloop.compare(
+        prompt=prompt,
+        outputs=[response_a, response_b],
+        metadata={"model_a": "gpt-4o-mini", "model_b": "gpt-4o"},
+    )
+
+print(f"Submitted {len(prompts)} comparisons — review them in the browser.")
+
+# Wait for all feedback, then export
+feedloop.wait()
+feedloop.export("gpt4o_vs_mini.jsonl")
+```
+
+---
+
+## Persisting Data Across Sessions
+
+By default, feedloop stores data in a temporary SQLite file that is deleted when the server stops. To keep your data:
+
+```python
+feedloop.start(db_path="./my_project_feedback.db")
+```
+
+The `.db` file will persist across runs. You can export from it at any time — even from a different script:
+
+```python
+feedloop.start(db_path="./my_project_feedback.db", open_browser=False)
+feedloop.export("all_preferences.jsonl")
+feedloop.stop()
+```
+
+---
+
+## FAQ
+
+**Q: Can I submit comparisons without waiting for them to be reviewed?**
+Yes. `feedloop.compare()` is non-blocking. You can submit all your comparisons up front and review them in the UI at your own pace.
+
+**Q: What happens if I skip a comparison?**
+Skipped comparisons are excluded from the export. They count toward `skipped` in `feedloop.status()`.
+
+**Q: Can I use feedloop without opening a browser?**
+Yes. Pass `open_browser=False` to `feedloop.start()`. The UI is still available at `http://localhost:7777` whenever you want it.
+
+**Q: Does feedloop work in Jupyter notebooks?**
+Yes. Call `feedloop.start()` in one cell and `feedloop.compare()` in subsequent cells.
+
+**Q: Is my data sent anywhere?**
+No. feedloop runs entirely on your local machine. No data leaves your environment.
+
+---
+
+## Links
+
+- **PyPI:** https://pypi.org/project/feedloop/

{feedloop-0.1.1 → feedloop-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: feedloop
-Version: 0.1.1
+Version: 0.1.2
 Summary: The fastest way to collect human preference data for LLMs
 Project-URL: Homepage, https://github.com/rammuthiah/feedloop
 Project-URL: Repository, https://github.com/rammuthiah/feedloop

{feedloop-0.1.1 → feedloop-0.1.2}/frontend/src/App.tsx

@@ -1,13 +1,19 @@
 import { useState, useEffect } from "react";
-import { fetchStats } from "./api";
+import { fetchSession, fetchStats } from "./api";
 import { useComparison } from "./hooks/useComparison";
 import ComparisonView from "./components/ComparisonView";
 import EmptyState from "./components/EmptyState";
 import ExportPanel from "./components/ExportPanel";
 
 export default function App() {
-  const { comparison, loading, refresh } = useComparison();
-  const doneState = useCheckDone(comparison, loading);
+  const [sessionId, setSessionId] = useState<string | null>(null);
+
+  useEffect(() => {
+    fetchSession().then(setSessionId).catch(() => {});
+  }, []);
+
+  const { comparison, loading, refresh } = useComparison(sessionId);
+  const doneState = useCheckDone(comparison, loading, sessionId);
 
   return (
     <div className="app">
@@ -22,7 +28,7 @@ export default function App() {
         ) : comparison ? (
           <ComparisonView comparison={comparison} onComplete={refresh} />
         ) : doneState === "done" ? (
-          <ExportPanel />
+          <ExportPanel sessionId={sessionId} />
         ) : (
           <EmptyState />
         )}
@@ -33,14 +39,15 @@ export default function App() {
 
 function useCheckDone(
   comparison: ReturnType<typeof useComparison>["comparison"],
-  loading: boolean
+  loading: boolean,
+  sessionId: string | null
 ): "done" | "empty" | "loading" {
   const [state, setState] = useState<"done" | "empty" | "loading">("loading");
 
   useEffect(() => {
     if (loading || comparison) return;
 
-    fetchStats()
+    fetchStats(sessionId)
       .then((stats) => {
         if (stats.total > 0 && stats.pending === 0) {
           setState("done");
@@ -49,7 +56,7 @@ function useCheckDone(
         }
       })
       .catch(() => setState("empty"));
-  }, [comparison, loading]);
+  }, [comparison, loading, sessionId]);
 
   return state;
 }

{feedloop-0.1.1 → feedloop-0.1.2}/frontend/src/api.ts

@@ -16,8 +16,18 @@ export interface Stats {
 
 const BASE = "/api";
 
-export async function fetchPending(): Promise<Comparison | null> {
-  const res = await fetch(`${BASE}/pending`);
+export async function fetchSession(): Promise<string | null> {
+  const res = await fetch(`${BASE}/session`);
+  if (!res.ok) return null;
+  const data = await res.json();
+  return data.session_id ?? null;
+}
+
+export async function fetchPending(sessionId: string | null): Promise<Comparison | null> {
+  const url = sessionId
+    ? `${BASE}/pending?session_id=${sessionId}`
+    : `${BASE}/pending`;
+  const res = await fetch(url);
   if (!res.ok) throw new Error(`Failed to fetch pending: ${res.status}`);
   const data = await res.json();
   return data;
@@ -38,12 +48,15 @@ export async function submitFeedback(
   if (!res.ok) throw new Error(`Failed to submit feedback: ${res.status}`);
 }
 
-export async function fetchStats(): Promise<Stats> {
-  const res = await fetch(`${BASE}/stats`);
+export async function fetchStats(sessionId: string | null): Promise<Stats> {
+  const url = sessionId
+    ? `${BASE}/stats?session_id=${sessionId}`
+    : `${BASE}/stats`;
+  const res = await fetch(url);
   if (!res.ok) throw new Error(`Failed to fetch stats: ${res.status}`);
   return res.json();
 }
 
-export function getExportUrl(): string {
-  return `${BASE}/export`;
+export function getExportUrl(sessionId: string | null): string {
+  return sessionId ? `${BASE}/export?session_id=${sessionId}` : `${BASE}/export`;
 }

{feedloop-0.1.1 → feedloop-0.1.2}/frontend/src/components/ExportPanel.tsx

@@ -1,16 +1,20 @@
 import { useState, useEffect } from "react";
 import { fetchStats, getExportUrl, type Stats } from "../api";
 
-export default function ExportPanel() {
+interface ExportPanelProps {
+  sessionId: string | null;
+}
+
+export default function ExportPanel({ sessionId }: ExportPanelProps) {
   const [stats, setStats] = useState<Stats | null>(null);
 
   useEffect(() => {
-    fetchStats().then(setStats).catch(() => {});
+    fetchStats(sessionId).then(setStats).catch(() => {});
     const interval = setInterval(() => {
-      fetchStats().then(setStats).catch(() => {});
+      fetchStats(sessionId).then(setStats).catch(() => {});
     }, 2000);
     return () => clearInterval(interval);
-  }, []);
+  }, [sessionId]);
 
   return (
     <div className="export-panel">
@@ -22,7 +26,7 @@ export default function ExportPanel() {
       )}
       <a
         className="btn btn-export"
-        href={getExportUrl()}
+        href={getExportUrl(sessionId)}
         download="preferences.jsonl"
       >
         Download JSONL

{feedloop-0.1.1 → feedloop-0.1.2}/frontend/src/hooks/useComparison.ts

@@ -1,20 +1,20 @@
 import { useEffect, useState, useCallback } from "react";
 import { fetchPending, type Comparison } from "../api";
 
-export function useComparison() {
+export function useComparison(sessionId: string | null) {
   const [comparison, setComparison] = useState<Comparison | null>(null);
   const [loading, setLoading] = useState(true);
 
   const poll = useCallback(async () => {
     try {
-      const data = await fetchPending();
+      const data = await fetchPending(sessionId);
       setComparison(data);
     } catch {
       // server may not be ready yet
     } finally {
       setLoading(false);
     }
-  }, []);
+  }, [sessionId]);
 
   useEffect(() => {
     poll();

{feedloop-0.1.1 → feedloop-0.1.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "feedloop"
-version = "0.1.1"
+version = "0.1.2"
 description = "The fastest way to collect human preference data for LLMs"
 readme = "README.md"
 license = "MIT"

{feedloop-0.1.1 → feedloop-0.1.2}/src/feedloop/__init__.py

@@ -12,7 +12,7 @@ from ._db import Database
 from ._export import export_dpo
 from ._server import BackgroundServer
 
-__version__ = "0.1.0"
+__version__ = "0.1.2"
 
 # ── module-level state ──────────────────────────────────────
 
@@ -46,7 +46,7 @@ def start(
 
     _session_id = uuid.uuid4().hex
     _db = Database(db_path=db_path)
-    _server = BackgroundServer(_db, port=port)
+    _server = BackgroundServer(_db, session_id=_session_id, port=port)
     _server.start()
 
     print(f"feedloop: server running at {_server.url}")

{feedloop-0.1.1 → feedloop-0.1.2}/src/feedloop/_routes.py

@@ -11,8 +11,9 @@ from ._models import ComparisonOut, FeedbackRequest, StatsResponse
 
 router = APIRouter(prefix="/api")
 
-# The database instance is set by _server.py at startup.
+# The database and session_id are set by _server.py at startup.
 _db: Database | None = None
+_session_id: str | None = None
 
 
 def set_db(db: Database) -> None:
@@ -20,12 +21,22 @@ def set_db(db: Database) -> None:
     _db = db
 
 
+def set_session_id(session_id: str) -> None:
+    global _session_id
+    _session_id = session_id
+
+
 def _get_db() -> Database:
     if _db is None:
         raise RuntimeError("Database not initialised")
     return _db
 
 
+@router.get("/session")
+def get_session() -> dict:
+    return {"session_id": _session_id}
+
+
 @router.get("/pending")
 def get_pending(session_id: str | None = None) -> ComparisonOut | None:
     db = _get_db()

{feedloop-0.1.1 → feedloop-0.1.2}/src/feedloop/_server.py

@@ -11,12 +11,12 @@ from fastapi.staticfiles import StaticFiles
 
 from ._config import DEFAULT_PORT, PORT_SCAN_RANGE
 from ._db import Database
-from ._routes import router, set_db
+from ._routes import router, set_db, set_session_id
 
 STATIC_DIR = Path(__file__).parent / "_static"
 
 
-def _create_app(db: Database) -> FastAPI:
+def _create_app(db: Database, session_id: str) -> FastAPI:
     app = FastAPI(title="feedloop", docs_url=None, redoc_url=None)
 
     app.add_middleware(
@@ -27,6 +27,7 @@ def _create_app(db: Database) -> FastAPI:
     )
 
     set_db(db)
+    set_session_id(session_id)
     app.include_router(router)
 
     # Serve React static files (only if built)
@@ -51,10 +52,10 @@ def _find_open_port(start: int) -> int:
 
 
 class BackgroundServer:
-    def __init__(self, db: Database, port: int = DEFAULT_PORT) -> None:
+    def __init__(self, db: Database, session_id: str, port: int = DEFAULT_PORT) -> None:
         self.db = db
         self.port = _find_open_port(port)
-        self.app = _create_app(db)
+        self.app = _create_app(db, session_id)
         self._thread: threading.Thread | None = None
         self._server: uvicorn.Server | None = None