feedloop 0.1.0__py3-none-any.whl

feedloop/__init__.py

@@ -0,0 +1,155 @@
+"""feedloop — The fastest way to collect human preference data for LLMs."""
+
+from __future__ import annotations
+
+import atexit
+import time
+import uuid
+import webbrowser
+
+from ._config import DEFAULT_PORT
+from ._db import Database
+from ._export import export_dpo
+from ._server import BackgroundServer
+
+__version__ = "0.1.0"
+
+# ── module-level state ──────────────────────────────────────
+
+_server: BackgroundServer | None = None
+_db: Database | None = None
+_session_id: str | None = None
+
+
+def _ensure_started() -> tuple[BackgroundServer, Database, str]:
+    if _server is None or _db is None or _session_id is None:
+        raise RuntimeError("Call feedloop.start() first")
+    return _server, _db, _session_id
+
+
+# ── public API ──────────────────────────────────────────────
+
+
+def start(
+    port: int = DEFAULT_PORT,
+    db_path: str | None = None,
+    open_browser: bool = True,
+) -> None:
+    """Launch the HITL feedback server in a background thread.
+
+    Idempotent — calling start() twice reuses the running server.
+    """
+    global _server, _db, _session_id
+
+    if _server is not None:
+        return
+
+    _session_id = uuid.uuid4().hex
+    _db = Database(db_path=db_path)
+    _server = BackgroundServer(_db, port=port)
+    _server.start()
+
+    print(f"feedloop: server running at {_server.url}")
+
+    if open_browser:
+        webbrowser.open(_server.url)
+
+    atexit.register(stop)
+
+
+def compare(
+    prompt: str,
+    outputs: list[str],
+    *,
+    metadata: dict | None = None,
+) -> str:
+    """Submit a comparison for human review. Returns comparison_id.
+
+    Non-blocking — returns immediately. The comparison appears
+    in the web UI for the human to evaluate.
+    """
+    _, db, session_id = _ensure_started()
+
+    if len(outputs) != 2:
+        raise ValueError("outputs must contain exactly 2 items")
+
+    return db.insert_comparison(
+        session_id=session_id,
+        prompt=prompt,
+        output_a=outputs[0],
+        output_b=outputs[1],
+        metadata=metadata,
+    )
+
+
+def wait(
+    comparison_id: str | None = None,
+    timeout: float | None = None,
+) -> dict | None:
+    """Block until a specific comparison (or all pending) is rated.
+
+    Returns the result dict or None on timeout.
+    """
+    _, db, session_id = _ensure_started()
+    deadline = time.monotonic() + timeout if timeout else None
+
+    while True:
+        if comparison_id:
+            row = db.get_comparison(comparison_id)
+            if row and row["status"] != "pending":
+                if row["chosen"] == "a":
+                    chosen, rejected = row["output_a"], row["output_b"]
+                else:
+                    chosen, rejected = row["output_b"], row["output_a"]
+                return {
+                    "prompt": row["prompt"],
+                    "chosen": chosen,
+                    "rejected": rejected,
+                }
+        else:
+            stats = db.get_stats(session_id=session_id)
+            if stats["pending"] == 0 and stats["total"] > 0:
+                return {"completed": stats["completed"], "total": stats["total"]}
+
+        if deadline and time.monotonic() >= deadline:
+            return None
+
+        time.sleep(0.3)
+
+
+def status() -> dict:
+    """Return {"pending": N, "completed": M, "skipped": S, "total": T}."""
+    _, db, session_id = _ensure_started()
+    return db.get_stats(session_id=session_id)
+
+
+def export(
+    path: str = "preferences.jsonl",
+    *,
+    session_id: str | None = None,
+    format: str = "dpo",
+) -> int:
+    """Export completed comparisons to JSONL. Returns count exported."""
+    _, db, current_session = _ensure_started()
+
+    if format != "dpo":
+        raise ValueError(f"Unsupported format: {format!r}. Only 'dpo' is supported.")
+
+    sid = session_id if session_id is not None else current_session
+    count = export_dpo(db, path, session_id=sid)
+    print(f"feedloop: exported {count} preferences to {path}")
+    return count
+
+
+def stop() -> None:
+    """Shut down the background server."""
+    global _server, _db, _session_id
+
+    if _server:
+        _server.stop()
+    if _db:
+        _db.close()
+
+    _server = None
+    _db = None
+    _session_id = None

feedloop/__main__.py

@@ -0,0 +1,44 @@
+"""CLI entry point: python -m feedloop"""
+
+from __future__ import annotations
+
+import argparse
+import signal
+import sys
+
+from . import start, stop
+from ._config import DEFAULT_PORT
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="feedloop",
+        description="The fastest way to collect human preference data for LLMs",
+    )
+    parser.add_argument(
+        "--port", type=int, default=DEFAULT_PORT, help="Port to listen on"
+    )
+    parser.add_argument("--db", type=str, default=None, help="Path to SQLite database")
+    parser.add_argument(
+        "--no-browser", action="store_true", help="Don't open browser automatically"
+    )
+    args = parser.parse_args()
+
+    start(port=args.port, db_path=args.db, open_browser=not args.no_browser)
+
+    print("feedloop: Press Ctrl+C to stop")
+
+    def _handle_signal(sig: int, frame: object) -> None:
+        print("\nfeedloop: shutting down")
+        stop()
+        sys.exit(0)
+
+    signal.signal(signal.SIGINT, _handle_signal)
+    signal.signal(signal.SIGTERM, _handle_signal)
+
+    # Block forever (server runs in daemon thread)
+    signal.pause()
+
+
+if __name__ == "__main__":
+    main()

feedloop/_config.py

@@ -0,0 +1,6 @@
+from pathlib import Path
+
+DEFAULT_PORT = 7856
+DEFAULT_DB_DIR = Path.home() / ".feedloop"
+DEFAULT_DB_PATH = DEFAULT_DB_DIR / "feedloop.db"
+PORT_SCAN_RANGE = 10  # try ports 7856-7866 if default is busy

feedloop/_db.py

@@ -0,0 +1,172 @@
+from __future__ import annotations
+
+import json
+import random
+import sqlite3
+import threading
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+
+from ._config import DEFAULT_DB_DIR, DEFAULT_DB_PATH
+
+_SCHEMA = """\
+CREATE TABLE IF NOT EXISTS comparisons (
+    id            TEXT PRIMARY KEY,
+    session_id    TEXT NOT NULL,
+    prompt        TEXT NOT NULL,
+    output_a      TEXT NOT NULL,
+    output_b      TEXT NOT NULL,
+    display_order TEXT NOT NULL,
+    status        TEXT NOT NULL DEFAULT 'pending',
+    chosen        TEXT,
+    created_at    TEXT NOT NULL,
+    completed_at  TEXT,
+    metadata      TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_comparisons_status ON comparisons(status);
+CREATE INDEX IF NOT EXISTS idx_comparisons_session ON comparisons(session_id);
+"""
+
+
+class Database:
+    def __init__(self, db_path: str | Path | None = None) -> None:
+        self._path = Path(db_path) if db_path else DEFAULT_DB_PATH
+        self._path.parent.mkdir(parents=True, exist_ok=True)
+        self._lock = threading.Lock()
+        self._conn = sqlite3.connect(str(self._path), check_same_thread=False)
+        self._conn.row_factory = sqlite3.Row
+        self._conn.execute("PRAGMA journal_mode=WAL")
+        self._conn.executescript(_SCHEMA)
+
+    # ── writes ──────────────────────────────────────────────
+
+    def insert_comparison(
+        self,
+        session_id: str,
+        prompt: str,
+        output_a: str,
+        output_b: str,
+        metadata: dict | None = None,
+    ) -> str:
+        cid = uuid.uuid4().hex
+        display_order = random.choice(["ab", "ba"])
+        now = datetime.now(timezone.utc).isoformat()
+        with self._lock:
+            self._conn.execute(
+                """INSERT INTO comparisons
+                   (id, session_id, prompt, output_a, output_b,
+                    display_order, status, created_at, metadata)
+                   VALUES (?, ?, ?, ?, ?, ?, 'pending', ?, ?)""",
+                (
+                    cid,
+                    session_id,
+                    prompt,
+                    output_a,
+                    output_b,
+                    display_order,
+                    now,
+                    json.dumps(metadata) if metadata else None,
+                ),
+            )
+            self._conn.commit()
+        return cid
+
+    def record_feedback(self, comparison_id: str, chosen_side: str) -> None:
+        """Record a preference. chosen_side is 'left' or 'right'."""
+        row = self.get_comparison(comparison_id)
+        if row is None:
+            raise ValueError(f"Comparison {comparison_id} not found")
+
+        # Map display side to original output
+        display_order = row["display_order"]
+        if chosen_side == "left":
+            chosen = display_order[0]  # 'a' or 'b'
+        else:
+            chosen = display_order[1]
+
+        now = datetime.now(timezone.utc).isoformat()
+        with self._lock:
+            self._conn.execute(
+                """UPDATE comparisons
+                   SET chosen = ?, status = 'completed', completed_at = ?
+                   WHERE id = ?""",
+                (chosen, now, comparison_id),
+            )
+            self._conn.commit()
+
+    def skip_comparison(self, comparison_id: str) -> None:
+        now = datetime.now(timezone.utc).isoformat()
+        with self._lock:
+            self._conn.execute(
+                """UPDATE comparisons
+                   SET status = 'skipped', completed_at = ?
+                   WHERE id = ?""",
+                (now, comparison_id),
+            )
+            self._conn.commit()
+
+    # ── reads ───────────────────────────────────────────────
+
+    def get_comparison(self, comparison_id: str) -> sqlite3.Row | None:
+        cur = self._conn.execute(
+            "SELECT * FROM comparisons WHERE id = ?", (comparison_id,)
+        )
+        return cur.fetchone()
+
+    def get_next_pending(self, session_id: str | None = None) -> sqlite3.Row | None:
+        if session_id:
+            cur = self._conn.execute(
+                """SELECT * FROM comparisons
+                   WHERE status = 'pending' AND session_id = ?
+                   ORDER BY created_at ASC LIMIT 1""",
+                (session_id,),
+            )
+        else:
+            cur = self._conn.execute(
+                """SELECT * FROM comparisons
+                   WHERE status = 'pending'
+                   ORDER BY created_at ASC LIMIT 1"""
+            )
+        return cur.fetchone()
+
+    def get_stats(self, session_id: str | None = None) -> dict:
+        where = "WHERE session_id = ?" if session_id else ""
+        params: tuple = (session_id,) if session_id else ()
+        cur = self._conn.execute(
+            f"""SELECT
+                   SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END) as pending,
+                   SUM(CASE WHEN status = 'completed' THEN 1 ELSE 0 END) as completed,
+                   SUM(CASE WHEN status = 'skipped' THEN 1 ELSE 0 END) as skipped,
+                   COUNT(*) as total
+               FROM comparisons {where}""",
+            params,
+        )
+        row = cur.fetchone()
+        return {
+            "pending": row["pending"] or 0,
+            "completed": row["completed"] or 0,
+            "skipped": row["skipped"] or 0,
+            "total": row["total"] or 0,
+        }
+
+    def get_completed(self, session_id: str | None = None) -> list[sqlite3.Row]:
+        if session_id:
+            cur = self._conn.execute(
+                """SELECT * FROM comparisons
+                   WHERE status = 'completed' AND session_id = ?
+                   ORDER BY completed_at ASC""",
+                (session_id,),
+            )
+        else:
+            cur = self._conn.execute(
+                """SELECT * FROM comparisons
+                   WHERE status = 'completed'
+                   ORDER BY completed_at ASC"""
+            )
+        return cur.fetchall()
+
+    # ── lifecycle ───────────────────────────────────────────
+
+    def close(self) -> None:
+        self._conn.close()

feedloop/_export.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from ._db import Database
+
+
+def export_dpo(db: Database, path: str | Path, session_id: str | None = None) -> int:
+    """Export completed comparisons as DPO-formatted JSONL. Returns row count."""
+    rows = db.get_completed(session_id=session_id)
+    path = Path(path)
+
+    with path.open("w", encoding="utf-8") as f:
+        for row in rows:
+            if row["chosen"] == "a":
+                chosen_text = row["output_a"]
+                rejected_text = row["output_b"]
+            else:
+                chosen_text = row["output_b"]
+                rejected_text = row["output_a"]
+
+            record = {
+                "prompt": row["prompt"],
+                "chosen": chosen_text,
+                "rejected": rejected_text,
+            }
+            f.write(json.dumps(record, ensure_ascii=False) + "\n")
+
+    return len(rows)

feedloop/_models.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+
+class ComparisonOut(BaseModel):
+    id: str
+    prompt: str
+    output_left: str
+    output_right: str
+    progress_completed: int
+    progress_total: int
+
+
+class FeedbackRequest(BaseModel):
+    comparison_id: str
+    chosen_side: str  # "left" | "right"
+
+
+class StatsResponse(BaseModel):
+    pending: int
+    completed: int
+    skipped: int
+    total: int
+
+
+class ExportRow(BaseModel):
+    prompt: str
+    chosen: str
+    rejected: str

feedloop/_routes.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import io
+import json
+
+from fastapi import APIRouter, HTTPException
+from fastapi.responses import StreamingResponse
+
+from ._db import Database
+from ._models import ComparisonOut, FeedbackRequest, StatsResponse
+
+router = APIRouter(prefix="/api")
+
+# The database instance is set by _server.py at startup.
+_db: Database | None = None
+
+
+def set_db(db: Database) -> None:
+    global _db
+    _db = db
+
+
+def _get_db() -> Database:
+    if _db is None:
+        raise RuntimeError("Database not initialised")
+    return _db
+
+
+@router.get("/pending")
+def get_pending(session_id: str | None = None) -> ComparisonOut | None:
+    db = _get_db()
+    row = db.get_next_pending(session_id=session_id)
+    if row is None:
+        return None
+
+    stats = db.get_stats(session_id=session_id)
+
+    # Apply display_order to decide what the UI sees as left/right
+    if row["display_order"] == "ab":
+        left, right = row["output_a"], row["output_b"]
+    else:
+        left, right = row["output_b"], row["output_a"]
+
+    return ComparisonOut(
+        id=row["id"],
+        prompt=row["prompt"],
+        output_left=left,
+        output_right=right,
+        progress_completed=stats["completed"],
+        progress_total=stats["total"],
+    )
+
+
+@router.post("/feedback")
+def post_feedback(req: FeedbackRequest) -> dict:
+    db = _get_db()
+    if req.chosen_side not in ("left", "right", "skip"):
+        raise HTTPException(400, "chosen_side must be 'left', 'right', or 'skip'")
+
+    if req.chosen_side == "skip":
+        db.skip_comparison(req.comparison_id)
+    else:
+        db.record_feedback(req.comparison_id, req.chosen_side)
+
+    return {"ok": True}
+
+
+@router.get("/stats")
+def get_stats(session_id: str | None = None) -> StatsResponse:
+    db = _get_db()
+    s = db.get_stats(session_id=session_id)
+    return StatsResponse(**s)
+
+
+@router.get("/export")
+def export_jsonl(session_id: str | None = None) -> StreamingResponse:
+    db = _get_db()
+    rows = db.get_completed(session_id=session_id)
+
+    lines: list[str] = []
+    for row in rows:
+        if row["chosen"] == "a":
+            chosen_text, rejected_text = row["output_a"], row["output_b"]
+        else:
+            chosen_text, rejected_text = row["output_b"], row["output_a"]
+
+        record = {
+            "prompt": row["prompt"],
+            "chosen": chosen_text,
+            "rejected": rejected_text,
+        }
+        lines.append(json.dumps(record, ensure_ascii=False))
+
+    content = "\n".join(lines) + "\n" if lines else ""
+    return StreamingResponse(
+        io.BytesIO(content.encode("utf-8")),
+        media_type="application/x-ndjson",
+        headers={"Content-Disposition": "attachment; filename=preferences.jsonl"},
+    )

feedloop/_server.py

@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import socket
+import threading
+from pathlib import Path
+
+import uvicorn
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+
+from ._config import DEFAULT_PORT, PORT_SCAN_RANGE
+from ._db import Database
+from ._routes import router, set_db
+
+STATIC_DIR = Path(__file__).parent / "_static"
+
+
+def _create_app(db: Database) -> FastAPI:
+    app = FastAPI(title="feedloop", docs_url=None, redoc_url=None)
+
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["*"],
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    set_db(db)
+    app.include_router(router)
+
+    # Serve React static files (only if built)
+    index = STATIC_DIR / "index.html"
+    if index.exists():
+        app.mount("/", StaticFiles(directory=STATIC_DIR, html=True), name="static")
+
+    return app
+
+
+def _find_open_port(start: int) -> int:
+    for port in range(start, start + PORT_SCAN_RANGE):
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            try:
+                s.bind(("127.0.0.1", port))
+                return port
+            except OSError:
+                continue
+    raise RuntimeError(
+        f"No open port found in range {start}-{start + PORT_SCAN_RANGE - 1}"
+    )
+
+
+class BackgroundServer:
+    def __init__(self, db: Database, port: int = DEFAULT_PORT) -> None:
+        self.db = db
+        self.port = _find_open_port(port)
+        self.app = _create_app(db)
+        self._thread: threading.Thread | None = None
+        self._server: uvicorn.Server | None = None
+
+    @property
+    def url(self) -> str:
+        return f"http://localhost:{self.port}"
+
+    def start(self) -> None:
+        if self._thread and self._thread.is_alive():
+            return  # already running
+
+        config = uvicorn.Config(
+            self.app,
+            host="127.0.0.1",
+            port=self.port,
+            log_level="warning",
+        )
+        self._server = uvicorn.Server(config)
+
+        self._thread = threading.Thread(target=self._server.run, daemon=True)
+        self._thread.start()
+
+        # Wait for server to be ready
+        self._server.started = True  # type: ignore[assignment]
+        import time
+
+        for _ in range(50):
+            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+                try:
+                    s.connect(("127.0.0.1", self.port))
+                    return
+                except ConnectionRefusedError:
+                    time.sleep(0.1)
+
+    def stop(self) -> None:
+        if self._server:
+            self._server.should_exit = True
+        if self._thread:
+            self._thread.join(timeout=3)
+            self._thread = None

feedloop-0.1.0.dist-info/METADATA

@@ -0,0 +1,41 @@
+Metadata-Version: 2.4
+Name: feedloop
+Version: 0.1.0
+Summary: The fastest way to collect human preference data for LLMs
+Project-URL: Homepage, https://github.com/rammuthiah/feedloop
+Project-URL: Repository, https://github.com/rammuthiah/feedloop
+Author: Ram Muthiah
+License-Expression: MIT
+License-File: LICENSE
+Keywords: dpo,human-feedback,llm,preference-data,rlhf
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Requires-Dist: fastapi>=0.100
+Requires-Dist: pydantic>=2.0
+Requires-Dist: uvicorn[standard]>=0.20
+Description-Content-Type: text/markdown
+
+# feedloop
+
+The fastest way to collect human preference data for LLMs.
+
+```python
+import feedloop
+
+feedloop.start()
+
+feedloop.compare(
+    prompt="Explain quantum computing",
+    outputs=["Response A", "Response B"],
+)
+
+# Rate in the browser, then export
+feedloop.export("preferences.jsonl")
+```

feedloop-0.1.0.dist-info/RECORD

@@ -0,0 +1,14 @@
+feedloop/__init__.py,sha256=U_UWo_v62o035CdBamG3CyXKhW1MFDAE2gbgz4bbng4,4327
+feedloop/__main__.py,sha256=b2_sMsf8GecNM9_wOJia0XzMh20rsmncisZT__5OJGQ,1164
+feedloop/_config.py,sha256=QBsZ_e-rxI5wexSzx50Ud0dV8URwPO-zGQZIBBY-HFw,201
+feedloop/_db.py,sha256=2xviZRyhUV0rGXlkCHIHDqdey6H9otwfIP48RXFO7ec,6282
+feedloop/_export.py,sha256=e2H-FNtqcf5nvlWFBtEFkYZF0xicLyZRy6A8g3dUyQk,910
+feedloop/_models.py,sha256=BESWGX574rqo1ZoJCipewZyQuNlusXzEP7NpDfCjA-s,506
+feedloop/_routes.py,sha256=uy3HOp5o5AdoYBH86twI_XVj9UFAfXZfEJqgNSApcSI,2727
+feedloop/_server.py,sha256=QU1w691CcBdH3KbJDDGEthynUCsi4Tsc0CwA1VpLyEs,2745
+feedloop/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+feedloop-0.1.0.dist-info/METADATA,sha256=CBieh0T40BwRcQGSBf_hp8SAh8URFM6m_GUgz6oZfdw,1235
+feedloop-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+feedloop-0.1.0.dist-info/entry_points.txt,sha256=54qG4ozlvOnlatw-PkW-BzU1rITiPtd-moyvwWOFmi4,52
+feedloop-0.1.0.dist-info/licenses/LICENSE,sha256=ubUN0vZDzv-csuMqe-LzQebe_3bHkuMazPGB0TstuOw,1068
+feedloop-0.1.0.dist-info/RECORD,,

feedloop-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

feedloop-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+feedloop = feedloop.__main__:main

feedloop-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Ram Muthiah
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.