librarian-code 0.1.0__py3-none-any.whl

librarian/memory/retriever.py

@@ -0,0 +1,62 @@
+import os
+import chromadb
+from sentence_transformers import SentenceTransformer
+from librarian.utils.config import CHROMA_PERSIST_DIR, EMBED_MODEL
+from librarian.memory.indexer import _sanitize_collection_name
+
+_model = None
+_client = None
+
+
+def _get_model():
+    global _model
+    if _model is None:
+        _model = SentenceTransformer(EMBED_MODEL)
+    return _model
+
+
+def _get_client():
+    global _client
+    if _client is None:
+        _client = chromadb.PersistentClient(path=CHROMA_PERSIST_DIR)
+    return _client
+
+
+def retrieve(query: str, n_results: int = 5) -> list[dict]:
+    model = _get_model()
+    client = _get_client()
+    project_name = _sanitize_collection_name(os.path.basename(os.getcwd()))
+
+    try:
+        collection = client.get_collection(name=project_name)
+    except Exception:
+        return []
+
+    query_embedding = model.encode([query]).tolist()
+    results = collection.query(
+        query_embeddings=query_embedding,
+        n_results=n_results,
+        include=["documents", "metadatas", "distances"],
+    )
+
+    if not results.get("documents") or not results["documents"][0]:
+        return []
+
+    chunks = []
+    for doc, meta, dist in zip(
+        results["documents"][0],
+        results["metadatas"][0],
+        results["distances"][0],
+    ):
+        if dist > 2.5:
+            continue
+        chunks.append({
+            "content": doc,
+            "metadata": meta,
+            "distance": dist,
+        })
+
+    if chunks and sum(c["distance"] for c in chunks) / len(chunks) > 2.0:
+        return []
+
+    return chunks

librarian/orchestrator/core.py

@@ -0,0 +1,47 @@
+from pathlib import Path
+from librarian.orchestrator.router import get_response
+from librarian.memory.retriever import retrieve
+from librarian.skills.loader import build_skill_context
+
+
+def build_system_prompt(project_conventions: str, skill_context: str = "") -> str:
+    parts = [
+        "You are Librarian, a CLI coding agent with memory of this project.",
+        f"Project conventions:\n{project_conventions}",
+    ]
+    if skill_context:
+        parts.append(f"Domain-specific best practices:\n{skill_context}")
+    parts.append("""Rules:
+- Always explain what you are about to do before doing it
+- Never delete files without explicit confirmation
+- Prefer editing existing code over rewriting from scratch
+- When uncertain, ask rather than assume
+- Be concise and direct in your answers
+- Cite source files when referencing code (e.g. "in auth.py line 42")
+""")
+    return "\n\n".join(parts)
+
+
+def read_librarian_md() -> str:
+    path = Path("LIBRARIAN.md")
+    if path.exists():
+        return path.read_text(encoding="utf-8")
+    return "No project conventions file found."
+
+
+def ask(question: str) -> tuple[str, str, int]:
+    conventions = read_librarian_md()
+    skill_ctx = build_skill_context()
+    system = build_system_prompt(conventions, skill_ctx)
+
+    chunks = retrieve(question, n_results=5)
+    context = ""
+    if chunks:
+        parts = []
+        for c in chunks:
+            meta = c["metadata"]
+            parts.append(f"--- {meta['file_path']}:{meta.get('start_line', '?')}-{meta.get('end_line', '?')} ---\n{c['content']}")
+        context = "\n\n".join(parts)
+
+    prompt = f"Relevant code context:\n{context}\n\nQuestion: {question}" if context else question
+    return get_response(system, prompt)

librarian/orchestrator/router.py

@@ -0,0 +1,17 @@
+from librarian.adapter.groq_adapter import GroqAdapter
+from librarian.adapter.openrouter_adapter import OpenRouterAdapter
+from librarian.exceptions import RateLimitError, ProviderUnavailableError
+from librarian.utils.logger import log_warning
+
+
+def get_response(system: str, prompt: str) -> tuple[str, str, int]:
+    primary = GroqAdapter()
+    fallback = OpenRouterAdapter()
+
+    try:
+        response = primary.complete(system, prompt)
+        return response, "groq", primary.tokens_used
+    except (RateLimitError, ProviderUnavailableError) as e:
+        log_warning(f"{e} — switching to OpenRouter")
+        response = fallback.complete(system, prompt)
+        return response, "openrouter", fallback.tokens_used

librarian/skills/bundled/api-design/conventions.md

@@ -0,0 +1,93 @@
+# API Design Conventions
+
+## URL Structure
+- Use nouns, not verbs (HTTP methods are the verbs)
+- Plural nouns for collections: `/users`, `/orders`
+- Hyphens for multi-word segments: `/managed-devices`
+- Lowercase consistently
+- No trailing slashes, no file extensions
+- Limit nesting depth to 3 levels max
+
+## HTTP Methods
+| Method | Purpose | Success Code |
+|--------|---------|-------------|
+| GET | Read resource | 200 |
+| POST | Create resource | 201 (with Location header) |
+| PUT | Full replace | 200, 201, 204 |
+| PATCH | Partial update | 200, 204 |
+| DELETE | Remove resource | 200, 204 |
+
+- POST to collection → 201 Created + Location header
+- PUT is idempotent, POST is not
+- Use PATCH for partial updates, not PUT
+
+## Status Codes
+- 200: Generic success (GET, PUT, PATCH)
+- 201: Resource created (POST) — always include Location header
+- 204: Success with no body (DELETE)
+- 400: Bad request / validation error
+- 401: Not authenticated (missing/invalid credentials)
+- 403: Not authorized (insufficient permissions)
+- 404: Resource not found
+- 409: Conflict (duplicate creation)
+- 422: Semantically invalid input
+- 429: Rate limited — include Retry-After header
+- 500: Server error
+
+401 ≠ 403: 401 = not logged in, 403 = logged in but no permission
+
+## Request/Response Patterns
+- Always use `Content-Type: application/json`
+- Collection responses: `{ "data": [...], "meta": { "total", "page", "hasNext" } }`
+- Single resource: `{ "id": 1, "name": "..." }`
+- Creation response: 201 + Location header + resource body
+- Error response: `{ "error": { "code": "VALIDATION_ERROR", "message": "...", "details": [...] } }`
+- Never expose stack traces or internal errors in production
+
+## Pagination
+- Offset-based: `?page=2&limit=25` (simple)
+- Cursor-based: `?after=<cursor>&first=25` (preferred for large datasets)
+- Always include: total, hasNext/hasPrev
+
+## Filtering & Sorting
+- Filter via query params: `?category=electronics&status=active`
+- Sort: `sort=field` (asc), `sort=-field` (desc)
+- Search: `?search=john`
+
+## Authentication
+- JWT: `Authorization: Bearer <token>` header
+- API Key: `X-API-Key: <key>` header (never in URLs)
+- Always use HTTPS
+- Hash passwords with bcrypt/scrypt
+- Token refresh: short-lived access + long-lived refresh
+
+## Rate Limiting
+- Return 429 with headers: `Retry-After`, `X-RateLimit-Limit`, `X-RateLimit-Remaining`
+- Implement per-user, per-IP, or per-API-key
+
+## CORS
+- Set explicit origins (never `*` with credentials)
+- Handle OPTIONS preflight
+- Return CORS headers on error responses too
+
+## Folder Structure
+```
+api/
+  routes/       # Route definitions (thin controllers)
+  services/     # Business logic
+  models/       # Database models
+  schemas/      # Request/response schemas (Pydantic/Zod)
+  middleware/    # Auth, rate limiting, CORS
+```
+
+## Anti-Patterns to Avoid
+- Verbs in URIs: `/getUser` → use `GET /users/{id}`
+- Returning 200 for errors → use proper 4xx/5xx
+- Missing Location header on 201
+- Exposing database internals in API responses
+- No input validation
+- Hardcoded credentials
+- Missing CORS configuration
+- No pagination on collection endpoints
+- GET for write operations
+- Not using dependency injection for DB connections

librarian/skills/bundled/python/conventions.md

@@ -0,0 +1,59 @@
+# Python Conventions
+
+## Project Structure
+- Use `src/` layout for libraries, flat layout for apps
+- Package dirs: lowercase with underscores (`my_package`)
+- Always include `__init__.py` in every package dir
+- Include `py.typed` marker for PEP 561
+- Config in `pyproject.toml` (never `setup.py` for new projects)
+- Build backend: `hatchling` or `uv_build`
+- Package manager: `uv` recommended, `poetry` acceptable
+
+## Code Style (PEP 8)
+- 4 spaces indentation, no tabs
+- 88 char line length (Black/ruff default)
+- 2 blank lines before top-level functions/classes
+- 1 blank line between methods inside classes
+- snake_case for functions/variables, PascalCase for classes, UPPER_CASE for constants
+
+## Imports
+- Order: stdlib → third-party → local (separate with blank lines)
+- One import per line
+- Prefer absolute imports over relative
+- Never use `from x import *`
+
+## Type Hints (Modern Python 3.10+)
+- Use built-in generics: `list[int]` not `List[int]`, `dict[str, Any]` not `Dict`
+- Use `X | None` not `Optional[X]`
+- Use `X | Y` not `Union[X, Y]`
+- Always annotate return types and parameters
+- Use `type` keyword for type aliases (Python 3.12+)
+
+## Patterns
+- Use `pathlib.Path` over `os.path`
+- Use f-strings over `.format()` or `%` formatting
+- Use `with` context managers for file I/O (always specify `encoding="utf-8"`)
+- Use `dataclasses` over manual `__init__`/`__repr__`/`__eq__`
+- Use `match/case` (Python 3.10+) over long if/elif chains
+- Use `@contextmanager` for custom context managers
+- Check `is None` / `is not None`, never `== None`
+- Check `if x:` not `if len(x) > 0:`
+
+## Testing (pytest)
+- Test files: `test_*.py`
+- Test functions: `test_<description>`
+- Fixtures in `conftest.py`
+- Use `@pytest.mark.parametrize` for multiple test cases
+- Use `tmp_path` fixture for file operations
+- Use `monkeypatch` for mocking
+
+## Anti-Patterns to Avoid
+- Bare `except:` → use `except Exception:`
+- Mutable default args `def f(x=[])` → use `None` + create inside
+- `os.path` → use `pathlib.Path`
+- `List[int]` → use `list[int]`
+- `print()` in libraries → use `logging`
+- `os.system()` → use `subprocess`
+- `eval()`/`exec()` → use `ast.literal_eval()` if needed
+- Missing `encoding=` on `open()` → always specify `encoding="utf-8"`
+- String concatenation in loops → use `"".join()`

librarian/skills/bundled/react/conventions.md

@@ -0,0 +1,83 @@
+# React / Next.js Conventions
+
+## Project Structure (App Router — always prefer over Pages Router)
+```
+src/
+  app/
+    layout.tsx          # Root layout (REQUIRED, must have <html> and <body>)
+    page.tsx            # Home page
+    loading.tsx         # Suspense fallback
+    error.tsx           # Error boundary (MUST be Client Component)
+    not-found.tsx       # 404 UI
+    (route-group)/      # Organize without affecting URL
+    [param]/            # Dynamic segments
+    api/route.ts        # API routes (.ts NOT .tsx)
+  components/
+    ui/                 # Low-level primitives
+  lib/                  # Utilities, helpers
+  hooks/                # Shared custom hooks
+public/                 # Static assets
+```
+
+## Component Rules
+- Functional components ONLY — never class components
+- Server Components by default (no directive needed)
+- Add `'use client'` only to leaf interactive components (NOT layouts/pages)
+- Server Components can be `async`, use `await` directly
+- Client Components need `'use client'` at top, can use hooks/events
+
+## File Naming
+- Components: PascalCase (`Button.tsx`)
+- Pages/Routes: lowercase (`page.tsx`, `layout.tsx`)
+- Hooks: camelCase with `use` prefix (`useMediaQuery.ts`)
+- Utilities: camelCase (`formatDate.ts`)
+- API routes: `route.ts` (NOT .tsx)
+
+## Import Order
+1. React/Next.js core (`react`, `next/navigation`, `next/link`)
+2. Third-party libraries
+3. Internal components (`@/components/...`)
+4. Internal hooks/utils (`@/hooks/...`, `@/lib/...`)
+5. Types (`import type { ... }`)
+
+Use `@/` path alias for all internal imports.
+
+## State Management
+- Local state: `useState` for simple, `useReducer` for complex
+- Global state: Context (sparingly) or Zustand
+- Server state: React Query / TanStack Query
+- Lift state up to closest common parent when shared
+
+## Styling
+- Tailwind CSS as default
+- Use `cn()` utility (clsx + tailwind-merge) for conditional classes
+- Use `className` not `class`
+- Avoid inline styles except for dynamic values
+
+## TypeScript
+- Never use `any` — use `unknown` and narrow
+- Use `React.ReactNode` for children props
+- Use type imports: `import type { User } from '@/types'`
+- Use Zod for runtime validation
+
+## Data Fetching
+- Server Components: direct `await fetch()` or DB queries
+- Client Components: React Query or SWR
+- Never use `useEffect` for data fetching in Server Components
+- Parallel fetching: `Promise.all()`
+
+## Performance
+- Don't overuse `memo`, `useMemo`, `useCallback` — only when measured
+- Lazy load with `dynamic()` for heavy components
+- Use `loading.tsx` for instant loading states
+
+## Anti-Patterns to Avoid
+- Class components → functional only
+- `'use client'` on root layout → only on leaf components
+- `useEffect` for data fetching → Server Components or React Query
+- `class` attribute → use `className`
+- Barrel exports (`index.ts`) → import directly from files
+- `.tsx` for API routes → use `.ts`
+- Passing non-serializable props Server→Client
+- Missing `<html>` and `<body>` in root layout
+- Using Pages Router for new projects

librarian/skills/bundled/web-dev/conventions.md

@@ -0,0 +1,54 @@
+# Web Development Conventions
+
+## File Structure
+- Single-file apps: `< 300 LOC` → single `index.html` with inline `<style>` and `<script>`
+- Multi-file apps: `index.html` + `css/` + `js/` + `assets/`
+- Files: lowercase-kebab-case (`my-component.js`)
+- CSS classes: BEM or kebab-case (`.card__title--large`)
+- JS variables/functions: camelCase
+
+## HTML Rules
+- Always include: `<!DOCTYPE html>`, `<html lang="en">`, `<meta charset="UTF-8">`, viewport meta
+- Use semantic elements: `<header>`, `<nav>`, `<main>`, `<article>`, `<section>`, `<footer>`
+- Every `<img>` must have `alt` attribute
+- Use `<button>` for actions, `<a>` for navigation (not `<div onclick>`)
+- Maintain heading hierarchy: h1 > h2 > h3 (never skip)
+- Add `rel="noopener noreferrer"` on `target="_blank"` links
+
+## CSS Rules
+- Use CSS custom properties for repeated values
+- Use `box-sizing: border-box` universally
+- Use `rem` for spacing/sizing, `px` only for borders
+- Mobile-first responsive design with `min-width` media queries
+- Flexbox for 1D layouts, Grid for 2D layouts
+- Use `gap` property over margin hacks
+- Never use `float` for layout, never use `!important`
+- Max selector depth: 3 levels
+- Use `clamp()` for fluid typography
+
+## JavaScript Rules
+- Always use `const` by default, `let` when reassignment needed, never `var`
+- Always use `addEventListener`, never inline handlers
+- Use `textContent` for user-provided text (XSS prevention), never `innerHTML` with unsanitized input
+- Always use `try/catch` with async operations
+- Use `===` not `==`
+- Use arrow functions for callbacks
+- Use template literals over string concatenation
+- Use optional chaining `?.` and nullish coalescing `??`
+- Never use `eval()`
+
+## Security
+- Never insert user input as HTML (use `textContent`)
+- Validate client-side AND server-side
+- Never store secrets in localStorage
+- Use HTTPS for all external resources
+- Don't expose API keys in client-side code
+
+## Anti-Patterns to Avoid
+- `<div onclick>` → use `<button>` or `addEventListener`
+- `float: left` for grid → use flexbox or grid
+- `var` → use `const`/`let`
+- `==` → use `===`
+- `innerHTML = userInput` → use `textContent`
+- Global variables → use modules or block scope
+- `.then()` chains → use `async/await`

librarian/skills/loader.py

@@ -0,0 +1,109 @@
+"""Skill loader — auto-detects project type and loads relevant conventions."""
+
+import os
+import json
+import functools
+from pathlib import Path
+
+SKILLS_DIR = Path(__file__).parent / "bundled"
+
+
+@functools.lru_cache(maxsize=1)
+def _detect_project_type() -> list[str]:
+    cwd = Path.cwd()
+    detected = []
+
+    indicators = {
+        "web-dev": {
+            "files": ["index.html", "*.html"],
+            "extensions": [".html", ".htm", ".css", ".scss", ".less"],
+            "keywords": ["html", "css", "javascript", "website", "webpage", "landing page"],
+        },
+        "python": {
+            "files": ["pyproject.toml", "setup.py", "requirements.txt", "Pipfile", "uv.lock"],
+            "extensions": [".py", ".pyi", ".ipynb"],
+            "keywords": ["python", "pip", "uv", "poetry", "pytest", "django", "flask", "fastapi"],
+        },
+        "react": {
+            "files": ["next.config.ts", "next.config.js", "next.config.mjs"],
+            "extensions": [".tsx", ".jsx"],
+            "keywords": ["react", "next.js", "nextjs", "remix", "vite"],
+            "package_deps": ["react", "next"],
+        },
+        "api-design": {
+            "files": ["routes.py", "models.py", "schemas.py"],
+            "extensions": [],
+            "keywords": ["api", "endpoint", "rest", "graphql", "server", "backend", "fastapi", "express", "flask"],
+        },
+    }
+
+    html_files = list(cwd.rglob("*.html"))
+    py_files = list(cwd.rglob("*.py"))
+    tsx_files = list(cwd.rglob("*.tsx")) + list(cwd.rglob("*.jsx"))
+
+    for skill_name, config in indicators.items():
+        score = 0
+
+        for pattern in config["files"]:
+            if list(cwd.glob(pattern)):
+                score += 3
+
+        for ext in config["extensions"]:
+            if skill_name == "web-dev" and ext == ".html" and len(html_files) > 0:
+                score += min(len(html_files), 3)
+            elif skill_name == "python" and ext == ".py" and len(py_files) > 0:
+                score += min(len(py_files), 3)
+            elif skill_name in ("react",) and ext in (".tsx", ".jsx") and len(tsx_files) > 0:
+                score += min(len(tsx_files), 3)
+
+        if skill_name == "react":
+            pkg_json = cwd / "package.json"
+            if pkg_json.exists():
+                try:
+                    pkg = json.loads(pkg_json.read_text())
+                    deps = {**pkg.get("dependencies", {}), **pkg.get("devDependencies", {})}
+                    for dep in config.get("package_deps", []):
+                        if dep in deps:
+                            score += 5
+                except Exception:
+                    pass
+
+        if skill_name == "python":
+            if (cwd / "pyproject.toml").exists():
+                score += 5
+            if any(cwd.rglob("fastapi")) or any((cwd / f).exists() for f in ["main.py", "app.py"]):
+                score += 2
+
+        if score >= 2:
+            detected.append((skill_name, score))
+
+    detected.sort(key=lambda x: x[1], reverse=True)
+    return [name for name, _ in detected]
+
+
+def load_skill(skill_name: str) -> str | None:
+    skill_dir = SKILLS_DIR / skill_name
+    if not skill_dir.exists():
+        return None
+    conventions_file = skill_dir / "conventions.md"
+    if conventions_file.exists():
+        return conventions_file.read_text(encoding="utf-8")
+    return None
+
+
+def get_relevant_skills() -> list[str]:
+    return _detect_project_type()
+
+
+def build_skill_context() -> str:
+    skills = get_relevant_skills()
+    if not skills:
+        return ""
+
+    parts = []
+    for skill_name in skills:
+        content = load_skill(skill_name)
+        if content:
+            parts.append(f"## {skill_name} conventions\n{content}")
+
+    return "\n\n".join(parts)

librarian/utils/config.py

@@ -0,0 +1,15 @@
+from dotenv import load_dotenv
+import os
+import warnings
+
+os.environ.setdefault("HF_HUB_DISABLE_SYMLINKS_WARNING", "1")
+os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
+warnings.filterwarnings("ignore", module="huggingface_hub")
+warnings.filterwarnings("ignore", module="sentence_transformers")
+
+load_dotenv()
+
+GROQ_API_KEY = os.getenv("GROQ_API_KEY")
+OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY")
+CHROMA_PERSIST_DIR = os.getenv("CHROMA_PERSIST_DIR", ".librarian/memory")
+EMBED_MODEL = os.getenv("EMBED_MODEL", "all-MiniLM-L6-v2")

librarian/utils/logger.py

@@ -0,0 +1,32 @@
+from rich.console import Console
+from rich.theme import Theme
+
+theme = Theme({
+    "info": "bold #6366F1",
+    "success": "bold #10B981",
+    "warning": "bold #F59E0B",
+    "error": "bold #EF4444",
+    "muted": "#6B7280",
+})
+
+console = Console(theme=theme)
+
+
+def log_info(msg):
+    console.print(f"[info]→[/info] {msg}")
+
+
+def log_success(msg):
+    console.print(f"[success]✓[/success] {msg}")
+
+
+def log_warning(msg):
+    console.print(f"[warning]![/warning] {msg}")
+
+
+def log_error(msg):
+    console.print(f"[error]✗[/error] {msg}")
+
+
+def log_muted(msg):
+    console.print(f"[muted]{msg}[/muted]")

librarian/utils/token_tracker.py

@@ -0,0 +1,16 @@
+class TokenTracker:
+    def __init__(self):
+        self.session_tokens = {"groq": 0, "openrouter": 0}
+
+    def add(self, provider: str, tokens: int):
+        if provider in self.session_tokens:
+            self.session_tokens[provider] += tokens
+
+    def total(self) -> int:
+        return sum(self.session_tokens.values())
+
+    def report(self) -> str:
+        return f"groq: {self.session_tokens['groq']} · openrouter: {self.session_tokens['openrouter']}"
+
+
+tracker = TokenTracker()