basemode 0.1.0__tar.gz

basemode-0.1.0/.env.example

@@ -0,0 +1,6 @@
+OPENAI_API_KEY=
+ANTHROPIC_API_KEY=
+OPENROUTER_API_KEY=
+GROQ_API_KEY=
+GEMINI_API_KEY=
+TOGETHER_API_KEY=

basemode-0.1.0/.gitignore

@@ -0,0 +1,9 @@
+.env
+__pycache__/
+*.py[cod]
+.venv/
+dist/
+.coverage
+htmlcov/
+.pytest_cache/
+*.egg-info/

basemode-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,15 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.5.0
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-toml
+      - id: check-added-large-files

basemode-0.1.0/Makefile

@@ -0,0 +1,13 @@
+.PHONY: publish test test-integration lint
+
+publish:
+	@export $$(grep UV_PUBLISH_TOKEN .env | xargs) && uv publish
+
+test:
+	uv run pytest
+
+test-integration:
+	uv run pytest -m integration
+
+lint:
+	uv run ruff check src tests && uv run ruff format --check src tests

basemode-0.1.0/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.4
+Name: basemode
+Version: 0.1.0
+Summary: Make any LLM do raw text continuation
+Requires-Python: >=3.11
+Requires-Dist: anyio>=4.0.0
+Requires-Dist: litellm>=1.63.0
+Requires-Dist: pydantic-settings>=2.0.0
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: typer>=0.12.0

basemode-0.1.0/README.md

@@ -0,0 +1,243 @@
+# basemode
+
+**Make any LLM do raw text continuation.**
+
+Most language models today are chat-tuned — they want to respond, acknowledge, and help. This is the opposite of what you need for loom-style tree exploration, creative writing, or any workflow that requires the model to simply *continue* text as if it wrote it. Feed a chat model a sentence mid-thought and you'll get "Sure! Here's a continuation:" instead of the next word.
+
+`basemode` solves this. It wraps any LLM in the right coercion strategy for that provider — native completions API, assistant prefill, or a carefully-tuned system prompt — so you always get a clean continuation back.
+
+```bash
+basemode "The defendant, who had been sitting quietly throughout the proceedings, suddenly"
+```
+
+```
+The defendant, who had been sitting quietly throughout the proceedings, suddenly
+rose to his feet, overturning his chair. "You have no idea what actually happened
+that night," he said, his voice barely above a whisper.
+```
+
+---
+
+## Install
+
+```bash
+pip install basemode
+```
+
+Requires Python 3.11+. Uses [LiteLLM](https://docs.litellm.ai/) under the hood, so any model LiteLLM supports is available.
+
+Set API keys as environment variables or in a `.env` file:
+
+```bash
+OPENAI_API_KEY=sk-...
+ANTHROPIC_API_KEY=sk-ant-...
+OPENROUTER_API_KEY=sk-or-...
+GROQ_API_KEY=gsk_...
+GEMINI_API_KEY=...
+TOGETHER_API_KEY=...
+```
+
+---
+
+## CLI
+
+```bash
+# Stream a single continuation (defaults to gpt-4o-mini)
+basemode "The ship rounded the headland and"
+
+# Choose a model
+basemode "The ship rounded" --model anthropic/claude-3-haiku-20240307
+
+# Generate 4 parallel branches, shown side by side
+basemode "The ship rounded" --branches 4
+
+# Pipe text in
+cat chapter1.txt | basemode --model groq/llama-3.3-70b-versatile
+
+# Show which coercion strategy is being used
+basemode "text" --show-strategy
+
+# Override the auto-detected strategy
+basemode "text" --strategy prefill
+
+# Explore available models
+basemode models --available          # only models with keys configured
+basemode models --provider openai
+basemode models --search claude
+
+# Check strategy detection for any model
+basemode info mistral-large-latest
+```
+
+---
+
+## Python API
+
+```python
+from basemode import continue_text, branch_text
+
+# Stream a single continuation
+async for token in continue_text(
+    "She opened the letter with trembling hands.",
+    model="gpt-4o-mini",
+    max_tokens=200,
+    temperature=0.9,
+):
+    print(token, end="", flush=True)
+
+# Generate n parallel branches as (branch_idx, token) tuples
+async for idx, token in branch_text(
+    "She opened the letter",
+    model="anthropic/claude-3-haiku-20240307",
+    n=4,
+    max_tokens=200,
+):
+    print(f"[{idx}] {token}", end="", flush=True)
+```
+
+Token boundaries are handled correctly: `prefix + "".join(tokens)` always produces clean, properly-spaced text regardless of which strategy was used.
+
+---
+
+## How it works
+
+Chat-tuned models need different tricks depending on the provider. `basemode` auto-detects the best strategy from the model name:
+
+| Strategy | When used | How |
+|----------|-----------|-----|
+| `completion` | OpenAI base models (`davinci-002`, `gpt-3.5-turbo-instruct`) | Native `/completions` endpoint — no coercion needed |
+| `prefill` | Anthropic (`claude-*`) | Splits the prefix: last 50 chars become the start of the assistant turn, forcing the model to continue from exactly that point |
+| `system` | Everything else | System prompt instructs the model to output only continuation, plus trailing-space normalization to prevent word smashing |
+| `few_shot` | Stubborn models | Four varied examples (fiction, technical, poetry, dialogue) in the system prompt |
+| `fim` | DeepSeek Coder, StarCoder, CodeLlama | Fill-in-the-middle special tokens |
+
+Override auto-detection with `--strategy` / `strategy=` parameter.
+
+---
+
+## Model compatibility
+
+Tested results from the integration suite. Reliability is assessed on two axes: **does it produce output** and **is the output a clean continuation** (no "Sure!", no preamble, correct word boundaries).
+
+### OpenAI
+
+| Model | Strategy | Reliability | Notes |
+|-------|----------|-------------|-------|
+| `gpt-4o` | `system` | ⭐⭐⭐⭐⭐ | Excellent continuation quality, zero preamble |
+| `gpt-4o-mini` | `system` | ⭐⭐⭐⭐⭐ | Fast, cheap, reliable. Recommended default |
+| `gpt-3.5-turbo-instruct` | `completion` | ⭐⭐⭐⭐⭐ | Native completions endpoint, no coercion needed |
+| `davinci-002` | `completion` | ⭐⭐⭐⭐⭐ | True base model, best raw continuation behavior |
+
+### Anthropic
+
+| Model | Strategy | Reliability | Notes |
+|-------|----------|-------------|-------|
+| `anthropic/claude-3-opus-20240229` | `prefill` | ⭐⭐⭐⭐⭐ | Superb quality, stays in voice perfectly |
+| `anthropic/claude-3-5-sonnet-20241022` | `prefill` | ⭐⭐⭐⭐⭐ | Best balance of quality and speed |
+| `anthropic/claude-3-haiku-20240307` | `prefill` | ⭐⭐⭐⭐⭐ | Fast and reliable, tested extensively here |
+| `anthropic/claude-3-5-haiku-20241022` | `prefill` | ⭐⭐⭐⭐ | Use dated model ID; `claude-3-5-haiku-latest` alias not supported by all API tiers |
+
+> **The prefill trick**: Anthropic's API lets you pre-fill the assistant's response. `basemode` puts the last 50 characters of your prefix into the assistant turn, so Claude is literally mid-sentence before it generates a single new token. This produces the cleanest continuation of any strategy tested.
+
+### Groq
+
+| Model | Strategy | Reliability | Notes |
+|-------|----------|-------------|-------|
+| `groq/llama-3.3-70b-versatile` | `system` | ⭐⭐⭐⭐⭐ | Very fast inference, high-quality output, no preamble |
+| `groq/llama-3.1-70b-versatile` | `system` | ⭐⭐⭐⭐ | Slightly older, still reliable |
+| `groq/mixtral-8x7b-32768` | `system` | ⭐⭐⭐⭐ | Good for longer context continuations |
+
+Groq's speed (often under 1s for short continuations) makes it excellent for interactive loom exploration.
+
+### Google Gemini
+
+| Model | Strategy | Reliability | Notes |
+|-------|----------|-------------|-------|
+| `gemini/gemini-2.5-flash` | `system` | ⭐⭐⭐⭐ | **Thinking model** — `basemode` automatically allocates a thinking token budget. Do not use `max_tokens < 1536` |
+| `gemini/gemini-2.5-pro` | `system` | ⭐⭐⭐⭐ | Same thinking model caveat |
+| `gemini/gemini-flash-latest` | `system` | ⭐⭐⭐ | Non-thinking, faster, but output sometimes truncates at low token counts |
+
+> **Gemini 2.5 thinking models**: These models spend tokens on internal reasoning before producing visible output. `basemode` detects Gemini 2.5 models and automatically sets a `thinking` budget, ensuring the visible output isn't starved. If you're passing `max_tokens` directly, use at least `1536`.
+
+### Together AI
+
+| Model | Strategy | Reliability | Notes |
+|-------|----------|-------------|-------|
+| `together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo` | `system` | ⭐⭐⭐⭐⭐ | Excellent quality and speed, well-behaved with system prompt |
+| `together_ai/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo` | `system` | ⭐⭐⭐⭐⭐ | Best open-weight quality available |
+| `together_ai/mistralai/Mixtral-8x22B-Instruct-v0.1` | `system` | ⭐⭐⭐⭐ | Strong multilingual continuation |
+
+### OpenRouter
+
+OpenRouter routes to many providers. Any model available there works via `openrouter/<provider>/<model>`:
+
+```bash
+basemode "text" --model openrouter/openai/gpt-4o
+basemode "text" --model openrouter/anthropic/claude-3-5-sonnet
+basemode "text" --model openrouter/mistralai/mistral-large
+```
+
+| Reliability | Notes |
+|-------------|-------|
+| ⭐⭐⭐⭐ | Model availability varies; use `basemode models --provider openrouter` to check |
+
+OpenRouter is particularly useful for accessing base model variants (e.g. `openrouter/mistralai/mistral-7b`) which don't require any coercion.
+
+---
+
+## Reliability ratings
+
+| ⭐⭐⭐⭐⭐ | Zero preamble, correct spacing, stays in voice |
+|-----------|---|
+| ⭐⭐⭐⭐ | Occasionally needs strategy override or specific model ID |
+| ⭐⭐⭐ | Works but has edge cases or quirks |
+| ⭐⭐ | Usable with manual `--strategy` override |
+| ⭐ | Unreliable, avoid |
+
+---
+
+## Development
+
+```bash
+git clone https://github.com/FergusFettes/basemode
+cd basemode
+uv sync
+cp .env.example .env  # add your API keys
+
+# Unit tests (no API calls)
+uv run pytest
+
+# Integration tests (hits real APIs, costs ~$0.01)
+uv run pytest -m integration
+```
+
+Pre-commit hooks (ruff format + lint):
+
+```bash
+uv run pre-commit install
+```
+
+---
+
+## Strategies reference
+
+```bash
+basemode strategies  # list all strategies
+basemode info <model>  # show detected strategy for any model
+```
+
+To force a specific strategy:
+
+```bash
+basemode "text" --strategy prefill   # Anthropic prefill trick
+basemode "text" --strategy system    # system prompt coercion
+basemode "text" --strategy few_shot  # few-shot examples
+basemode "text" --strategy completion # OpenAI /completions endpoint
+basemode "text" --strategy fim       # fill-in-the-middle
+```
+
+---
+
+## Why this exists
+
+This is the model layer for [loom](https://github.com/FergusFettes/loom) — a multiverse writing interface for human-AI collaboration. Every loom node needs raw continuation. Getting that reliably out of modern chat models is surprisingly hard, and the solution is different for every provider. `basemode` packages that knowledge into a single interface so loom (and anything else that needs it) doesn't have to think about it.

basemode-0.1.0/pyproject.toml

@@ -0,0 +1,62 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "basemode"
+version = "0.1.0"
+description = "Make any LLM do raw text continuation"
+requires-python = ">=3.11"
+dependencies = [
+    "litellm>=1.63.0",
+    "typer>=0.12.0",
+    "rich>=13.0.0",
+    "pydantic-settings>=2.0.0",
+    "python-dotenv>=1.0.0",
+    "anyio>=4.0.0",
+]
+
+[project.scripts]
+basemode = "basemode.cli:app"
+
+[dependency-groups]
+dev = [
+    "pytest>=8.0.0",
+    "pytest-asyncio>=0.23.0",
+    "pytest-cov>=5.0.0",
+    "ruff>=0.5.0",
+    "pre-commit>=3.7.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/basemode"]
+
+[tool.ruff]
+line-length = 88
+src = ["src"]
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "RUF"]
+ignore = ["E501"]
+
+[tool.ruff.lint.isort]
+known-first-party = ["basemode"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+addopts = "--cov=src/basemode --cov-report=term-missing -m 'not integration'"
+testpaths = ["tests"]
+markers = ["integration: hits real APIs, not run by default"]
+
+[tool.coverage.run]
+source = ["src/basemode"]
+omit = ["*/tests/*"]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "raise NotImplementedError",
+    "if TYPE_CHECKING:",
+]

basemode-0.1.0/src/basemode/__init__.py

@@ -0,0 +1,5 @@
+from .continue_ import branch_text, continue_text
+from .detect import detect_strategy
+from .params import GenerationParams
+
+__all__ = ["continue_text", "branch_text", "detect_strategy", "GenerationParams"]

basemode-0.1.0/src/basemode/cli.py

@@ -0,0 +1,136 @@
+import asyncio
+import sys
+from typing import Annotated
+
+import typer
+from rich.columns import Columns
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+from rich.text import Text
+
+from .continue_ import branch_text, continue_text
+from .detect import detect_strategy, normalize_model
+from .models import list_models, list_providers
+from .strategies import REGISTRY
+
+app = typer.Typer(help="Make any LLM do raw text continuation.")
+console = Console()
+
+_BRANCH_COLORS = ["green", "blue", "yellow", "magenta", "cyan"]
+
+
+def _read_prefix(prefix: str | None) -> str | None:
+    """Return prefix from arg, stdin pipe, or None."""
+    if prefix is not None:
+        return prefix
+    if not sys.stdin.isatty():
+        return sys.stdin.read()
+    return None
+
+
+@app.callback(invoke_without_command=True)
+def main(
+    ctx: typer.Context,
+    prefix: Annotated[str | None, typer.Argument(help="Text to continue (or pipe via stdin)")] = None,
+    model: Annotated[str, typer.Option("-m", "--model")] = "gpt-4o-mini",
+    n: Annotated[int, typer.Option("-n", "--branches", help="Number of parallel continuations")] = 1,
+    max_tokens: Annotated[int, typer.Option("--max-tokens")] = 200,
+    temperature: Annotated[float, typer.Option("-t", "--temperature")] = 0.9,
+    strategy: Annotated[str | None, typer.Option("-s", "--strategy")] = None,
+    show_strategy: Annotated[bool, typer.Option("--show-strategy")] = False,
+) -> None:
+    if ctx.invoked_subcommand is not None:
+        return
+
+    text = _read_prefix(prefix)
+    if text is None:
+        console.print(ctx.get_help())
+        return
+
+    prefix = text.rstrip("\n")
+
+    if show_strategy:
+        strat = detect_strategy(normalize_model(model), strategy)
+        console.print(f"[dim]strategy: {strat.name}[/dim]")
+
+    if n == 1:
+        asyncio.run(_stream_one(prefix, model, max_tokens, temperature, strategy))
+    else:
+        asyncio.run(_stream_branches(prefix, model, n, max_tokens, temperature, strategy))  # noqa: E501
+
+
+async def _stream_one(prefix: str, model: str, max_tokens: int, temperature: float, strategy: str | None) -> None:
+    console.print(f"[dim]{prefix}[/dim]", end="")
+    async for token in continue_text(prefix, model, max_tokens=max_tokens, temperature=temperature, strategy=strategy):
+        console.print(token, end="")
+    console.print()
+
+
+async def _stream_branches(
+    prefix: str, model: str, n: int, max_tokens: int, temperature: float, strategy: str | None
+) -> None:
+    buffers: list[list[str]] = [[] for _ in range(n)]
+    console.print(f"[dim]{prefix}[/dim]\n")
+
+    async for idx, token in branch_text(
+        prefix, model, n=n, max_tokens=max_tokens, temperature=temperature, strategy=strategy
+    ):
+        buffers[idx].append(token)
+
+    panels = []
+    for i, buf in enumerate(buffers):
+        color = _BRANCH_COLORS[i % len(_BRANCH_COLORS)]
+        text = Text(prefix, style="dim")
+        text.append("".join(buf), style=color)
+        panels.append(Panel(text, title=f"[{color}]Branch {i + 1}[/{color}]"))
+
+    console.print(Columns(panels, equal=True))
+
+
+@app.command()
+def models(
+    provider: Annotated[str | None, typer.Option("-p", "--provider")] = None,
+    search: Annotated[str | None, typer.Option("-s", "--search")] = None,
+    available: Annotated[bool, typer.Option("-a", "--available", help="Only show models with keys set")] = False,
+) -> None:
+    """List available models."""
+    results = list_models(provider=provider, search=search, available_only=available)
+    if not results:
+        console.print("[yellow]No models found.[/yellow]")
+        return
+
+    table = Table("Model", show_header=True, header_style="bold")
+    for m in results:
+        table.add_row(m)
+    console.print(table)
+
+
+@app.command()
+def providers() -> None:
+    """List all known providers."""
+    for p in list_providers():
+        console.print(p)
+
+
+@app.command()
+def strategies() -> None:
+    """List available continuation strategies."""
+    table = Table("Name", "Description", show_header=True, header_style="bold")
+    descriptions = {
+        "completion": "OpenAI /completions endpoint — for true base models",
+        "prefill": "Anthropic assistant prefill trick",
+        "system": "System prompt coercion — generic fallback for any chat model",
+        "few_shot": "Few-shot examples in system prompt — for stubborn models",
+        "fim": "Fill-in-the-middle tokens — DeepSeek, StarCoder, CodeLlama",
+    }
+    for name in REGISTRY:
+        table.add_row(name, descriptions.get(name, ""))
+    console.print(table)
+
+
+@app.command()
+def info(model: Annotated[str, typer.Argument(help="Model name to inspect")]) -> None:
+    """Show which strategy would be used for a given model."""
+    strat = detect_strategy(model)
+    console.print(f"[bold]{model}[/bold] → [green]{strat.name}[/green]")

basemode-0.1.0/src/basemode/continue_.py

@@ -0,0 +1,56 @@
+import asyncio
+from collections.abc import AsyncGenerator
+
+from .detect import detect_strategy, normalize_model
+from .params import GenerationParams
+
+
+async def continue_text(
+    prefix: str,
+    model: str = "gpt-4o-mini",
+    *,
+    max_tokens: int = 200,
+    temperature: float = 0.9,
+    strategy: str | None = None,
+    **extra,
+) -> AsyncGenerator[str, None]:
+    """Stream a single continuation."""
+    model = normalize_model(model)
+    params = GenerationParams(model=model, max_tokens=max_tokens, temperature=temperature, extra=extra)
+    strat = detect_strategy(model, strategy)
+    async for token in strat.stream(prefix, params):
+        yield token
+
+
+async def branch_text(
+    prefix: str,
+    model: str = "gpt-4o-mini",
+    *,
+    n: int = 4,
+    max_tokens: int = 200,
+    temperature: float = 0.9,
+    strategy: str | None = None,
+    **extra,
+) -> AsyncGenerator[tuple[int, str], None]:
+    """Stream n parallel continuations as (branch_idx, token) tuples."""
+    model = normalize_model(model)
+    params = GenerationParams(model=model, max_tokens=max_tokens, temperature=temperature, extra=extra)
+    strat = detect_strategy(model, strategy)
+
+    queue: asyncio.Queue[tuple[int, str] | None] = asyncio.Queue()
+
+    async def run_branch(idx: int) -> None:
+        async for token in strat.stream(prefix, params):
+            await queue.put((idx, token))
+        await queue.put(None)
+
+    tasks = [asyncio.create_task(run_branch(i)) for i in range(n)]
+    done = 0
+    while done < n:
+        item = await queue.get()
+        if item is None:
+            done += 1
+        else:
+            yield item
+
+    await asyncio.gather(*tasks)

basemode-0.1.0/src/basemode/detect.py

@@ -0,0 +1,64 @@
+from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
+
+from .strategies import (
+    REGISTRY,
+    CompletionStrategy,
+    ContinuationStrategy,
+    FIMStrategy,
+    PrefillStrategy,
+    SystemPromptStrategy,
+)
+
+# Models that use the native completions API
+_COMPLETION_MODELS = {
+    "gpt-3.5-turbo-instruct",
+    "davinci-002",
+    "babbage-002",
+}
+_COMPLETION_SUBSTRINGS = ["text-davinci", "text-curie", "text-babbage", "text-ada"]
+
+# Models where FIM is the right move
+_FIM_SUBSTRINGS = ["deepseek-coder", "starcoder", "codellama", "fim"]
+
+# Provider prefix to add when litellm can't auto-detect from model name alone
+_PREFIX_MAP = {
+    "claude": "anthropic",
+    "gemini": "gemini",
+    "command": "cohere",
+}
+
+
+def normalize_model(model: str) -> str:
+    """Add provider prefix if litellm can't resolve the model name."""
+    if "/" in model:
+        return model
+    try:
+        get_llm_provider(model)
+        return model
+    except Exception:
+        m = model.lower()
+        for fragment, provider in _PREFIX_MAP.items():
+            if fragment in m:
+                return f"{provider}/{model}"
+        return model
+
+
+def detect_strategy(model: str, override: str | None = None) -> ContinuationStrategy:
+    if override:
+        if override not in REGISTRY:
+            valid = ", ".join(REGISTRY)
+            raise ValueError(f"Unknown strategy {override!r}. Valid: {valid}")
+        return REGISTRY[override]()
+
+    m = model.lower()
+
+    if "claude" in m:
+        return PrefillStrategy()
+
+    if model in _COMPLETION_MODELS or any(s in m for s in _COMPLETION_SUBSTRINGS):
+        return CompletionStrategy()
+
+    if any(s in m for s in _FIM_SUBSTRINGS):
+        return FIMStrategy()
+
+    return SystemPromptStrategy()

basemode-0.1.0/src/basemode/models.py

@@ -0,0 +1,28 @@
+import litellm
+
+from .settings import settings
+
+
+def list_models(
+    provider: str | None = None,
+    search: str | None = None,
+    available_only: bool = False,
+) -> list[str]:
+    by_provider: dict[str, list[str]] = litellm.models_by_provider
+
+    if available_only:
+        providers = settings.available_providers
+        models = [m for p in providers for m in by_provider.get(p, [])]
+    elif provider:
+        models = by_provider.get(provider, [])
+    else:
+        models = [m for ms in by_provider.values() for m in ms]
+
+    if search:
+        models = [m for m in models if search.lower() in m.lower()]
+
+    return sorted(set(models))
+
+
+def list_providers() -> list[str]:
+    return sorted(litellm.models_by_provider.keys())

basemode-0.1.0/src/basemode/params.py

@@ -0,0 +1,9 @@
+from dataclasses import dataclass, field
+
+
+@dataclass
+class GenerationParams:
+    model: str
+    max_tokens: int = 200
+    temperature: float = 0.9
+    extra: dict = field(default_factory=dict)