colloquial 0.1.0__py3-none-any.whl

colloquial/__init__.py

@@ -0,0 +1,3 @@
+"""Colloquial — rewrite machine text into colloquial, native-sounding language."""
+
+__version__ = "0.1.0"

colloquial/__main__.py

@@ -0,0 +1,4 @@
+from colloquial.cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

colloquial/cli.py

@@ -0,0 +1,500 @@
+"""Colloquial CLI.
+
+Commands:
+  colloquial detect <path> [--lang LANG] [--report] [--format text|json]
+  colloquial humanize <path> [--lang] [--to LANG] [--self-check] [--report]
+  colloquial providers             -- list OpenAI-compatible providers
+  colloquial models                -- live model list from your configured provider
+  colloquial setup                 -- configure provider + key + model (interactive)
+  colloquial config                -- show the resolved configuration
+  colloquial langs                 -- list available language profiles
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any, TypedDict, cast
+
+from colloquial import config as cfg
+from colloquial.core import surface
+from colloquial.core.detect import detect, prepare_text
+from colloquial.core.langdetect import guess_language
+from colloquial.core.report import format_report, score_document
+from colloquial.providers import PROVIDERS, requires_key
+from colloquial.rewrite.llm import LLMClient, LLMError
+
+_TEXT_SUFFIXES = {".md", ".mdx", ".markdown", ".txt"}
+
+
+class DetectResult(TypedDict):
+    path: str
+    language: str
+    words: int
+    sentences: int
+    ai_score: float
+    humanity_score: float
+    findings: int
+    burstiness: float
+    mattr: float
+    findings_detail: list[dict[str, Any]]
+
+
+def _read(path: str) -> str:
+    p = Path(path)
+    if p.is_dir():
+        files = (
+            f.read_text(encoding="utf-8", errors="replace")
+            for f in sorted(p.rglob("*"))
+            if f.suffix in _TEXT_SUFFIXES
+        )
+        return "\n\n".join(files)
+    return p.read_text(encoding="utf-8", errors="replace")
+
+
+def _detect_one(path: str, lang: str, fmt: str) -> DetectResult:
+    raw = _read(path)
+    det = detect(raw, language=lang)
+    ai, humanity = score_document(det)
+    result: DetectResult = {
+        "path": path,
+        "language": lang,
+        "words": det.text_stats.word_count,
+        "sentences": det.text_stats.sentence_count,
+        "ai_score": ai,
+        "humanity_score": humanity,
+        "findings": len(det.findings),
+        "burstiness": det.text_stats.burstiness,
+        "mattr": det.text_stats.mattr,
+        "findings_detail": [],
+    }
+    if fmt == "json":
+        result["findings_detail"] = [
+            {
+                "line": f.line,
+                "severity": f.severity,
+                "note": f.note,
+                "weight": f.weight,
+                "snippet": f.snippet[:80],
+            }
+            for f in det.findings[:20]
+        ]
+    return result
+
+
+def _cmd_detect(args: argparse.Namespace) -> int:
+    if args.lang == "auto":
+        # M0: default to pt-br when the text is mostly accented; else en.
+        # A real language detector lands with the profile work.
+        args.lang = _guess_lang(args.paths[0])
+    results = [_detect_one(p, args.lang, args.format) for p in _expand(args.paths)]
+    if args.format == "json":
+        print(json.dumps(results, ensure_ascii=False, indent=2))
+        return 0
+
+    for r in results:
+        print(
+            f"{r['path']}: ai={r['ai_score']}/100 "
+            f"humanity={r['humanity_score']}/100 "
+            f"({r['words']} words, {r['findings']} surface hits)"
+        )
+        if args.report:
+            raw = _read(r["path"])
+            det = detect(raw, language=r["language"])
+            print()
+            print(format_report(det, prepare_text(raw)))
+    return 0
+
+
+def _expand(paths: list[str]) -> list[str]:
+    out: list[str] = []
+    for p in paths:
+        path = Path(p)
+        if path.is_dir():
+            out.extend(
+                str(f)
+                for f in sorted(path.rglob("*"))
+                if f.suffix in _TEXT_SUFFIXES and ".git" not in f.parts
+            )
+        else:
+            out.append(str(path))
+    return out
+
+
+def _guess_lang(path: str) -> str:
+    raw = _read(path)[:2000]
+    return guess_language(raw)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="colloquial",
+        description=(
+            "Rewrite machine text into colloquial, native-sounding language — in any tongue."
+        ),
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    det = sub.add_parser("detect", help="score text for machine-likeness (deterministic, offline)")
+    det.add_argument("paths", nargs="+", help="files or directories (.md/.mdx/.txt)")
+    det.add_argument("--lang", default="auto", help="language profile (auto, en, pt-br, ...)")
+    det.add_argument("--report", action="store_true", help="print full humanization report")
+    det.add_argument("--format", choices=["text", "json"], default="text")
+    det.set_defaults(func=_cmd_detect)
+
+    langs = sub.add_parser("langs", help="list available language profiles")
+    langs.set_defaults(func=_cmd_langs)
+
+    prov = sub.add_parser("providers", help="list built-in OpenAI-compatible providers")
+    prov.set_defaults(func=_cmd_providers)
+
+    mod = sub.add_parser(
+        "models", help="live model list from your configured provider (GET /models)"
+    )
+    mod.add_argument("--provider", default=None, help="override the configured provider")
+    mod.set_defaults(func=_cmd_models)
+
+    setup = sub.add_parser(
+        "setup",
+        help="configure provider, API key and model (writes ~/.config/colloquial/config.toml)",
+    )
+    setup.add_argument("--provider", default=None, help="provider key from `colloquial providers`")
+    setup.add_argument("--base-url", default=None, help="override the provider base URL")
+    setup.add_argument("--model", default=None, help="model id (skips the live list)")
+    setup.add_argument(
+        "--api-key",
+        default=None,
+        help="API key (prefer the interactive prompt; shells keep history)",
+    )
+    setup.set_defaults(func=_cmd_setup)
+
+    conf = sub.add_parser("config", help="show the resolved configuration (key masked)")
+    conf.set_defaults(func=_cmd_config)
+
+    mcp = sub.add_parser(
+        "mcp",
+        help="run the MCP server over stdio (Model Context Protocol; tools for any agent)",
+    )
+    mcp.set_defaults(func=_cmd_mcp)
+
+    hum = sub.add_parser(
+        "humanize",
+        help="rewrite text so it reads human (same language), on your configured provider",
+    )
+    hum.add_argument("paths", nargs="+", help="files or directories (.md/.mdx/.txt)")
+    hum.add_argument("--lang", default="auto", help="language profile (auto, en, pt-br, ar, ...)")
+    hum.add_argument(
+        "--to",
+        default=None,
+        help="translate-and-humanize into this language in one pass",
+    )
+    hum.add_argument(
+        "--from",
+        dest="from_lang",
+        default=None,
+        help="source language for --to (default: auto-detected)",
+    )
+    hum.add_argument(
+        "--self-check",
+        action="store_true",
+        help="re-detect the output and iterate once more if it still reads machine-made",
+    )
+    hum.add_argument(
+        "--report", action="store_true", help="print before/after scores and applied rewrites"
+    )
+    hum.add_argument("--out", default=None, help="write output to file instead of stdout")
+    hum.set_defaults(func=_cmd_humanize)
+
+    return parser
+
+
+def _cmd_langs(args: argparse.Namespace) -> int:
+    for code in surface.available():
+        prof = surface.get_profile(code)
+        print(f"{code:8} {prof.name if prof else ''}")
+    return 0
+
+
+def _cmd_providers(args: argparse.Namespace) -> int:
+    for key, p in PROVIDERS.items():
+        key_note = "" if p.requires_key else "  (no key needed)"
+        print(f"{key:12} {p.label:28} {p.base_url}{key_note}")
+        if p.note:
+            print(f"{'':12} {p.note}")
+    return 0
+
+
+def _cmd_models(args: argparse.Namespace) -> int:
+    try:
+        client = LLMClient(provider=args.provider)
+        models = client.list_models()
+    except LLMError as exc:
+        print(f"colloquial: error: {exc}", file=sys.stderr)
+        return 2
+    marker = client.model
+    for model in models:
+        current = "  <- configured" if model == marker else ""
+        print(f"{model}{current}")
+    print(f"\n{len(models)} models available from {client.base_url}")
+    return 0
+
+
+def _cmd_config(args: argparse.Namespace) -> int:
+    settings = cfg.resolve_llm_settings()
+    path = cfg.config_path()
+    print(f"config file : {path}{'' if path.exists() else '  (not created yet)'}")
+    print(f"provider    : {settings.provider or '(none)'}")
+    print(f"base_url    : {settings.base_url or '(none)'}")
+    print(f"model       : {settings.model or '(none)'}")
+    print(f"api key     : {cfg.mask_key(settings.api_key)}")
+    print(f"key needed  : {'yes' if settings.needs_key else 'no'}")
+    print(f"source      : {settings.source}")
+    return 0
+
+
+def _ask(prompt: str) -> str:
+    try:
+        return input(prompt).strip()
+    except EOFError:
+        return ""
+
+
+def _ask_key(prompt: str) -> str:
+    import getpass
+
+    try:
+        return getpass.getpass(prompt).strip()
+    except (EOFError, getpass.GetPassWarning):
+        return ""
+
+
+def _pick_model(client: LLMClient, catalog_default: str) -> str:
+    """List live models when possible and let the user choose one."""
+    try:
+        models = client.list_models()
+    except LLMError as exc:
+        print(f"  could not list models ({exc}).")
+        typed = _ask(f"  model id [{catalog_default or 'required'}]: ")
+        return typed or catalog_default
+    if not models:
+        return catalog_default
+    for i, model in enumerate(models[:40], 1):
+        print(f"  {i:3}. {model}")
+    if len(models) > 40:
+        print(f"  ... and {len(models) - 40} more")
+    choice = _ask(f"  model number or id [{catalog_default or '1'}]: ")
+    if not choice:
+        return catalog_default or models[0]
+    if choice.isdigit() and 1 <= int(choice) <= len(models[:40]):
+        return models[int(choice) - 1]
+    return choice
+
+
+def _cmd_setup(args: argparse.Namespace) -> int:
+    provider_key: str = args.provider or ""
+    base_url: str = args.base_url or ""
+    model: str = args.model or ""
+    api_key: str = args.api_key or ""
+
+    interactive = not provider_key
+    if interactive:
+        # Easiest first: providers that need no account at all, then the rest.
+        ordered = sorted(PROVIDERS.items(), key=lambda item: item[1].requires_key)
+        print("Providers (all OpenAI-compatible; no-key options first):\n")
+        for i, (key, provider) in enumerate(ordered, 1):
+            suffix = "  (no key needed)" if not provider.requires_key else ""
+            print(f"{i:3}. {key:12} {provider.label:28}{suffix}")
+        print(f"{len(ordered) + 1:3}. {'custom':12} any OpenAI-compatible endpoint")
+        choice = _ask("\nprovider number or name: ")
+        if choice.isdigit() and 1 <= int(choice) <= len(ordered):
+            provider_key = ordered[int(choice) - 1][0]
+        else:
+            provider_key = choice
+
+    if provider_key == "custom":
+        provider_key = ""
+        base_url = base_url or _ask("base_url (e.g. https://example.com/v1): ")
+    elif provider_key in PROVIDERS:
+        base_url = base_url or PROVIDERS[provider_key].base_url
+        model = model or PROVIDERS[provider_key].default_model
+    elif provider_key:
+        print(f"colloquial: error: unknown provider '{provider_key}'.", file=sys.stderr)
+        print("Run `colloquial providers` to see the catalog.", file=sys.stderr)
+        return 2
+
+    if not base_url:
+        print("colloquial: error: no base_url given; nothing to configure.", file=sys.stderr)
+        return 2
+
+    provider_catalog = PROVIDERS.get(provider_key)
+    needs_key = requires_key(provider_key, base_url)
+    if needs_key and not api_key:
+        if interactive:
+            api_key = _ask_key("API key (input hidden, leave empty to use env vars only): ")
+        if not api_key and not api_key_present_in_env():
+            print(
+                "colloquial: error: this provider needs an API key. "
+                "Pass --api-key, or set COLLOQUIAL_API_KEY later.",
+                file=sys.stderr,
+            )
+            return 2
+
+    if not model:
+        print("\nChecking the connection and fetching models...")
+        try:
+            client = LLMClient(
+                provider=provider_key or None,
+                base_url=base_url,
+                api_key=api_key or None,
+            )
+            model = _pick_model(client, provider_catalog.default_model if provider_catalog else "")
+            print("  connection ok.")
+        except LLMError as exc:
+            print(f"  could not connect ({exc}).")
+            print("  (the configuration is still saved; fix the connection later)")
+            model = _ask(f"model id [{model or 'required'}]: ") or model
+        if not model:
+            print("colloquial: error: no model selected.", file=sys.stderr)
+            return 2
+
+    path = cfg.save_llm(provider=provider_key, base_url=base_url, model=model, api_key=api_key)
+    print(f"\nSaved {path} (mode 0600).")
+    print(f"  provider : {provider_key or 'custom'}")
+    print(f"  base_url : {base_url}")
+    print(f"  model    : {model}")
+    print(f"  api key  : {cfg.mask_key(api_key)}")
+    print("\nTry it:  colloquial humanize README.md --report")
+    return 0
+
+
+def api_key_present_in_env() -> bool:
+    import os
+
+    return bool(os.environ.get("COLLOQUIAL_API_KEY") or os.environ.get("OPENAI_API_KEY"))
+
+
+def _cmd_mcp(args: argparse.Namespace) -> int:
+    from colloquial.mcp import serve
+
+    return serve()
+
+
+_NO_PROVIDER_CLI_HELP = (
+    "colloquial: error: no provider configured — rewriting needs a model.\n"
+    "  run `colloquial setup` (a local model such as Ollama needs no account), or\n"
+    "  set COLLOQUIAL_API_KEY with COLLOQUIAL_BASE_URL and COLLOQUIAL_MODEL.\n"
+    "  `colloquial detect` works without a provider."
+)
+
+
+def _provider_ready() -> bool:
+    from colloquial.config import resolve_llm_settings
+
+    settings = resolve_llm_settings()
+    return bool(settings.base_url) and (bool(settings.api_key) or not settings.needs_key)
+
+
+def _ensure_provider() -> bool:
+    """Guarantee a provider before rewriting; walk the user through setup once.
+
+    On a terminal the first `humanize` with nothing configured opens the setup
+    interactively, because there is no rewriting without a model. In a pipe or
+    cron there is nobody to answer prompts, so the caller gets a clear error and
+    a non-zero exit instead.
+    """
+    if _provider_ready():
+        return True
+    if not sys.stdin.isatty():
+        print(_NO_PROVIDER_CLI_HELP, file=sys.stderr)
+        return False
+    print("colloquial: no provider configured yet — rewriting needs a model.")
+    print("Let's set one up now (Ctrl-C to cancel).\n")
+    code = _cmd_setup(argparse.Namespace(provider=None, base_url=None, model=None, api_key=None))
+    if code != 0 or not _provider_ready():
+        print(_NO_PROVIDER_CLI_HELP, file=sys.stderr)
+        return False
+    print()
+    return True
+
+
+def _cmd_humanize(args: argparse.Namespace) -> int:
+    from colloquial.rewrite.engine import humanize_same_lang, humanize_translate
+
+    if not _ensure_provider():
+        return 2
+
+    if args.to:
+        for path in _expand(args.paths):
+            raw = _read(path)
+            result = humanize_translate(
+                raw,
+                to_lang=args.to,
+                from_lang=args.from_lang,
+                self_check=args.self_check,
+            )
+            if args.report:
+                print(f"== {path} ({result.note})")
+                print(f"   ai before: {result.before_ai}/100 → after: {result.after_ai}/100")
+                if result.applied:
+                    print(f"   rewrites: {', '.join(result.applied[:12])}")
+                print()
+            if args.out:
+                Path(args.out).write_text(result.output, encoding="utf-8")
+                print(f"wrote {args.out}")
+            else:
+                print(result.output)
+        return 0
+
+    if args.lang == "auto":
+        args.lang = _guess_lang(args.paths[0])
+    for path in _expand(args.paths):
+        raw = _read(path)
+        result = humanize_same_lang(
+            raw,
+            lang=args.lang,
+            self_check=args.self_check,
+        )
+        if args.report:
+            print(f"== {path} ({args.lang}, {result.note})")
+            print(f"   ai before: {result.before_ai}/100 → after: {result.after_ai}/100")
+            if result.applied:
+                print(f"   rewrites: {', '.join(result.applied[:12])}")
+            print()
+        if args.out:
+            Path(args.out).write_text(result.output, encoding="utf-8")
+            print(f"wrote {args.out}")
+        else:
+            print(result.output)
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    try:
+        args = parser.parse_args(argv)
+    except SystemExit as exc:
+        # argparse raises SystemExit(0) for --help/--version and a non-zero code
+        # for usage errors. Return the code as-is: turning the help exit (0)
+        # into an error would make `colloquial --help` fail in a script.
+        code = exc.code
+        if code is None:
+            return 2
+        return code if isinstance(code, int) else 2
+    func = cast(Callable[[argparse.Namespace], int] | None, getattr(args, "func", None))
+    if func is None:
+        parser.print_help()
+        return 2
+    try:
+        return func(args)
+    except KeyboardInterrupt:
+        return 130
+    except Exception as exc:
+        print(f"colloquial: error: {exc}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

colloquial/config.py

@@ -0,0 +1,192 @@
+"""User configuration for Colloquial's LLM mode.
+
+Settings live in a small TOML file written with mode 0600:
+
+    ~/.config/colloquial/config.toml
+
+overridable with ``$COLLOQUIAL_CONFIG`` (full path) or ``$XDG_CONFIG_HOME``.
+The file is hand-parsed with a minimal parser so Python 3.10 stays supported
+(stdlib ``tomllib`` only exists from 3.11). Only a flat ``[llm]`` section of
+string values is read or written; unknown lines are ignored, so editing the
+file by hand is safe.
+
+Resolution precedence, highest first:
+    explicit arguments > environment variables > config file > catalog defaults
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+from colloquial.providers import PROVIDERS, requires_key
+
+_SECTION = re.compile(r"^\[(?P<name>[A-Za-z0-9_.-]+)\]$")
+_KV = re.compile(r'^(?P<key>[A-Za-z0-9_-]+)\s*=\s*"(?P<value>.*)"\s*(?:#.*)?$')
+_ESCAPES = {"n": "\n", "t": "\t", '"': '"', "\\": "\\"}
+
+
+class ConfigError(RuntimeError):
+    """Raised when the configuration file is unreadable or invalid."""
+
+
+def config_path() -> Path:
+    """Where the config file lives (respects env overrides)."""
+    explicit = os.environ.get("COLLOQUIAL_CONFIG")
+    if explicit:
+        return Path(explicit)
+    xdg = os.environ.get("XDG_CONFIG_HOME")
+    base = Path(xdg) if xdg else Path.home() / ".config"
+    return base / "colloquial" / "config.toml"
+
+
+def _unescape(value: str) -> str:
+    out: list[str] = []
+    i = 0
+    while i < len(value):
+        char = value[i]
+        if char == "\\" and i + 1 < len(value):
+            out.append(_ESCAPES.get(value[i + 1], value[i + 1]))
+            i += 2
+        else:
+            out.append(char)
+            i += 1
+    return "".join(out)
+
+
+def _escape(value: str) -> str:
+    return value.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n")
+
+
+def load_llm() -> dict[str, str]:
+    """Read the ``[llm]`` section; missing file yields an empty dict."""
+    path = config_path()
+    if not path.exists():
+        return {}
+    try:
+        text = path.read_text(encoding="utf-8")
+    except OSError as exc:  # pragma: no cover - unreadable file edge
+        raise ConfigError(f"cannot read {path}: {exc}") from exc
+
+    section = ""
+    values: dict[str, str] = {}
+    for raw in text.splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#"):
+            continue
+        section_match = _SECTION.match(line)
+        if section_match:
+            section = section_match.group("name")
+            continue
+        kv_match = _KV.match(line)
+        if kv_match and section == "llm":
+            values[kv_match.group("key")] = _unescape(kv_match.group("value"))
+    return values
+
+
+def save_llm(
+    *,
+    provider: str,
+    base_url: str,
+    model: str,
+    api_key: str,
+) -> Path:
+    """Write the ``[llm]`` section with 0600 permissions; return the path."""
+    path = config_path()
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        lines = [
+            "# Colloquial configuration. Written by `colloquial setup`.",
+            "# The API key below is stored in plain text with mode 0600.",
+            "# Prefer `COLLOQUIAL_API_KEY` in the environment to keep it out of files.",
+            "",
+            "[llm]",
+            f'provider = "{_escape(provider)}"',
+        ]
+        if base_url:
+            lines.append(f'base_url = "{_escape(base_url)}"')
+        if model:
+            lines.append(f'model = "{_escape(model)}"')
+        if api_key:
+            lines.append(f'api_key = "{_escape(api_key)}"')
+        path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+        path.chmod(0o600)
+    except OSError as exc:
+        raise ConfigError(f"cannot write {path}: {exc}") from exc
+    return path
+
+
+def mask_key(key: str) -> str:
+    """Show only a short prefix/suffix so logs and screens stay safe."""
+    if not key:
+        return "(none)"
+    if len(key) <= 8:
+        return "***"
+    return f"{key[:4]}...{key[-4:]}"
+
+
+@dataclass(frozen=True)
+class LLMSettings:
+    """Fully resolved LLM settings plus where they came from."""
+
+    provider: str
+    base_url: str
+    model: str
+    api_key: str
+    source: str
+    needs_key: bool
+
+
+def resolve_llm_settings(
+    *,
+    provider: str | None = None,
+    base_url: str | None = None,
+    model: str | None = None,
+    api_key: str | None = None,
+) -> LLMSettings:
+    """Merge arguments, environment variables and the config file."""
+    file_values = load_llm()
+
+    resolved_provider = (
+        provider or os.environ.get("COLLOQUIAL_PROVIDER") or file_values.get("provider") or ""
+    )
+    catalog = PROVIDERS.get(resolved_provider)
+    resolved_base_url = (
+        base_url
+        or os.environ.get("COLLOQUIAL_BASE_URL")
+        or file_values.get("base_url")
+        or (catalog.base_url if catalog else "")
+    ).rstrip("/")
+    resolved_model = (
+        model
+        or os.environ.get("COLLOQUIAL_MODEL")
+        or file_values.get("model")
+        or (catalog.default_model if catalog else "")
+    )
+    resolved_key = (
+        api_key
+        or os.environ.get("COLLOQUIAL_API_KEY")
+        or os.environ.get("OPENAI_API_KEY")
+        or file_values.get("api_key")
+        or ""
+    )
+
+    if provider or os.environ.get("COLLOQUIAL_PROVIDER"):
+        source = "arguments/environment" if provider else "environment"
+    elif file_values:
+        source = "config file"
+    else:
+        # Nothing configured and no provider selected: the caller decides what
+        # to do about it (the CLI opens setup, the library raises).
+        source = "none"
+
+    return LLMSettings(
+        provider=resolved_provider,
+        base_url=resolved_base_url,
+        model=resolved_model,
+        api_key=resolved_key,
+        source=source,
+        needs_key=requires_key(resolved_provider, resolved_base_url),
+    )

colloquial/core/__init__.py

@@ -0,0 +1 @@
+"""Colloquial core — deterministic detection and scoring."""