doctracker 0.1.0__py3-none-any.whl

doctracker/__init__.py

@@ -0,0 +1,5 @@
+"""doctracker: centralized documentation for Python projects developed with LLM agents."""
+
+from importlib.metadata import version
+
+__version__ = version("doctracker")

doctracker/__main__.py

@@ -0,0 +1,4 @@
+from doctracker.cli import main
+
+if __name__ == "__main__":
+    main()

doctracker/cli.py

@@ -0,0 +1,55 @@
+"""Command line interface: `doctracker init` and `doctracker start`."""
+
+from pathlib import Path
+
+import click
+
+from doctracker import __version__
+from doctracker.config import CONFIG_PATH, Config, detect_packages, load_config, write_config
+
+
+@click.group()
+@click.version_option(__version__, prog_name="doctracker")
+def main() -> None:
+    """Centralized documentation for Python projects developed with LLM agents."""
+
+
+@main.command()
+@click.option("--force", is_flag=True, help="Overwrite an existing configuration.")
+def init(force: bool) -> None:
+    """Create doc/config/config.json in the current project."""
+    project_root = Path.cwd()
+    if (project_root / CONFIG_PATH).exists() and not force:
+        raise click.ClickException(f"{CONFIG_PATH.as_posix()} already exists (use --force to overwrite).")
+    _initialize(project_root)
+
+
+@main.command()
+@click.option("--host", help="Override the host from the configuration.")
+@click.option("--port", type=int, help="Override the port from the configuration.")
+def start(host: str | None, port: int | None) -> None:
+    """Start the server (MCP endpoint, wiki and watcher) for the current project."""
+    from doctracker.server.app import run
+
+    project_root = Path.cwd()
+    if not (project_root / CONFIG_PATH).exists():
+        _initialize(project_root)
+    config = load_config(project_root)
+    if host:
+        config.host = host
+    if port:
+        config.port = port
+    if not config.packages:
+        raise click.ClickException(f"No packages to document. Add them to {CONFIG_PATH.as_posix()}.")
+    scheme = "https" if config.ssl_certfile else "http"
+    click.echo(f"Wiki: {scheme}://{config.host}:{config.port}/")
+    click.echo(f"MCP:  {scheme}://{config.host}:{config.port}/mcp")
+    run(project_root, config)
+
+
+def _initialize(project_root: Path) -> None:
+    config = Config()
+    config.packages = detect_packages(project_root, config.search_paths)
+    write_config(project_root, config)
+    found = ", ".join(config.packages) or "none (edit the file to add them)"
+    click.echo(f"Created {CONFIG_PATH.as_posix()}. Packages: {found}")

doctracker/config.py

@@ -0,0 +1,65 @@
+"""Load and write the doctracker configuration (`doc/config/config.json`)."""
+
+import json
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+
+DOC_DIR = "doc"
+CONFIG_PATH = Path(DOC_DIR) / "config" / "config.json"
+GITIGNORE_PATH = Path(DOC_DIR) / ".gitignore"
+GITIGNORE_CONTENT = "sqlite/\nmodels/\n"
+
+
+@dataclass
+class Config:
+    """Settings read from `doc/config/config.json`."""
+
+    packages: list[str] = field(default_factory=list)
+    search_paths: list[str] = field(default_factory=lambda: ["src", "."])
+    host: str = "127.0.0.1"
+    port: int = 8765
+    docstring_style: str = "google"
+    show_private: bool = True
+    code_theme: str = "default"
+    ssl_certfile: str | None = None
+    ssl_keyfile: str | None = None
+    watch_interval: float = 1.0
+
+
+def load_config(project_root: Path) -> Config:
+    """Read the config file of a project, filling missing keys with defaults."""
+    path = project_root / CONFIG_PATH
+    data = json.loads(path.read_text(encoding="utf-8"))
+    known = Config.__dataclass_fields__
+    return Config(**{key: value for key, value in data.items() if key in known})
+
+
+def write_config(project_root: Path, config: Config) -> None:
+    """Write the config file and the `doc/.gitignore`."""
+    path = project_root / CONFIG_PATH
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(asdict(config), indent=2) + "\n", encoding="utf-8")
+    (project_root / GITIGNORE_PATH).write_text(GITIGNORE_CONTENT, encoding="utf-8")
+
+
+def update_config_value(project_root: Path, key: str, value) -> None:
+    """Change one key in the config file, keeping everything else as it is on disk."""
+    path = project_root / CONFIG_PATH
+    data = json.loads(path.read_text(encoding="utf-8"))
+    data[key] = value
+    path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
+
+
+def detect_packages(project_root: Path, search_paths: list[str]) -> list[str]:
+    """Find top-level Python packages (folders with `__init__.py`) in the search paths."""
+    packages = []
+    for search_path in search_paths:
+        base = project_root / search_path
+        if not base.is_dir():
+            continue
+        for child in sorted(base.iterdir()):
+            if child.name.startswith(".") or child.name == DOC_DIR:
+                continue
+            if (child / "__init__.py").is_file() and child.name not in packages:
+                packages.append(child.name)
+    return packages

doctracker/extraction/__init__.py

@@ -0,0 +1,5 @@
+"""Extraction of technical data from the project's code."""
+
+from doctracker.extraction.griffe_loader import extract_project
+
+__all__ = ["extract_project"]

doctracker/extraction/griffe_loader.py

@@ -0,0 +1,148 @@
+"""Read the technical structure of a project with griffe and turn it into plain data.
+
+The result only holds built-in types (dicts, lists, strings), so it can cross
+process boundaries and be stored later without depending on griffe objects.
+"""
+
+import logging
+from pathlib import Path
+
+import griffe
+
+# Members griffe can report for each kind of object, in display order.
+MEMBER_KINDS = ("module", "class", "function", "attribute", "type alias")
+
+
+def extract_project(
+    project_root: str,
+    packages: list[str],
+    search_paths: list[str],
+    docstring_style: str,
+    show_private: bool,
+) -> dict:
+    """Load every package and return `{"roots": [...], "entities": {path: entity}}`."""
+    logging.getLogger("griffe").setLevel(logging.ERROR)
+    root = Path(project_root)
+    paths = [str(root / path) for path in search_paths]
+    entities: dict[str, dict] = {}
+    roots = []
+    errors = []
+    for package in packages:
+        try:
+            module = griffe.load(
+                package,
+                search_paths=paths,
+                docstring_parser=docstring_style,
+                allow_inspection=False,
+            )
+        except (griffe.GriffeError, ImportError, OSError) as error:
+            errors.append(f"{package}: {error}")
+            continue
+        roots.append(module.path)
+        _collect(module, root, docstring_style, show_private, entities)
+    return {"roots": roots, "entities": entities, "errors": errors}
+
+
+def _collect(obj, root: Path, style: str, show_private: bool, entities: dict) -> None:
+    """Add `obj` and its members (recursively) to `entities`."""
+    entity = _to_entity(obj, root, style)
+    members = []
+    for member in obj.members.values():
+        if member.is_alias:
+            continue
+        if member.is_private and not show_private:
+            continue
+        if member.kind.value not in MEMBER_KINDS:
+            continue
+        members.append(member.path)
+        _collect(member, root, style, show_private, entities)
+    entity["members"] = members
+    entities[obj.path] = entity
+
+
+def _to_entity(obj, root: Path, style: str) -> dict:
+    """Convert one griffe object to a plain dict."""
+    entity = {
+        "path": obj.path,
+        "name": obj.name,
+        "kind": obj.kind.value,
+        "parent": obj.parent.path if obj.parent else None,
+        "filepath": _relative(obj.filepath, root),
+        "lineno": obj.lineno,
+        "endlineno": obj.endlineno,
+        "is_private": obj.is_private,
+        "docstring": obj.docstring.value if obj.docstring else "",
+        "sections": _sections(obj.docstring, style),
+        "decorators": [str(decorator.value) for decorator in getattr(obj, "decorators", [])],
+    }
+    if obj.is_function:
+        entity["parameters"] = [_parameter(parameter) for parameter in obj.parameters]
+        entity["returns"] = str(obj.returns) if obj.returns else ""
+    if obj.is_class:
+        entity["bases"] = [str(base) for base in obj.bases]
+        init = obj.members.get("__init__")
+        params = init.parameters if init is not None and init.is_function else []
+        entity["parameters"] = [_parameter(parameter) for parameter in params if parameter.name != "self"]
+    if obj.is_attribute:
+        entity["annotation"] = str(obj.annotation) if obj.annotation else ""
+        entity["value"] = str(obj.value) if obj.value is not None else ""
+    if obj.is_module:
+        entity["exports"] = [str(name) for name in obj.exports] if obj.exports else []
+        entity["is_package"] = entity["filepath"].endswith("__init__.py") or isinstance(obj.filepath, list)
+    return entity
+
+
+def _parameter(parameter) -> dict:
+    kind = parameter.kind.value
+    # griffe fills `*args`/`**kwargs` with `()`/`{}`, which are not real defaults.
+    variadic = kind in ("variadic positional", "variadic keyword")
+    return {
+        "name": parameter.name,
+        "annotation": str(parameter.annotation) if parameter.annotation else "",
+        "default": str(parameter.default) if parameter.default is not None and not variadic else "",
+        "kind": kind,
+    }
+
+
+def _sections(docstring, style: str) -> list[dict]:
+    """Parse a docstring into a list of `{"kind", "title", "text", "items"}` sections."""
+    if docstring is None:
+        return []
+    sections = []
+    for section in docstring.parse(style):
+        value = section.value
+        entry = {"kind": section.kind.value, "title": section.title or "", "text": "", "items": []}
+        if isinstance(value, str):
+            entry["text"] = value
+        elif isinstance(value, list):
+            for item in value:
+                if isinstance(item, tuple):  # examples: (kind, text), kind is "text" or "examples" (code)
+                    entry["items"].append(
+                        {"name": "", "annotation": "", "description": item[1], "is_code": item[0].value == "examples"}
+                    )
+                else:
+                    entry["items"].append(_section_item(item))
+        else:  # admonition, deprecated
+            entry["text"] = getattr(value, "description", "") or ""
+            entry["title"] = entry["title"] or str(getattr(value, "annotation", "") or "")
+        sections.append(entry)
+    return sections
+
+
+def _section_item(item) -> dict:
+    annotation = getattr(item, "annotation", None)
+    return {
+        "name": getattr(item, "name", "") or "",
+        "annotation": str(annotation) if annotation else "",
+        "description": getattr(item, "description", "") or "",
+    }
+
+
+def _relative(filepath, root: Path) -> str:
+    """Path of the source file relative to the project root, with forward slashes."""
+    if isinstance(filepath, list):  # namespace packages have several folders
+        filepath = filepath[0]
+    try:
+        return Path(filepath).relative_to(root).as_posix()
+    except ValueError:
+        return Path(filepath).as_posix()

doctracker/server/__init__.py

@@ -0,0 +1 @@
+"""MCP server and the services it provides (frontend, watcher)."""

doctracker/server/app.py

@@ -0,0 +1,78 @@
+"""Build the doctracker server: MCP endpoint at `/mcp`, wiki frontend and watcher."""
+
+import asyncio
+import contextlib
+from pathlib import Path
+
+import uvicorn
+from mcp.server import MCPServer
+from mcp.server.transport_security import TransportSecuritySettings
+from starlette.applications import Starlette
+
+from doctracker import __version__
+from doctracker.config import Config
+from doctracker.server.state import DocState
+from doctracker.server.tools import register_tools
+from doctracker.server.watcher import watch
+from doctracker.server.web import web_routes
+
+LOCAL_HOSTS = ("127.0.0.1", "localhost", "::1")
+
+
+def create_app(project_root: Path, config: Config) -> Starlette:
+    """Create the ASGI app with every service the server provides."""
+    state = DocState()
+    mcp = MCPServer(
+        name="doctracker",
+        version=__version__,
+        instructions=(
+            "Technical documentation of this project, built from its code. "
+            "Use list_entities to explore and get_entity for details. "
+            "Before writing or changing docstrings, call documentation_guide (also available as the "
+            "doctracker://guide resource) and follow it."
+        ),
+    )
+    register_tools(mcp, state, config)
+
+    app = mcp.streamable_http_app(host=config.host, transport_security=_transport_security(config))
+    app.router.routes.extend(web_routes(state, project_root, config))
+
+    mcp_lifespan = app.router.lifespan_context
+
+    @contextlib.asynccontextmanager
+    async def lifespan(starlette_app: Starlette):
+        async with mcp_lifespan(starlette_app) as lifespan_state:
+            watcher = asyncio.create_task(watch(project_root, config, state))
+            try:
+                yield lifespan_state
+            finally:
+                watcher.cancel()
+                with contextlib.suppress(asyncio.CancelledError):
+                    await watcher
+
+    app.router.lifespan_context = lifespan
+    return app
+
+
+def _transport_security(config: Config) -> TransportSecuritySettings | None:
+    """Host/Origin validation. The SDK already protects local hosts; other hosts are allowed explicitly."""
+    if config.host in LOCAL_HOSTS:
+        return None
+    scheme = "https" if config.ssl_certfile else "http"
+    return TransportSecuritySettings(
+        enable_dns_rebinding_protection=True,
+        allowed_hosts=[f"{config.host}:*"],
+        allowed_origins=[f"{scheme}://{config.host}:*"],
+    )
+
+
+def run(project_root: Path, config: Config) -> None:
+    """Start the server and block until it stops."""
+    uvicorn.run(
+        create_app(project_root, config),
+        host=config.host,
+        port=config.port,
+        ssl_certfile=config.ssl_certfile,
+        ssl_keyfile=config.ssl_keyfile,
+        log_level="info",
+    )

doctracker/server/guide.md

@@ -0,0 +1,62 @@
+# How to document code for doctracker
+
+doctracker builds the technical reference of this project by reading the code
+statically (it never imports or runs it). Everything shown in the wiki comes from
+the code itself: names, signatures, type hints, decorators and docstrings. Follow
+these rules so every entity gets a complete page.
+
+## Settings of this project
+
+- Docstring style: **{docstring_style}** (`docstring_style` in `doc/config/config.json`).
+- Private members (`_name`): **{private_members}** (`show_private`).
+
+## What to document
+
+1. **Every module and package** starts with a module docstring: one summary line,
+   then a short paragraph saying what the module is for. A package's docstring goes
+   in its `__init__.py`.
+2. **Every public class, function and method** has a docstring.
+3. **Type hints on every parameter and return value.** The wiki shows them in the
+   signature and in the parameters table; do not repeat types inside the docstring
+   unless the hint is missing.
+4. **Module-level constants and class attributes** get a docstring written as a
+   string literal on the line right after the assignment:
+
+   ```python
+   DEFAULT_UNITS: Units = Units.METRIC
+   """Units used when the caller does not choose any."""
+   ```
+
+   Instance attributes set in `__init__` (and dataclass fields) are documented in
+   the class docstring's `Attributes` section instead.
+5. **Public API of a package:** list it in `__all__`; the wiki shows it on the
+   package page.
+
+## Docstring layout
+
+- First line: one sentence, imperative mood for functions ("Fetch the forecast…"),
+  ending with a period. It is used as the summary in lists and search results.
+- Blank line, then any longer explanation (Markdown is rendered).
+- Then the sections below, in this order. Leave out the sections that do not apply.
+
+| Section | When |
+|---|---|
+| Args | every parameter, including `*args` and `**kwargs` |
+| Returns / Yields | what is returned or yielded, and what it means |
+| Raises | every exception the caller should expect, and when |
+| Attributes | instance attributes and dataclass fields (class docstrings) |
+| Examples | short `>>>` sessions; they are shown as code |
+| Note / Warning | anything the caller must not miss |
+
+## Example in the {docstring_style} style
+
+```python
+{example}
+```
+
+## Things to avoid
+
+- Docstrings that only repeat the name ("Get weather." for `get_weather`).
+- Describing the implementation instead of the behavior; the wiki already shows the source.
+- Parameters missing from `Args`: the table shows "-" in their description.
+- Raw HTML in docstrings: it is escaped, not rendered.

doctracker/server/guide.py

@@ -0,0 +1,68 @@
+"""The documentation guide agents read through the MCP server."""
+
+from importlib.resources import files
+
+from doctracker.config import Config
+
+EXAMPLES = {
+    "google": '''def get_weather(city: str, units: str = "metric") -> Forecast:
+    """Fetch the forecast for a city.
+
+    Args:
+        city: City name, e.g. `Lisbon`.
+        units: `metric` or `imperial`.
+
+    Returns:
+        Forecast data for the city.
+
+    Raises:
+        CityNotFound: When the city is unknown.
+
+    Examples:
+        >>> get_weather("Lisbon").temperature
+        21.5
+    """''',
+    "numpy": '''def get_weather(city: str, units: str = "metric") -> Forecast:
+    """Fetch the forecast for a city.
+
+    Parameters
+    ----------
+    city
+        City name, e.g. `Lisbon`.
+    units
+        `metric` or `imperial`.
+
+    Returns
+    -------
+    Forecast
+        Forecast data for the city.
+
+    Raises
+    ------
+    CityNotFound
+        When the city is unknown.
+
+    Examples
+    --------
+    >>> get_weather("Lisbon").temperature
+    21.5
+    """''',
+    "sphinx": '''def get_weather(city: str, units: str = "metric") -> Forecast:
+    """Fetch the forecast for a city.
+
+    :param city: City name, e.g. `Lisbon`.
+    :param units: `metric` or `imperial`.
+    :return: Forecast data for the city.
+    :raises CityNotFound: When the city is unknown.
+    """''',
+}
+
+
+def documentation_guide(config: Config) -> str:
+    """The guide in Markdown, filled in with this project's settings."""
+    template = files("doctracker.server").joinpath("guide.md").read_text(encoding="utf-8")
+    return template.format(
+        docstring_style=config.docstring_style,
+        private_members="shown in the wiki" if config.show_private else "hidden from the wiki",
+        example=EXAMPLES.get(config.docstring_style, EXAMPLES["google"]),
+    )

doctracker/server/state.py

@@ -0,0 +1,32 @@
+"""In-memory documentation data shared by the MCP tools and the frontend."""
+
+import asyncio
+from dataclasses import dataclass, field
+
+
+@dataclass
+class DocState:
+    """Latest extraction result plus the open frontend connections to notify."""
+
+    roots: list[str] = field(default_factory=list)
+    entities: dict[str, dict] = field(default_factory=dict)
+    errors: list[str] = field(default_factory=list)
+    ready: bool = False
+    listeners: set[asyncio.Queue] = field(default_factory=set)
+
+    def update(self, result: dict) -> None:
+        """Replace the data with a new extraction result and notify the frontend."""
+        self.roots = result["roots"]
+        self.entities = result["entities"]
+        self.errors = result["errors"]
+        self.ready = True
+        for queue in self.listeners:
+            queue.put_nowait("changed")
+
+    def subscribe(self) -> asyncio.Queue:
+        queue: asyncio.Queue = asyncio.Queue()
+        self.listeners.add(queue)
+        return queue
+
+    def unsubscribe(self, queue: asyncio.Queue) -> None:
+        self.listeners.discard(queue)

doctracker/server/tools.py

@@ -0,0 +1,76 @@
+"""MCP tools that let agents read the technical documentation."""
+
+from mcp.server import MCPServer
+
+from doctracker.config import Config
+from doctracker.server.guide import documentation_guide as build_guide
+from doctracker.server.state import DocState
+
+GUIDE_URI = "doctracker://guide"
+
+
+def register_tools(mcp: MCPServer, state: DocState, config: Config) -> None:
+    """Add the documentation tools and the documentation guide to the MCP server."""
+
+    @mcp.tool()
+    def documentation_guide() -> str:
+        """Explain how to document this project's code so doctracker builds complete pages.
+
+        Read it before writing or changing docstrings, module docstrings, type hints or `__all__`.
+
+        Returns:
+            The guide in Markdown, adapted to this project's settings (docstring style, private members).
+        """
+        return build_guide(config)
+
+    @mcp.resource(GUIDE_URI, name="documentation-guide", title="How to document code for doctracker",
+                  mime_type="text/markdown")
+    def guide_resource() -> str:
+        """How to document this project's code so doctracker builds complete pages."""
+        return build_guide(config)
+
+    @mcp.tool()
+    def list_entities(prefix: str = "", kind: str = "") -> list[dict]:
+        """List documented entities.
+
+        Args:
+            prefix: Only entities whose qualified path starts with this text (e.g. `mypkg.api`).
+            kind: Only entities of this kind: `module`, `class`, `function`, `attribute` or `type alias`.
+
+        Returns:
+            One item per entity with its path, kind and the first line of its docstring.
+        """
+        items = []
+        for path, entity in sorted(state.entities.items()):
+            if prefix and not path.startswith(prefix):
+                continue
+            if kind and entity["kind"] != kind:
+                continue
+            summary = entity["docstring"].strip().split("\n", 1)[0] if entity["docstring"] else ""
+            items.append({"path": path, "kind": entity["kind"], "summary": summary})
+        return items
+
+    @mcp.tool()
+    def get_entity(path: str) -> dict:
+        """Get the full technical reference of one entity.
+
+        Args:
+            path: Qualified path of the entity (e.g. `mypkg.api.get_weather`).
+
+        Returns:
+            Signature data, parameters, docstring sections, location in the code and members.
+        """
+        entity = state.entities.get(path)
+        if entity is None:
+            raise ValueError(f"Unknown entity: {path}")
+        return entity
+
+    @mcp.tool()
+    def status() -> dict:
+        """Report whether the documentation is built and any loading errors."""
+        return {
+            "ready": state.ready,
+            "packages": state.roots,
+            "entities": len(state.entities),
+            "errors": state.errors,
+        }

doctracker/server/watcher.py

@@ -0,0 +1,63 @@
+"""Watch the project's Python files and rebuild the documentation when they change.
+
+Change detection polls file modification times (standard library only). The
+extraction itself runs in a separate process so the server stays responsive.
+"""
+
+import asyncio
+from concurrent.futures import ProcessPoolExecutor
+from pathlib import Path
+
+from doctracker.config import DOC_DIR, Config
+from doctracker.extraction import extract_project
+from doctracker.server.state import DocState
+
+SKIPPED_DIRS = {DOC_DIR, ".git", ".venv", "venv", "__pycache__", "node_modules", ".mypy_cache", ".ruff_cache"}
+
+
+def snapshot(project_root: Path, search_paths: list[str]) -> dict[str, int]:
+    """Map every watched `.py` file to its modification time."""
+    files = {}
+    for search_path in search_paths:
+        base = project_root / search_path
+        if base.is_dir():
+            _scan(base, files)
+    return files
+
+
+def _scan(folder: Path, files: dict[str, int]) -> None:
+    try:
+        entries = list(folder.iterdir())
+    except OSError:
+        return
+    for entry in entries:
+        if entry.is_dir():
+            if entry.name not in SKIPPED_DIRS and not entry.name.startswith("."):
+                _scan(entry, files)
+        elif entry.suffix == ".py":
+            try:
+                files[str(entry)] = entry.stat().st_mtime_ns
+            except OSError:
+                pass
+
+
+async def watch(project_root: Path, config: Config, state: DocState) -> None:
+    """Build the documentation once, then rebuild it whenever a watched file changes."""
+    loop = asyncio.get_running_loop()
+    with ProcessPoolExecutor(max_workers=1) as executor:
+        previous = None
+        while True:
+            current = await loop.run_in_executor(executor, snapshot, project_root, config.search_paths)
+            if current != previous:
+                result = await loop.run_in_executor(
+                    executor,
+                    extract_project,
+                    str(project_root),
+                    config.packages,
+                    config.search_paths,
+                    config.docstring_style,
+                    config.show_private,
+                )
+                state.update(result)
+                previous = current
+            await asyncio.sleep(config.watch_interval)