docforge-gen 0.1.0__py3-none-any.whl

docforge/__init__.py

@@ -0,0 +1,3 @@
+"""DocForge - Automatically generate documentation for any GitHub repository using LLMs."""
+
+__version__ = "0.1.0"

docforge/analysis/__init__.py

@@ -0,0 +1,5 @@
+from .context_builder import RepoContext, CommitGroup, build_context
+from .repo_analyzer import analyze_repo
+from .dependency_detector import detect_dependencies
+
+__all__ = ["RepoContext", "CommitGroup", "build_context", "analyze_repo", "detect_dependencies"]

docforge/analysis/context_builder.py

@@ -0,0 +1,131 @@
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from ..parsing.base_parser import ParsedFile
+
+
+@dataclass
+class CommitGroup:
+    tag: str  # "v1.2.0" or "Unreleased"
+    date: str
+    commits: list[dict] = field(default_factory=list)
+
+
+@dataclass
+class RepoContext:
+    name: str
+    description: str
+    primary_language: str
+    languages: list[str]
+    file_tree: list[str]
+    parsed_files: list[ParsedFile]
+    dependencies: dict[str, list[str]]
+    commit_groups: list[CommitGroup]
+    github_url: str | None = None
+    topics: list[str] = field(default_factory=list)
+    license: str | None = None
+    has_tests: bool = False
+    has_ci: bool = False
+    entry_points: list[str] = field(default_factory=list)
+
+
+def build_context(
+    repo_path: Path,
+    parsed_files: list[ParsedFile],
+    dependencies: dict[str, list[str]],
+    commit_groups: list[CommitGroup],
+    gh_metadata: dict | None = None,
+) -> RepoContext:
+    gh = gh_metadata or {}
+
+    name = gh.get("name") or repo_path.name
+    description = gh.get("description") or _infer_description(parsed_files, repo_path)
+    languages = _detect_languages(parsed_files)
+    primary_language = languages[0] if languages else "unknown"
+    file_tree = _build_file_tree(repo_path, parsed_files)
+    has_tests = _has_tests(repo_path)
+    has_ci = _has_ci(repo_path)
+    entry_points = _find_entry_points(repo_path, parsed_files)
+
+    return RepoContext(
+        name=name,
+        description=description,
+        primary_language=primary_language,
+        languages=languages,
+        file_tree=file_tree,
+        parsed_files=parsed_files,
+        dependencies=dependencies,
+        commit_groups=commit_groups,
+        github_url=gh.get("html_url"),
+        topics=gh.get("topics", []),
+        license=gh.get("license", {}).get("name") if gh.get("license") else None,
+        has_tests=has_tests,
+        has_ci=has_ci,
+        entry_points=entry_points,
+    )
+
+
+def _detect_languages(parsed_files: list[ParsedFile]) -> list[str]:
+    counts: dict[str, int] = {}
+    for f in parsed_files:
+        counts[f.language] = counts.get(f.language, 0) + 1
+    return sorted(counts, key=lambda k: counts[k], reverse=True)
+
+
+def _build_file_tree(repo_path: Path, parsed_files: list[ParsedFile]) -> list[str]:
+    paths = []
+    for f in parsed_files:
+        try:
+            rel = Path(f.path).relative_to(repo_path)
+            paths.append(str(rel))
+        except ValueError:
+            paths.append(f.path)
+    return sorted(paths)
+
+
+def _infer_description(parsed_files: list[ParsedFile], repo_path: Path) -> str:
+    # Try to read existing README for description
+    for readme_name in ["README.md", "README.rst", "README.txt", "README"]:
+        readme = repo_path / readme_name
+        if readme.exists():
+            try:
+                content = readme.read_text(errors="replace")
+                lines = [l.strip() for l in content.splitlines() if l.strip()]
+                # Find first non-heading line
+                for line in lines:
+                    if not line.startswith("#") and len(line) > 20:
+                        return line[:200]
+            except Exception:
+                pass
+    return "A software project"
+
+
+def _has_tests(repo_path: Path) -> bool:
+    test_indicators = ["tests/", "test/", "spec/", "__tests__/", "test_*.py", "*_test.go"]
+    for indicator in test_indicators:
+        if list(repo_path.glob(f"**/{indicator}")):
+            return True
+    return False
+
+
+def _has_ci(repo_path: Path) -> bool:
+    ci_paths = [
+        ".github/workflows",
+        ".gitlab-ci.yml",
+        ".circleci/config.yml",
+        "Jenkinsfile",
+        ".travis.yml",
+    ]
+    for ci_path in ci_paths:
+        if (repo_path / ci_path).exists():
+            return True
+    return False
+
+
+def _find_entry_points(repo_path: Path, parsed_files: list[ParsedFile]) -> list[str]:
+    entry_points = []
+    candidates = ["main.py", "app.py", "cli.py", "main.go", "main.rs", "index.js", "index.ts"]
+    for candidate in candidates:
+        if (repo_path / candidate).exists():
+            entry_points.append(candidate)
+    return entry_points

docforge/analysis/dependency_detector.py

@@ -0,0 +1,120 @@
+import json
+import tomllib
+from pathlib import Path
+
+
+def detect_dependencies(repo_path: Path) -> dict[str, list[str]]:
+    """Read manifest files and return dependencies per language/ecosystem."""
+    deps: dict[str, list[str]] = {}
+
+    # Python
+    py_deps = _detect_python(repo_path)
+    if py_deps:
+        deps["python"] = py_deps
+
+    # JavaScript/TypeScript
+    js_deps = _detect_javascript(repo_path)
+    if js_deps:
+        deps["javascript"] = js_deps
+
+    # Go
+    go_deps = _detect_go(repo_path)
+    if go_deps:
+        deps["go"] = go_deps
+
+    # Rust
+    rust_deps = _detect_rust(repo_path)
+    if rust_deps:
+        deps["rust"] = rust_deps
+
+    # Java
+    java_deps = _detect_java(repo_path)
+    if java_deps:
+        deps["java"] = java_deps
+
+    return deps
+
+
+def _detect_python(repo_path: Path) -> list[str]:
+    deps = []
+
+    pyproject = repo_path / "pyproject.toml"
+    if pyproject.exists():
+        try:
+            with open(pyproject, "rb") as f:
+                data = tomllib.load(f)
+            deps += data.get("project", {}).get("dependencies", [])
+        except Exception:
+            pass
+
+    requirements = repo_path / "requirements.txt"
+    if requirements.exists():
+        try:
+            lines = requirements.read_text().splitlines()
+            deps += [l.strip() for l in lines if l.strip() and not l.startswith("#")]
+        except Exception:
+            pass
+
+    return deps
+
+
+def _detect_javascript(repo_path: Path) -> list[str]:
+    package_json = repo_path / "package.json"
+    if not package_json.exists():
+        return []
+    try:
+        data = json.loads(package_json.read_text())
+        deps = list(data.get("dependencies", {}).keys())
+        deps += list(data.get("devDependencies", {}).keys())
+        return deps
+    except Exception:
+        return []
+
+
+def _detect_go(repo_path: Path) -> list[str]:
+    go_mod = repo_path / "go.mod"
+    if not go_mod.exists():
+        return []
+    deps = []
+    try:
+        for line in go_mod.read_text().splitlines():
+            line = line.strip()
+            if line.startswith("require ") or (line and not line.startswith("//") and " v" in line):
+                parts = line.split()
+                if len(parts) >= 2 and "/" in parts[0]:
+                    deps.append(parts[0])
+    except Exception:
+        pass
+    return deps
+
+
+def _detect_rust(repo_path: Path) -> list[str]:
+    cargo_toml = repo_path / "Cargo.toml"
+    if not cargo_toml.exists():
+        return []
+    try:
+        with open(cargo_toml, "rb") as f:
+            data = tomllib.load(f)
+        deps = list(data.get("dependencies", {}).keys())
+        deps += list(data.get("dev-dependencies", {}).keys())
+        return deps
+    except Exception:
+        return []
+
+
+def _detect_java(repo_path: Path) -> list[str]:
+    # Basic pom.xml detection
+    pom = repo_path / "pom.xml"
+    if pom.exists():
+        try:
+            content = pom.read_text()
+            # Very basic extraction - just flag that Maven is used
+            return ["[Maven - see pom.xml]"]
+        except Exception:
+            pass
+
+    build_gradle = repo_path / "build.gradle"
+    if build_gradle.exists():
+        return ["[Gradle - see build.gradle]"]
+
+    return []

docforge/analysis/file_walker.py

@@ -0,0 +1,63 @@
+import fnmatch
+from pathlib import Path
+
+
+BINARY_EXTENSIONS = {
+    ".png", ".jpg", ".jpeg", ".gif", ".bmp", ".ico", ".svg",
+    ".pdf", ".zip", ".tar", ".gz", ".bz2", ".xz", ".rar",
+    ".exe", ".dll", ".so", ".dylib", ".a", ".o",
+    ".woff", ".woff2", ".ttf", ".eot",
+    ".mp3", ".mp4", ".wav", ".avi", ".mov",
+    ".db", ".sqlite", ".lock",
+}
+
+
+def walk_repo(
+    repo_path: Path,
+    exclude_patterns: list[str],
+    max_file_size_kb: int = 512,
+    supported_extensions: list[str] | None = None,
+):
+    """
+    Yield (file_path, extension) for all code files in repo_path.
+    Respects exclude patterns and size limits.
+    """
+    max_bytes = max_file_size_kb * 1024
+
+    for file_path in sorted(repo_path.rglob("*")):
+        if not file_path.is_file():
+            continue
+
+        # Check binary extension
+        if file_path.suffix.lower() in BINARY_EXTENSIONS:
+            continue
+
+        # Check exclude patterns
+        relative = str(file_path.relative_to(repo_path))
+        if _is_excluded(relative, exclude_patterns):
+            continue
+
+        # Check supported extension
+        ext = file_path.suffix.lower()
+        if supported_extensions and ext not in supported_extensions:
+            continue
+
+        # Check file size
+        try:
+            if file_path.stat().st_size > max_bytes:
+                continue
+        except OSError:
+            continue
+
+        yield file_path, ext
+
+
+def _is_excluded(relative_path: str, patterns: list[str]) -> bool:
+    for pattern in patterns:
+        if fnmatch.fnmatch(relative_path, pattern):
+            return True
+        # Also match path components
+        parts = relative_path.replace("\\", "/")
+        if fnmatch.fnmatch(parts, pattern):
+            return True
+    return False

docforge/analysis/repo_analyzer.py

@@ -0,0 +1,53 @@
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, TaskProgressColumn
+
+from ..config.schema import DocForgeConfig
+from ..parsing.base_parser import ParsedFile
+from ..parsing.registry import get_parser, supported_extensions
+from .file_walker import walk_repo
+
+
+def analyze_repo(repo_path: Path, config: DocForgeConfig) -> list[ParsedFile]:
+    """Parse all code files in repo_path and return ParsedFile list."""
+    exts = supported_extensions()
+    files_to_parse = list(walk_repo(
+        repo_path,
+        exclude_patterns=config.exclude_patterns,
+        max_file_size_kb=config.max_file_size_kb,
+        supported_extensions=exts,
+    ))
+
+    parsed: list[ParsedFile] = []
+
+    with Progress(
+        SpinnerColumn(),
+        TextColumn("[bold blue]Parsing files..."),
+        BarColumn(),
+        TaskProgressColumn(),
+        transient=True,
+    ) as progress:
+        task = progress.add_task("parsing", total=len(files_to_parse))
+
+        def parse_file(args):
+            file_path, ext = args
+            parser = get_parser(ext)
+            if parser is None:
+                return None
+            try:
+                source = file_path.read_bytes()
+                return parser.parse(file_path, source)
+            except Exception:
+                return None
+            finally:
+                progress.advance(task)
+
+        with ThreadPoolExecutor(max_workers=8) as executor:
+            futures = {executor.submit(parse_file, item): item for item in files_to_parse}
+            for future in as_completed(futures):
+                result = future.result()
+                if result is not None:
+                    parsed.append(result)
+
+    return parsed

docforge/cli/__init__.py

@@ -0,0 +1,3 @@
+from docforge import __version__
+
+__all__ = ["__version__"]

docforge/cli/app.py

@@ -0,0 +1,29 @@
+import typer
+from . import __version__
+from .commands.generate import generate
+
+app = typer.Typer(
+    name="docforge",
+    help="AI-powered documentation generator for any GitHub repository.",
+    add_completion=False,
+    rich_markup_mode="rich",
+)
+
+app.command(name="generate", help="Generate documentation for a repo.")(generate)
+
+
+def version_callback(value: bool):
+    if value:
+        typer.echo(f"DocForge v{__version__}")
+        raise typer.Exit()
+
+
+@app.callback()
+def main_callback(
+    version: bool = typer.Option(None, "--version", "-v", callback=version_callback, is_eager=True),
+):
+    pass
+
+
+def main():
+    app()

docforge/cli/commands/generate.py

@@ -0,0 +1,171 @@
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+import typer
+from rich.panel import Panel
+from rich.table import Table
+from rich import box
+
+from ..console import console
+from ...config.loader import load_config
+from ...source.resolver import SourceResolver
+from ...git.repo_reader import read_repo
+from ...git.changelog_builder import build_changelog
+from ...analysis.repo_analyzer import analyze_repo
+from ...analysis.dependency_detector import detect_dependencies
+from ...analysis.context_builder import build_context
+from ...llm.client import LLMClient
+from ...llm.prompt_manager import PromptManager
+from ...generators.readme_generator import ReadmeGenerator
+from ...generators.installation_generator import InstallationGenerator
+from ...generators.api_generator import ApiGenerator
+from ...generators.architecture_generator import ArchitectureGenerator
+from ...generators.changelog_generator import ChangelogGenerator
+from ...generators.base_generator import GeneratorResult
+from ...output.writer import write_results
+from ...output.mkdocs_builder import build_mkdocs
+from ...output.github_action_writer import write_github_action
+
+
+def generate(
+    source: str = typer.Argument(..., help="GitHub URL or local path to repository"),
+    model: str = typer.Option(None, "--model", "-m", help="LLM model (e.g. gpt-4o, claude-sonnet-4-6, ollama/llama3)"),
+    api_base: str = typer.Option(None, "--api-base", help="Custom API base URL (for Ollama: http://localhost:11434)"),
+    output_dir: str = typer.Option(None, "--output-dir", "-o", help="Output directory (default: ./docs)"),
+    mkdocs: bool = typer.Option(False, "--mkdocs", help="Generate mkdocs.yml"),
+    github_action: bool = typer.Option(False, "--github-action", help="Generate GitHub Action workflow"),
+    overwrite: bool = typer.Option(False, "--overwrite", help="Overwrite existing docs"),
+    no_api: bool = typer.Option(False, "--no-api", help="Skip per-file API docs generation"),
+    no_changelog: bool = typer.Option(False, "--no-changelog", help="Skip changelog generation"),
+):
+    """Generate documentation for a GitHub repository or local project."""
+
+    console.print(Panel.fit(
+        "[bold blue]DocForge[/bold blue] — AI-powered documentation generator",
+        border_style="blue",
+    ))
+
+    # 1. Resolve source
+    with console.status("[bold]Resolving source..."):
+        resolver = SourceResolver()
+        try:
+            repo_path, gh_metadata = resolver.resolve(source)
+        except Exception as e:
+            console.print(f"[red]Error resolving source:[/red] {e}")
+            raise typer.Exit(1)
+
+    console.print(f"[green]✓[/green] Source: [bold]{repo_path}[/bold]")
+
+    # 2. Load config
+    config = load_config(
+        repo_path=repo_path,
+        model=model,
+        api_base=api_base,
+        output_dir=output_dir,
+        mkdocs=mkdocs,
+        github_action=github_action,
+        overwrite=overwrite,
+    )
+    out_dir = Path(config.output.directory)
+
+    console.print(f"[green]✓[/green] Model: [bold]{config.llm.model}[/bold]")
+    console.print(f"[green]✓[/green] Output: [bold]{out_dir}[/bold]")
+
+    # 3. Git history
+    with console.status("[bold]Reading git history..."):
+        commits, tags = read_repo(repo_path)
+        commit_groups = build_changelog(commits) if commits else []
+
+    console.print(f"[green]✓[/green] Git: {len(commits)} commits, {len(tags)} tags")
+
+    # 4. Parse code
+    console.print("[bold]Parsing code...[/bold]")
+    parsed_files = analyze_repo(repo_path, config)
+    console.print(f"[green]✓[/green] Parsed {len(parsed_files)} files")
+
+    # 5. Detect dependencies
+    with console.status("[bold]Detecting dependencies..."):
+        dependencies = detect_dependencies(repo_path)
+
+    # 6. Build context
+    context = build_context(
+        repo_path=repo_path,
+        parsed_files=parsed_files,
+        dependencies=dependencies,
+        commit_groups=commit_groups,
+        gh_metadata=gh_metadata,
+    )
+
+    console.print(f"[green]✓[/green] Project: [bold]{context.name}[/bold] ({', '.join(context.languages) or 'unknown'})")
+
+    # 7. Initialize LLM
+    llm = LLMClient(config.llm)
+    prompts = PromptManager()
+
+    # 8. Run generators
+    console.print("\n[bold]Generating documentation...[/bold]")
+    all_results: list[GeneratorResult] = []
+
+    generators = [
+        ("README", ReadmeGenerator(llm, prompts)),
+        ("Installation", InstallationGenerator(llm, prompts)),
+        ("Architecture", ArchitectureGenerator(llm, prompts)),
+    ]
+
+    if not no_changelog:
+        generators.append(("Changelog", ChangelogGenerator(llm, prompts)))
+
+    def run_generator(name_gen):
+        name, gen = name_gen
+        with console.status(f"[bold]Generating {name}..."):
+            result = gen.generate(context)
+        console.print(f"[green]✓[/green] {name}")
+        return result
+
+    # Run core generators (README, Installation, Architecture, Changelog) in parallel
+    with ThreadPoolExecutor(max_workers=4) as executor:
+        futures = {executor.submit(run_generator, ng): ng[0] for ng in generators}
+        for future in as_completed(futures):
+            try:
+                all_results.append(future.result())
+            except Exception as e:
+                name = futures[future]
+                console.print(f"[red]✗[/red] {name}: {e}")
+
+    # API docs (sequential to avoid rate limits)
+    if not no_api:
+        with console.status("[bold]Generating API docs..."):
+            api_gen = ApiGenerator(llm, prompts)
+            try:
+                api_results = api_gen.generate_all(context)
+                all_results.extend(api_results)
+                console.print(f"[green]✓[/green] API docs ({len(api_results)} files)")
+            except Exception as e:
+                console.print(f"[red]✗[/red] API docs: {e}")
+
+    # 9. Write output
+    with console.status("[bold]Writing files..."):
+        written = write_results(all_results, out_dir, overwrite=config.output.overwrite)
+
+    # 10. Optional outputs
+    if config.output.mkdocs:
+        with console.status("[bold]Generating mkdocs.yml..."):
+            mkdocs_path = build_mkdocs(context, all_results, out_dir, prompts)
+        console.print(f"[green]✓[/green] MkDocs config: {mkdocs_path}")
+
+    if config.output.github_action:
+        with console.status("[bold]Writing GitHub Action..."):
+            action_path = write_github_action(repo_path, config.llm, prompts)
+        console.print(f"[green]✓[/green] GitHub Action: {action_path}")
+
+    # 11. Summary table
+    table = Table(title="\nGenerated Documentation", box=box.ROUNDED)
+    table.add_column("File", style="cyan")
+    table.add_column("Size", justify="right", style="green")
+
+    for path in sorted(written):
+        size = path.stat().st_size
+        table.add_row(str(path.relative_to(Path.cwd()) if path.is_relative_to(Path.cwd()) else path), f"{size:,} bytes")
+
+    console.print(table)
+    console.print(f"\n[bold green]Done![/bold green] {len(written)} files written to [bold]{out_dir}[/bold]")

docforge/cli/console.py

@@ -0,0 +1,3 @@
+from rich.console import Console
+
+console = Console()

docforge/config/__init__.py

@@ -0,0 +1,4 @@
+from .schema import DocForgeConfig, LLMConfig, OutputConfig, GeneratorConfig
+from .loader import load_config
+
+__all__ = ["DocForgeConfig", "LLMConfig", "OutputConfig", "GeneratorConfig", "load_config"]

docforge/config/loader.py

@@ -0,0 +1,48 @@
+import os
+import tomllib
+from pathlib import Path
+
+from .schema import DocForgeConfig, LLMConfig, OutputConfig
+
+
+def load_config(
+    repo_path: Path,
+    model: str | None = None,
+    api_base: str | None = None,
+    output_dir: str | None = None,
+    mkdocs: bool = False,
+    github_action: bool = False,
+    overwrite: bool = False,
+) -> DocForgeConfig:
+    """Load config from .docforge.toml if present, then apply CLI overrides."""
+    config = DocForgeConfig()
+
+    config_file = repo_path / ".docforge.toml"
+    if config_file.exists():
+        with open(config_file, "rb") as f:
+            data = tomllib.load(f)
+        config = DocForgeConfig.model_validate(data)
+
+    # Apply CLI flag overrides
+    if model:
+        config.llm.model = model
+    if api_base:
+        config.llm.api_base = api_base
+    if output_dir:
+        config.output.directory = output_dir
+    if mkdocs:
+        config.output.mkdocs = True
+    if github_action:
+        config.output.github_action = True
+    if overwrite:
+        config.output.overwrite = True
+
+    # Apply environment variable overrides
+    if env_key := os.getenv("OPENAI_API_KEY"):
+        config.llm.api_key = config.llm.api_key or env_key
+    if env_model := os.getenv("DOCFORGE_MODEL"):
+        config.llm.model = env_model
+    if env_base := os.getenv("DOCFORGE_API_BASE"):
+        config.llm.api_base = env_base
+
+    return config

docforge/config/schema.py

@@ -0,0 +1,40 @@
+from pydantic import BaseModel, Field
+
+
+class LLMConfig(BaseModel):
+    model: str = "gpt-4o"
+    api_base: str | None = None  # For Ollama: "http://localhost:11434"
+    api_key: str | None = None
+    max_tokens: int = 4096
+    temperature: float = 0.2
+
+
+class GeneratorConfig(BaseModel):
+    enabled: bool = True
+    extra_instructions: str = ""
+
+
+class OutputConfig(BaseModel):
+    directory: str = "./docs"
+    mkdocs: bool = False
+    github_action: bool = False
+    overwrite: bool = False
+
+
+class DocForgeConfig(BaseModel):
+    llm: LLMConfig = Field(default_factory=LLMConfig)
+    output: OutputConfig = Field(default_factory=OutputConfig)
+    generators: dict[str, GeneratorConfig] = Field(default_factory=dict)
+    exclude_patterns: list[str] = [
+        "**/vendor/**",
+        "**/node_modules/**",
+        "**/.git/**",
+        "**/dist/**",
+        "**/build/**",
+        "**/__pycache__/**",
+        "**/*.pyc",
+        "**/*.min.js",
+        "**/*.min.css",
+    ]
+    max_file_size_kb: int = 512
+    languages: list[str] = ["python", "javascript", "typescript", "go", "rust", "java"]