ai-docs-gen 0.1.2__py3-none-any.whl

ai_docs/llm.py

@@ -0,0 +1,82 @@
+import json
+import os
+import threading
+from typing import Dict, List, Optional
+
+import requests
+
+from .utils import sha256_text
+
+
+class LLMClient:
+    def __init__(
+        self,
+        api_key: str,
+        base_url: str,
+        model: str,
+        temperature: float = 0.2,
+        max_tokens: int = 1200,
+        context_limit: int = 8192,
+    ):
+        self.api_key = api_key
+        self.base_url = base_url.rstrip("/")
+        self.model = model
+        self.temperature = temperature
+        self.max_tokens = max_tokens
+        self.context_limit = context_limit
+        self._cache_lock = threading.Lock()
+
+    def _cache_key(self, payload: Dict) -> str:
+        return sha256_text(json.dumps(payload, sort_keys=True))
+
+    def chat(self, messages: List[Dict[str, str]], cache: Optional[Dict[str, str]] = None) -> str:
+        payload = {
+            "model": self.model,
+            "messages": messages,
+            "temperature": self.temperature,
+            "max_tokens": self.max_tokens,
+        }
+        key = self._cache_key(payload)
+        if cache is not None:
+            with self._cache_lock:
+                if key in cache:
+                    return cache[key]
+
+        if self.base_url.endswith("/v1"):
+            url = f"{self.base_url}/chat/completions"
+        else:
+            url = f"{self.base_url}/v1/chat/completions"
+        headers = {
+            "Content-Type": "application/json",
+            "Authorization": f"Bearer {self.api_key}",
+        }
+        response = requests.post(url, headers=headers, json=payload, timeout=(120, 480))
+        response.raise_for_status()
+        data = response.json()
+        try:
+            content = data["choices"][0]["message"]["content"]
+        except Exception as exc:
+            raise RuntimeError(f"LLM response missing content: {data}") from exc
+        if cache is not None:
+            with self._cache_lock:
+                cache[key] = content
+        return content
+
+
+def from_env() -> LLMClient:
+    api_key = os.getenv("OPENAI_API_KEY")
+    if not api_key:
+        raise RuntimeError("OPENAI_API_KEY is not set")
+    base_url = os.getenv("OPENAI_BASE_URL", "https://api.openai.com/v1")
+    model = os.getenv("OPENAI_MODEL", "gpt-4o-mini")
+    temperature = float(os.getenv("OPENAI_TEMPERATURE", "0.2"))
+    max_tokens = int(os.getenv("OPENAI_MAX_TOKENS", "1200"))
+    context_limit = int(os.getenv("OPENAI_CONTEXT_TOKENS", "8192"))
+    return LLMClient(
+        api_key=api_key,
+        base_url=base_url,
+        model=model,
+        temperature=temperature,
+        max_tokens=max_tokens,
+        context_limit=context_limit,
+    )

ai_docs/mkdocs.py

@@ -0,0 +1,161 @@
+from pathlib import Path
+from typing import Dict, List
+
+import yaml
+
+
+class _YamlPythonName(str):
+    pass
+
+
+class _YamlSafeDumper(yaml.SafeDumper):
+    pass
+
+
+def _python_name_representer(dumper: yaml.Dumper, data: _YamlPythonName) -> yaml.nodes.ScalarNode:
+    return dumper.represent_scalar(f"tag:yaml.org,2002:python/name:{data}", "")
+
+
+_YamlSafeDumper.add_representer(_YamlPythonName, _python_name_representer)
+
+
+def build_mkdocs_yaml(
+    site_name: str,
+    sections: Dict[str, str],
+    configs: Dict[str, str],
+    local_site: bool = False,
+    has_modules: bool = False,
+    module_nav_paths: List[str] | None = None,
+    project_config_nav_paths: List[str] | None = None,
+) -> str:
+    nav = [
+        {"Главная": "index.md"},
+    ]
+    if "architecture" in sections:
+        nav.append({"Архитектура": "architecture.md"})
+    if "runtime" in sections:
+        nav.append({"Запуск": "runtime.md"})
+    if "dependencies" in sections:
+        nav.append({"Зависимости": "dependencies.md"})
+    if "testing" in sections:
+        nav.append({"Тестирование": "testing.md"})
+    if "conventions" in sections:
+        nav.append({"Соглашения": "conventions.md"})
+    if "glossary" in sections:
+        nav.append({"Глоссарий": "glossary.md"})
+
+    if configs:
+        cfg_nav: List[Dict[str, str]] = []
+        for key, filename in configs.items():
+            title = {
+                "kubernetes": "Kubernetes",
+                "helm": "Helm",
+                "terraform": "Terraform",
+                "ansible": "Ansible",
+                "docker": "Docker",
+                "ci": "CI/CD",
+            }.get(key, key)
+            cfg_nav.append({title: f"configs/{filename}"})
+        nav.append({"Конфиги": cfg_nav})
+
+    if project_config_nav_paths:
+        project_cfg_nav: List[Dict[str, object]] = [{"Обзор": "configs/index.md"}]
+        project_cfg_nav.extend(_build_tree_nav(project_config_nav_paths, "configs/files/"))
+        nav.append({"Конфигурация проекта": project_cfg_nav})
+
+    if has_modules:
+        modules_nav: List[Dict[str, object]] = [{"Обзор": "modules/index.md"}]
+        if module_nav_paths:
+            modules_nav.extend(_build_tree_nav(module_nav_paths, "modules/"))
+        nav.append({"Модули": modules_nav})
+
+    nav.append({"Изменения": "changes.md"})
+
+    data = {
+        "site_name": site_name,
+        "docs_dir": ".ai-docs",
+        "site_dir": "ai_docs_site",
+        "plugins": [
+            "search",
+            {"mermaid2": {"javascript": "js/mermaid.min.js"}},
+        ],
+        "markdown_extensions": [
+            "tables",
+            "sane_lists",
+            "attr_list",
+            "def_list",
+            "footnotes",
+            "admonition",
+            "fenced_code",
+            {
+                "pymdownx.superfences": {
+                    "custom_fences": [
+                        {
+                            "name": "mermaid",
+                            "class": "mermaid",
+                            "format": _YamlPythonName("mermaid2.fence_mermaid"),
+                        }
+                    ]
+                }
+            },
+        ],
+        "nav": nav,
+    }
+    if local_site:
+        data["site_url"] = ""
+        data["use_directory_urls"] = False
+    return yaml.dump(data, allow_unicode=True, sort_keys=False, Dumper=_YamlSafeDumper)
+
+
+def _build_tree_nav(paths: List[str], strip_prefix: str) -> List[Dict[str, object]]:
+    tree: Dict[str, object] = {}
+
+    for rel_path in paths:
+        rel = Path(rel_path).as_posix()
+        if rel.startswith(strip_prefix):
+            rel = rel[len(strip_prefix) :]
+        parts = rel.split("/")
+        if parts:
+            last = Path(parts[-1]).with_suffix("").name
+            sep = last.rfind("__")
+            if sep != -1 and sep + 2 < len(last):
+                base = last[:sep]
+                ext = last[sep + 2 :]
+                parts[-1] = f"{base}.{ext}"
+            else:
+                parts[-1] = last
+        _insert_nav_node(tree, parts, rel_path)
+
+    return _tree_to_nav(tree)
+
+
+def _insert_nav_node(tree: Dict[str, object], parts: List[str], rel_path: str) -> None:
+    key = parts[0]
+    if len(parts) == 1:
+        tree[key] = rel_path
+        return
+    node = tree.get(key)
+    if not isinstance(node, dict):
+        node = {}
+        tree[key] = node
+    _insert_nav_node(node, parts[1:], rel_path)
+
+
+def _tree_to_nav(tree: Dict[str, object]) -> List[Dict[str, object]]:
+    nav: List[Dict[str, object]] = []
+    for key in sorted(tree.keys(), key=lambda k: (not isinstance(tree[k], dict), k.lower())):
+        value = tree[key]
+        if isinstance(value, dict):
+            label = key if key.startswith("/") else f"/{key}"
+            nav.append({label: _tree_to_nav(value)})
+        else:
+            nav.append({key: value})
+    return nav
+
+
+def write_docs_files(docs_dir: Path, files: Dict[str, str]) -> None:
+    docs_dir.mkdir(parents=True, exist_ok=True)
+    for rel_path, content in files.items():
+        out_path = docs_dir / rel_path
+        out_path.parent.mkdir(parents=True, exist_ok=True)
+        out_path.write_text(content, encoding="utf-8")

ai_docs/scanner.py

@@ -0,0 +1,237 @@
+import os
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+from typing import Dict, List, Optional, Set, Tuple
+
+import pathspec
+import yaml
+
+from .domain import (
+    CODE_EXTENSION_DESCRIPTIONS,
+    CONFIG_EXTENSION_DESCRIPTIONS,
+    DOC_EXTENSION_DESCRIPTIONS,
+    classify_type,
+    detect_domains,
+    is_infra,
+)
+from .utils import is_binary_file, is_url, read_text_file, to_posix
+
+
+FIXED_INCLUDE_PATTERNS = {
+    "*.tf", "*.tfvars",
+    "Dockerfile*", "docker-compose*.yml", "docker-compose*.yaml", "compose.yml", "compose.yaml",
+    "Jenkinsfile", ".gitlab-ci.yml", "azure-pipelines.yml",
+    "requirements.txt", "pyproject.toml", "package.json", "package-lock.json",
+}
+
+DEFAULT_EXCLUDE_PATTERNS = {
+    ".git/*", "**/.git/*",
+    ".venv/*", ".venv/**", "**/.venv/*", "**/.venv/**",
+    "venv/*", "venv/**", "**/venv/*", "**/venv/**",
+    "**/node_modules/*",
+    "**/dist/*", "**/build/*",
+    "**/.idea/*", "**/.vscode/*", "**/__pycache__/*",
+    "**/.pytest_cache/*", "**/.mypy_cache/*",
+    "**/.ai_docs_cache/*", "**/.ai_docs_cache/**", ".ai_docs_cache/**", ".ai_docs_cache/*",
+    "**/ai_docs_site/*", "**/ai_docs_site/**", "ai_docs_site/**", "ai_docs_site/*",
+    ".ai-docs/*", ".ai-docs/**", "**/.ai-docs/*", "**/.ai-docs/**",
+    ".github/*", ".github/**", "**/.github/*", "**/.github/**",
+    "mkdocs.yml", "**/mkdocs.yml", "mkdocs_yml.md", "**/mkdocs_yml.md",
+    ".ai-docs.yaml", "**/.ai-docs.yaml",
+}
+
+
+class ScanResult:
+    def __init__(self, root: Path, files: List[Dict], source: str, repo_name: str):
+        self.root = root
+        self.files = files
+        self.source = source
+        self.repo_name = repo_name
+
+
+def _normalize_extensions(raw: object, defaults: Dict[str, str]) -> Dict[str, str]:
+    normalized: Dict[str, str] = {}
+    if isinstance(raw, dict):
+        items = raw.items()
+        for key, value in items:
+            ext = str(key).strip()
+            if not ext:
+                continue
+            if not ext.startswith("."):
+                ext = f".{ext}"
+            desc = value if isinstance(value, str) and value.strip() else defaults.get(ext, "")
+            normalized[ext] = desc
+    elif isinstance(raw, list):
+        for item in raw:
+            ext = str(item).strip()
+            if not ext:
+                continue
+            if not ext.startswith("."):
+                ext = f".{ext}"
+            normalized[ext] = defaults.get(ext, "")
+    return normalized or defaults.copy()
+
+
+def _normalize_excludes(raw: object) -> Set[str]:
+    if not isinstance(raw, list):
+        return set()
+    return {str(item).strip() for item in raw if str(item).strip()}
+
+
+def _load_extension_config(root: Path) -> Dict[str, object]:
+    config_path = root / ".ai-docs.yaml"
+    defaults = {
+        "code_extensions": CODE_EXTENSION_DESCRIPTIONS,
+        "doc_extensions": DOC_EXTENSION_DESCRIPTIONS,
+        "config_extensions": CONFIG_EXTENSION_DESCRIPTIONS,
+    }
+
+    if not config_path.exists():
+        payload = {
+            "code_extensions": defaults["code_extensions"],
+            "doc_extensions": defaults["doc_extensions"],
+            "config_extensions": defaults["config_extensions"],
+        }
+        config_path.write_text(
+            yaml.safe_dump(payload, allow_unicode=True, sort_keys=False),
+            encoding="utf-8",
+        )
+        return {**{key: value.copy() for key, value in defaults.items()}, "exclude": set()}
+
+    try:
+        raw = yaml.safe_load(config_path.read_text(encoding="utf-8", errors="ignore")) or {}
+    except yaml.YAMLError:
+        return {**{key: value.copy() for key, value in defaults.items()}, "exclude": set()}
+
+    if not isinstance(raw, dict):
+        return {**{key: value.copy() for key, value in defaults.items()}, "exclude": set()}
+
+    code_raw = raw.get("code_extensions") or {}
+    doc_raw = raw.get("doc_extensions") or {}
+    config_raw = raw.get("config_extensions") or {}
+    exclude_raw = raw.get("exclude") or []
+
+    return {
+        "code_extensions": _normalize_extensions(code_raw, defaults["code_extensions"]),
+        "doc_extensions": _normalize_extensions(doc_raw, defaults["doc_extensions"]),
+        "config_extensions": _normalize_extensions(config_raw, defaults["config_extensions"]),
+        "exclude": _normalize_excludes(exclude_raw),
+    }
+
+
+def _build_default_include_patterns(extension_config: Dict[str, object]) -> Set[str]:
+    extensions: Set[str] = set()
+    for key in ("code_extensions", "doc_extensions", "config_extensions"):
+        extensions.update(extension_config.get(key, {}).keys())
+    return {f"*{ext}" for ext in extensions} | FIXED_INCLUDE_PATTERNS
+
+
+def _load_ignore_specs(root: Path) -> List[pathspec.PathSpec]:
+    specs: List[pathspec.PathSpec] = []
+    for name in (".gitignore", ".build_ignore"):
+        ignore_file = root / name
+        if not ignore_file.exists():
+            continue
+        patterns = ignore_file.read_text(encoding="utf-8", errors="ignore").splitlines()
+        specs.append(pathspec.PathSpec.from_lines("gitignore", patterns))
+    return specs
+
+
+def _should_include(rel_path: str, include: Optional[Set[str]], exclude: Optional[Set[str]], ignore_specs: List[pathspec.PathSpec]) -> bool:
+    for spec in ignore_specs:
+        if spec.match_file(rel_path):
+            return False
+    if exclude:
+        for pattern in exclude:
+            if pathspec.PathSpec.from_lines("gitignore", [pattern]).match_file(rel_path):
+                return False
+    if not include:
+        return True
+    for pattern in include:
+        if pathspec.PathSpec.from_lines("gitignore", [pattern]).match_file(rel_path):
+            return True
+    return False
+
+
+def _scan_directory(root: Path, include: Optional[Set[str]], exclude: Optional[Set[str]], max_size: int) -> List[Dict]:
+    files: List[Dict] = []
+    ignore_specs = _load_ignore_specs(root)
+
+    for dirpath, dirnames, filenames in os.walk(root):
+        # Avoid .git directory traversal
+        dirnames[:] = [d for d in dirnames if d != ".git"]
+        for filename in filenames:
+            abs_path = Path(dirpath) / filename
+            rel_path = abs_path.relative_to(root)
+            rel_path_str = to_posix(rel_path)
+
+            if not _should_include(rel_path_str, include, exclude, ignore_specs):
+                continue
+
+            if abs_path.is_symlink():
+                continue
+
+            try:
+                size = abs_path.stat().st_size
+            except OSError:
+                continue
+
+            if max_size and size > max_size:
+                continue
+
+            if is_binary_file(abs_path):
+                continue
+
+            content = read_text_file(abs_path)
+            content_snippet = content[:4000]
+            file_type = classify_type(abs_path)
+            domains = detect_domains(abs_path, content_snippet)
+            if is_infra(domains):
+                file_type = "infra"
+
+            files.append(
+                {
+                    "path": rel_path_str,
+                    "abs_path": abs_path,
+                    "size": size,
+                    "content": content,
+                    "type": file_type,
+                    "domains": sorted(domains),
+                }
+            )
+
+    return files
+
+
+def _clone_repo(repo_url: str) -> Tuple[Path, str]:
+    tmpdir = Path(tempfile.mkdtemp(prefix="ai_docs_"))
+    try:
+        subprocess.check_call(["git", "clone", "--depth", "1", repo_url, str(tmpdir)])
+    except Exception as exc:
+        shutil.rmtree(tmpdir, ignore_errors=True)
+        raise RuntimeError(f"Failed to clone repo: {exc}")
+    repo_name = repo_url.rstrip("/").split("/")[-1].replace(".git", "")
+    return tmpdir, repo_name
+
+
+def scan_source(source: str, include: Optional[Set[str]] = None, exclude: Optional[Set[str]] = None, max_size: int = 200_000) -> ScanResult:
+    exclude = exclude or DEFAULT_EXCLUDE_PATTERNS
+
+    if is_url(source):
+        root, repo_name = _clone_repo(source)
+        extension_config = _load_extension_config(root)
+        include = include or _build_default_include_patterns(extension_config)
+        exclude = set(exclude) | set(extension_config.get("exclude", set()))
+        files = _scan_directory(root, include, exclude, max_size)
+        return ScanResult(root=root, files=files, source=source, repo_name=repo_name)
+
+    root = Path(source).expanduser().resolve()
+    if not root.exists():
+        raise FileNotFoundError(f"Source path not found: {root}")
+    extension_config = _load_extension_config(root)
+    include = include or _build_default_include_patterns(extension_config)
+    exclude = set(exclude) | set(extension_config.get("exclude", set()))
+    files = _scan_directory(root, include, exclude, max_size)
+    return ScanResult(root=root, files=files, source=str(root), repo_name=root.name)