gdmcode 0.1.0__py3-none-any.whl

src/tools/dep_tools.py

@@ -0,0 +1,210 @@
+"""Dependency guard tools — CVE scanning and version pinning.
+
+DependencyGuardTool: runs pip-audit (preferred) or safety (fallback) to detect
+known CVEs before any new package is installed.
+
+PinDepsTool: generates a pinned requirements.lock from the live environment,
+filtering development-only packages out by default.
+"""
+from __future__ import annotations
+
+import logging
+import os
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+from typing import Any, ClassVar
+
+from src.tools import REGISTRY, ToolBase, ToolResult
+
+__all__ = ["DependencyGuardTool", "PinDepsTool"]
+
+log = logging.getLogger(__name__)
+
+_AUDIT_TIMEOUT: int = 60
+_PIN_TIMEOUT: int = 30
+
+# Packages that should be stripped from a production lock file
+_DEV_PKG_PREFIXES: frozenset[str] = frozenset({
+    "pytest", "pytest-cov", "pytest-xdist", "ruff", "mypy", "black",
+    "isort", "pre-commit", "build", "twine", "wheel", "setuptools",
+    "pip-audit", "safety", "coverage", "hypothesis", "faker",
+    "flake8", "pylint", "bandit", "pipdeptree",
+})
+
+
+# ---------------------------------------------------------------------------
+# DependencyGuardTool
+# ---------------------------------------------------------------------------
+
+class DependencyGuardTool(ToolBase):
+    """Scan Python dependencies for known CVEs before installing packages.
+
+    Runs ``pip-audit`` (preferred) or ``safety check`` (fallback).
+    Returns JSON-formatted vulnerability data or a clean bill of health.
+
+    **Always call this before pip install <package>.**
+    """
+
+    name: ClassVar[str] = "check_deps"
+    description: ClassVar[str] = (
+        "Scan Python dependencies for known CVEs. "
+        "Run before any 'pip install' to catch security vulnerabilities. "
+        "Returns vulnerable packages with CVE IDs, or confirms clean."
+    )
+    input_schema: ClassVar[dict[str, Any]] = {
+        "type": "object",
+        "properties": {
+            "packages": {
+                "type": "array",
+                "items": {"type": "string"},
+                "description": (
+                    "Specific packages to check (e.g. ['requests==2.28.0']). "
+                    "Omit to scan all installed packages."
+                ),
+            },
+            "requirements_file": {
+                "type": "string",
+                "description": "Path to a requirements file to scan (optional).",
+            },
+        },
+    }
+
+    def execute(self, args: dict[str, Any]) -> ToolResult:
+        packages: list[str] = args.get("packages") or []
+        req_file: str | None = args.get("requirements_file")
+
+        if shutil.which("pip-audit") is not None:
+            return self._run_pip_audit(packages, req_file)
+        if shutil.which("safety") is not None:
+            return self._run_safety(packages, req_file)
+
+        return ToolResult(output=(
+            "⚠  pip-audit and safety are not installed — cannot scan for CVEs.\n"
+            "Install: pip install pip-audit\n"
+            + (f"Packages requested: {', '.join(packages)}" if packages else "")
+        ))
+
+    # ------------------------------------------------------------------
+    # pip-audit
+    # ------------------------------------------------------------------
+
+    def _run_pip_audit(self, packages: list[str], req_file: str | None) -> ToolResult:
+        cmd = ["pip-audit", "--format", "json"]
+        if req_file:
+            cmd += ["-r", req_file]
+            return self._exec(cmd)
+        if packages:
+            return self._with_temp_req(packages, lambda tmp: self._exec(cmd + ["-r", tmp]))
+        return self._exec(cmd)
+
+    # ------------------------------------------------------------------
+    # safety
+    # ------------------------------------------------------------------
+
+    def _run_safety(self, packages: list[str], req_file: str | None) -> ToolResult:
+        cmd = ["safety", "check", "--json"]
+        if req_file:
+            cmd += ["-r", req_file]
+            return self._exec(cmd)
+        if packages:
+            return self._with_temp_req(packages, lambda tmp: self._exec(cmd + ["-r", tmp]))
+        return self._exec(cmd)
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _with_temp_req(
+        packages: list[str],
+        fn: Any,
+    ) -> ToolResult:
+        """Write a temporary requirements file, call fn(path), then delete it."""
+        fd, tmp = tempfile.mkstemp(suffix=".txt")
+        try:
+            with os.fdopen(fd, "w") as f:
+                f.write("\n".join(packages))
+            return fn(tmp)
+        finally:
+            try:
+                os.unlink(tmp)
+            except OSError:
+                pass
+
+    @staticmethod
+    def _exec(cmd: list[str]) -> ToolResult:
+        try:
+            res = subprocess.run(
+                cmd, capture_output=True, text=True, timeout=_AUDIT_TIMEOUT
+            )
+            output = res.stdout or res.stderr or "(no output)"
+            # pip-audit exits 1 when vulnerabilities are found — that is normal, not an error
+            ok = res.returncode in (0, 1)
+            return ToolResult(output=output, error="" if ok else res.stderr)
+        except subprocess.TimeoutExpired:
+            return ToolResult(output="", error=f"Audit timed out after {_AUDIT_TIMEOUT}s")
+        except Exception as exc:  # noqa: BLE001
+            return ToolResult(output="", error=str(exc))
+
+
+# ---------------------------------------------------------------------------
+# PinDepsTool
+# ---------------------------------------------------------------------------
+
+class PinDepsTool(ToolBase):
+    """Generate a pinned requirements.lock from the current Python environment.
+
+    Uses ``pip freeze`` and filters out development-only packages by default.
+    Call after installing new dependencies before committing changes.
+    """
+
+    name: ClassVar[str] = "pin_deps"
+    description: ClassVar[str] = (
+        "Generate a pinned requirements.lock from the current Python environment. "
+        "Filters out dev/test packages. Use after adding new dependencies."
+    )
+    input_schema: ClassVar[dict[str, Any]] = {
+        "type": "object",
+        "properties": {
+            "output_path": {
+                "type": "string",
+                "description": "File to write. Defaults to 'requirements.lock'.",
+            },
+            "exclude_dev": {
+                "type": "boolean",
+                "description": "Strip dev/test packages from output (default: true).",
+            },
+        },
+    }
+
+    def execute(self, args: dict[str, Any]) -> ToolResult:
+        out_path = Path(args.get("output_path") or "requirements.lock")
+        exclude_dev: bool = args.get("exclude_dev", True)
+
+        try:
+            res = subprocess.run(
+                ["pip", "freeze"], capture_output=True, text=True, timeout=_PIN_TIMEOUT
+            )
+            if res.returncode != 0:
+                return ToolResult(output="", error=res.stderr)
+
+            lines = res.stdout.splitlines()
+            if exclude_dev:
+                lines = [
+                    ln for ln in lines
+                    if not any(ln.lower().startswith(p.lower()) for p in _DEV_PKG_PREFIXES)
+                ]
+
+            out_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+            return ToolResult(output=f"Pinned {len(lines)} packages → {out_path}")
+        except subprocess.TimeoutExpired:
+            return ToolResult(output="", error=f"pip freeze timed out after {_PIN_TIMEOUT}s")
+        except Exception as exc:  # noqa: BLE001
+            return ToolResult(output="", error=str(exc))
+
+
+# Self-register
+REGISTRY.register(DependencyGuardTool())
+REGISTRY.register(PinDepsTool())

src/tools/document_reader.py

@@ -0,0 +1,167 @@
+"""DocumentReader — unified reader for Word, Excel, PDF, and CSV files.
+
+All document library imports are guarded with try/except so the module loads
+even when optional deps are not installed.  Install with:
+    pip install 'gdm-code[docs]'
+"""
+from __future__ import annotations
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+import csv, io, logging
+
+log = logging.getLogger(__name__)
+
+__all__ = ["DocumentReader", "DocumentContent", "SheetData"]
+
+
+@dataclass
+class SheetData:
+    name: str
+    headers: list[str]
+    rows: list[list[str]]  # all values coerced to str
+
+    def to_text(self) -> str:
+        lines = ["\t".join(self.headers)]
+        lines += ["\t".join(row) for row in self.rows]
+        return "\n".join(lines)
+
+
+@dataclass
+class DocumentContent:
+    file_path: str
+    format: str                      # "docx" | "xlsx" | "xls" | "pdf" | "csv" | "txt"
+    text: str                        # full plain-text representation
+    title: Optional[str] = None
+    author: Optional[str] = None
+    page_count: Optional[int] = None
+    sheets: list[SheetData] = field(default_factory=list)  # non-empty for spreadsheets
+    error: Optional[str] = None      # set if partial parse failure
+
+    @property
+    def success(self) -> bool:
+        return self.error is None
+
+
+class DocumentReader:
+    def read(self, path: Path | str) -> DocumentContent:
+        path = Path(path)
+        suffix = path.suffix.lower().lstrip(".")
+        dispatch = {
+            "docx": self._read_docx,
+            "xlsx": self._read_xlsx,
+            "xls": self._read_xls,
+            "pdf": self._read_pdf,
+            "csv": self._read_csv,
+            "txt": self._read_text,
+            "md": self._read_text,
+        }
+        reader = dispatch.get(suffix)
+        if reader is None:
+            return DocumentContent(
+                file_path=str(path), format=suffix, text="",
+                error=f"Unsupported format: .{suffix}"
+            )
+        try:
+            return reader(path)
+        except Exception as exc:
+            log.warning("Document read failed for %s: %s", path, exc)
+            return DocumentContent(
+                file_path=str(path), format=suffix, text="",
+                error=str(exc)
+            )
+
+    def _read_docx(self, path: Path) -> DocumentContent:
+        try:
+            import docx
+        except ImportError:
+            raise ImportError("python-docx required. pip install 'gdm-code[docs]'")
+        doc = docx.Document(str(path))
+        paragraphs = [p.text for p in doc.paragraphs if p.text.strip()]
+        for table in doc.tables:
+            for row in table.rows:
+                paragraphs.append("\t".join(c.text for c in row.cells))
+        props = doc.core_properties
+        return DocumentContent(
+            file_path=str(path), format="docx",
+            text="\n".join(paragraphs),
+            title=props.title or None,
+            author=props.author or None,
+        )
+
+    def _read_xlsx(self, path: Path) -> DocumentContent:
+        try:
+            import openpyxl
+        except ImportError:
+            raise ImportError("openpyxl required. pip install 'gdm-code[docs]'")
+        wb = openpyxl.load_workbook(str(path), read_only=True, data_only=True)
+        sheets, all_text = [], []
+        for ws in wb.worksheets:
+            rows = [[str(c.value) if c.value is not None else "" for c in row]
+                    for row in ws.iter_rows()]
+            headers = rows[0] if rows else []
+            data_rows = rows[1:] if len(rows) > 1 else []
+            sd = SheetData(name=ws.title, headers=headers, rows=data_rows)
+            sheets.append(sd)
+            all_text.append(f"[Sheet: {ws.title}]\n{sd.to_text()}")
+        wb.close()
+        return DocumentContent(
+            file_path=str(path), format="xlsx",
+            text="\n\n".join(all_text), sheets=sheets,
+        )
+
+    def _read_xls(self, path: Path) -> DocumentContent:
+        try:
+            import xlrd
+        except ImportError:
+            raise ImportError("xlrd required. pip install 'gdm-code[docs]'")
+        wb = xlrd.open_workbook(str(path))
+        sheets, all_text = [], []
+        for ws in wb.sheets():
+            rows = [[str(ws.cell_value(r, c)) for c in range(ws.ncols)]
+                    for r in range(ws.nrows)]
+            headers = rows[0] if rows else []
+            data_rows = rows[1:] if len(rows) > 1 else []
+            sd = SheetData(name=ws.name, headers=headers, rows=data_rows)
+            sheets.append(sd)
+            all_text.append(f"[Sheet: {ws.name}]\n{sd.to_text()}")
+        return DocumentContent(
+            file_path=str(path), format="xls",
+            text="\n\n".join(all_text), sheets=sheets,
+        )
+
+    def _read_pdf(self, path: Path) -> DocumentContent:
+        try:
+            import pdfplumber
+        except ImportError:
+            raise ImportError("pdfplumber required. pip install 'gdm-code[docs]'")
+        pages_text = []
+        with pdfplumber.open(str(path)) as pdf:
+            page_count = len(pdf.pages)
+            for page in pdf.pages:
+                t = page.extract_text() or ""
+                if t.strip():
+                    pages_text.append(t)
+        return DocumentContent(
+            file_path=str(path), format="pdf",
+            text="\n\n".join(pages_text),
+            page_count=page_count,
+        )
+
+    def _read_csv(self, path: Path) -> DocumentContent:
+        text = path.read_text(encoding="utf-8-sig", errors="replace")
+        reader = csv.reader(io.StringIO(text))
+        rows = list(reader)
+        headers = rows[0] if rows else []
+        data_rows = rows[1:] if len(rows) > 1 else []
+        sd = SheetData(name="Sheet1", headers=headers, rows=data_rows)
+        return DocumentContent(
+            file_path=str(path), format="csv",
+            text=sd.to_text(), sheets=[sd],
+        )
+
+    def _read_text(self, path: Path) -> DocumentContent:
+        text = path.read_text(encoding="utf-8", errors="replace")
+        return DocumentContent(
+            file_path=str(path), format=path.suffix.lstrip("."), text=text,
+        )

src/tools/document_tool.py

@@ -0,0 +1,240 @@
+"""Document tools — LLM-callable wrappers for document read/write/index/search.
+
+Registered tools:
+  - read_document   : read Word/Excel/PDF/CSV → text
+  - generate_document: create docx or xlsx from spec
+  - index_document  : index a document into the FTS search index
+  - search_documents: full-text search across indexed documents
+"""
+from __future__ import annotations
+import json, logging
+from pathlib import Path
+from typing import Any, ClassVar
+
+from src.tools import REGISTRY, ToolBase, ToolResult
+from src.tools.document_reader import DocumentReader
+from src.tools.document_writer import DocumentWriter, DocxSpec, XlsxSpec
+
+log = logging.getLogger(__name__)
+
+__all__ = [
+    "is_document_path",
+    "read_document_tool",
+    "generate_document_tool",
+    "DOCUMENT_EXTENSIONS",
+    "READ_DOCUMENT_SCHEMA",
+    "GENERATE_DOCUMENT_SCHEMA",
+]
+
+DOCUMENT_EXTENSIONS = frozenset({"docx", "xlsx", "xls", "pdf", "csv"})
+
+
+def is_document_path(path: str) -> bool:
+    return Path(path).suffix.lower().lstrip(".") in DOCUMENT_EXTENSIONS
+
+
+def read_document_tool(path: str, include_sheet_data: bool = False) -> str:
+    """Read a document and return its text content. Never raises."""
+    result = DocumentReader().read(Path(path))
+    if not result.success:
+        return f"Error reading {path}: {result.error}"
+    if include_sheet_data and result.sheets:
+        sheets_json = [
+            {"name": s.name, "headers": s.headers, "rows": s.rows}
+            for s in result.sheets
+        ]
+        return json.dumps({"text": result.text, "sheets": sheets_json}, indent=2)
+    meta = []
+    if result.title:
+        meta.append(f"Title: {result.title}")
+    if result.author:
+        meta.append(f"Author: {result.author}")
+    if result.page_count is not None:
+        meta.append(f"Pages: {result.page_count}")
+    header = "\n".join(meta)
+    return f"{header}\n\n{result.text}".strip() if header else result.text
+
+
+def generate_document_tool(format: str, output_path: str, spec: dict) -> str:
+    """Create a docx or xlsx from a spec dict. Never raises."""
+    writer = DocumentWriter()
+    out = Path(output_path)
+    if format == "docx":
+        doc_spec = DocxSpec(
+            title=spec.get("title", ""),
+            author=spec.get("author", ""),
+            sections=spec.get("sections", []),
+        )
+        result = writer.create_docx(doc_spec, out)
+    elif format == "xlsx":
+        xl_spec = XlsxSpec(sheets=spec.get("sheets", []))
+        result = writer.create_xlsx(xl_spec, out)
+    else:
+        return f"Error: unsupported format '{format}'. Use 'docx' or 'xlsx'."
+    if not result.success:
+        return f"Error generating {format}: {result.error}"
+    return f"Created {result.path} ({result.bytes_written:,} bytes)"
+
+
+def index_document_tool(path: str) -> str:
+    """Index a document for search. Returns chunk count."""
+    try:
+        from src.memory.document_index import DocumentIndex
+    except ImportError:
+        return "Error: document_index module not available."
+    count = DocumentIndex().index_document(path)
+    if count < 0:
+        return f"Error: could not index {path} (file not found or unreadable)"
+    return f"Indexed {path}: {count} chunks" if count else f"{path} already up to date"
+
+
+def search_documents_tool(query: str, limit: int = 10) -> str:
+    """Search all indexed documents for query. Returns ranked passages."""
+    try:
+        from src.memory.document_index import DocumentIndex
+    except ImportError:
+        return "Error: document_index module not available."
+    results = DocumentIndex().search(query, limit=limit)
+    if not results:
+        return f"No results for '{query}'"
+    lines = []
+    for r in results:
+        lines.append(f"[{r.file_path} — {r.source_label}]\n{r.snippet}")
+    return "\n\n".join(lines)
+
+
+# ---------------------------------------------------------------------------
+# LLM function-call schemas
+# ---------------------------------------------------------------------------
+
+READ_DOCUMENT_SCHEMA = {
+    "name": "read_document",
+    "description": "Read a Word (.docx), Excel (.xlsx/.xls), PDF, or CSV file and return its text content.",
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "path": {"type": "string", "description": "Absolute or relative path to the document"},
+            "include_sheet_data": {
+                "type": "boolean",
+                "description": "For spreadsheets: return JSON with per-sheet structured data",
+                "default": False,
+            },
+        },
+        "required": ["path"],
+    },
+}
+
+GENERATE_DOCUMENT_SCHEMA = {
+    "name": "generate_document",
+    "description": "Create a new Word (.docx) or Excel (.xlsx) document from structured content.",
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "format": {"type": "string", "enum": ["docx", "xlsx"]},
+            "output_path": {"type": "string", "description": "File path to write the document to"},
+            "spec": {
+                "type": "object",
+                "description": "Document spec. For docx: {title, sections:[{heading, paragraphs, table}]}. For xlsx: {sheets:[{name, headers, rows}]}",
+            },
+        },
+        "required": ["format", "output_path", "spec"],
+    },
+}
+
+
+# ---------------------------------------------------------------------------
+# ToolBase subclasses
+# ---------------------------------------------------------------------------
+
+class ReadDocumentTool(ToolBase):
+    """Read a Word, Excel, PDF, or CSV file."""
+
+    name: ClassVar[str] = "read_document"
+    description: ClassVar[str] = (
+        "Read a Word (.docx), Excel (.xlsx/.xls), PDF, or CSV file and return its text content. "
+        "For spreadsheets returns tab-separated table text per sheet."
+    )
+    input_schema: ClassVar[dict[str, Any]] = READ_DOCUMENT_SCHEMA["parameters"]
+
+    def execute(self, params: dict[str, Any]) -> ToolResult:
+        path = params["path"]
+        include_sheet_data = params.get("include_sheet_data", False)
+        text = read_document_tool(path, include_sheet_data=include_sheet_data)
+        if text.startswith("Error"):
+            return ToolResult(output="", error=text)
+        return ToolResult(output=text, metadata={"path": path})
+
+
+class GenerateDocumentTool(ToolBase):
+    """Create a Word or Excel document from structured content."""
+
+    name: ClassVar[str] = "generate_document"
+    description: ClassVar[str] = (
+        "Create a new Word (.docx) or Excel (.xlsx) document from structured content. "
+        "Returns the output file path and byte count."
+    )
+    input_schema: ClassVar[dict[str, Any]] = GENERATE_DOCUMENT_SCHEMA["parameters"]
+
+    def execute(self, params: dict[str, Any]) -> ToolResult:
+        fmt = params["format"]
+        output_path = params["output_path"]
+        spec = params.get("spec", {})
+        result = generate_document_tool(fmt, output_path, spec)
+        if result.startswith("Error"):
+            return ToolResult(output="", error=result)
+        return ToolResult(output=result)
+
+
+class IndexDocumentTool(ToolBase):
+    """Index a document for full-text search."""
+
+    name: ClassVar[str] = "index_document"
+    description: ClassVar[str] = (
+        "Index a document (Word, Excel, PDF, CSV) into the local search index. "
+        "Run this before searching for document content."
+    )
+    input_schema: ClassVar[dict[str, Any]] = {
+        "type": "object",
+        "properties": {
+            "path": {"type": "string", "description": "Path to the document to index"},
+        },
+        "required": ["path"],
+    }
+
+    def execute(self, params: dict[str, Any]) -> ToolResult:
+        result = index_document_tool(params["path"])
+        if result.startswith("Error"):
+            return ToolResult(output="", error=result)
+        return ToolResult(output=result)
+
+
+class SearchDocumentsTool(ToolBase):
+    """Full-text search across all indexed documents."""
+
+    name: ClassVar[str] = "search_documents"
+    description: ClassVar[str] = (
+        "Search all indexed documents for a query. "
+        "Returns ranked passages with source file and location."
+    )
+    input_schema: ClassVar[dict[str, Any]] = {
+        "type": "object",
+        "properties": {
+            "query": {"type": "string", "description": "Search query"},
+            "limit": {"type": "integer", "description": "Max results (default 10)", "default": 10},
+        },
+        "required": ["query"],
+    }
+
+    def execute(self, params: dict[str, Any]) -> ToolResult:
+        result = search_documents_tool(params["query"], limit=params.get("limit", 10))
+        return ToolResult(output=result)
+
+
+# ---------------------------------------------------------------------------
+# Auto-register
+# ---------------------------------------------------------------------------
+
+REGISTRY.register(ReadDocumentTool())
+REGISTRY.register(GenerateDocumentTool())
+REGISTRY.register(IndexDocumentTool())
+REGISTRY.register(SearchDocumentsTool())