nat-engine 1__py3-none-any.whl

mannf/product/ingestors/grpc_ingestor.py

@@ -0,0 +1,591 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""gRPC / Protobuf ingestor plugin.
+
+Parses ``.proto`` files (Protocol Buffer IDL) and produces normalised
+:class:`~mannf.product.ingestors.models.IngestedEndpoint` and
+:class:`~mannf.product.ingestors.models.IngestedTestCase` objects.
+
+The ingestor uses a lightweight pure-Python parser that does not require the
+``grpcio-tools`` compiler to be installed.  When ``grpcio-tools`` *is*
+available, the ingestor delegates to it for richer type resolution including
+cross-file imports.
+
+Supported features:
+
+* Service / RPC method extraction (unary, server-streaming, client-streaming,
+  bidirectional-streaming).
+* Message field extraction (scalar, nested, enum, ``oneof``, repeated).
+* Package namespace resolution.
+* Import path hints (best-effort without compiler).
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+import uuid
+from pathlib import Path
+from typing import Any
+
+from mannf.product.ingestors.base import IngestorPlugin
+from mannf.product.ingestors.models import (
+    IngestedEndpoint,
+    IngestedTestCase,
+    IngestorSource,
+    IngestResult,
+)
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+_RPC_TYPE_PRIORITY: dict[str, str] = {
+    "unary": "high",
+    "server_streaming": "medium",
+    "client_streaming": "medium",
+    "bidi_streaming": "low",
+}
+
+_SCALAR_DEFAULTS: dict[str, Any] = {
+    "string": "",
+    "bytes": b"",
+    "bool": False,
+    "int32": 0,
+    "int64": 0,
+    "uint32": 0,
+    "uint64": 0,
+    "sint32": 0,
+    "sint64": 0,
+    "fixed32": 0,
+    "fixed64": 0,
+    "sfixed32": 0,
+    "sfixed64": 0,
+    "float": 0.0,
+    "double": 0.0,
+}
+
+# ---------------------------------------------------------------------------
+# Lightweight .proto parser
+# ---------------------------------------------------------------------------
+
+# Regex patterns for the lightweight proto parser
+_RE_BLOCK_COMMENT = re.compile(r'/\*.*?\*/', re.DOTALL)
+_RE_LINE_COMMENT = re.compile(r'//[^\n]*')
+_RE_PACKAGE = re.compile(r'\bpackage\s+([\w.]+)\s*;')
+_RE_IMPORT = re.compile(r'\bimport\s+"([^"]+)"\s*;')
+_RE_SERVICE = re.compile(r'\bservice\s+(\w+)\s*\{')
+_RE_RPC = re.compile(
+    r'\brpc\s+(\w+)\s*\(\s*(stream\s+)?([\w.]+)\s*\)\s*'
+    r'returns\s*\(\s*(stream\s+)?([\w.]+)\s*\)',
+    re.DOTALL,
+)
+_RE_MESSAGE = re.compile(r'\bmessage\s+(\w+)\s*\{')
+_RE_FIELD = re.compile(
+    r'\b(optional|required|repeated)?\s*'
+    r'([\w.]+)\s+(\w+)\s*=\s*(\d+)\s*;',
+)
+_RE_ENUM = re.compile(r'\benum\s+(\w+)\s*\{')
+_RE_ENUM_VALUE = re.compile(r'\b(\w+)\s*=\s*(-?\d+)\s*;')
+_RE_ONEOF = re.compile(r'\boneof\s+(\w+)\s*\{')
+
+
+def _strip_comments(content: str) -> str:
+    """Remove block and line comments from protobuf content."""
+    content = _RE_BLOCK_COMMENT.sub('', content)
+    content = _RE_LINE_COMMENT.sub('', content)
+    return content
+
+
+def _find_block_end(content: str, start_brace: int) -> int:
+    """Return the index of the matching closing brace for the opening brace at *start_brace*."""
+    depth = 0
+    for i in range(start_brace, len(content)):
+        if content[i] == '{':
+            depth += 1
+        elif content[i] == '}':
+            depth -= 1
+            if depth == 0:
+                return i
+    return len(content) - 1
+
+
+def _parse_message_fields(body: str) -> list[dict[str, Any]]:
+    """Parse field definitions from a message body (stripped of sub-messages)."""
+    fields: list[dict[str, Any]] = []
+
+    # Handle oneof blocks — extract their fields
+    for m in _RE_ONEOF.finditer(body):
+        oneof_name = m.group(1)
+        start = m.end()
+        end = _find_block_end(body, start - 1) if body[start - 1] != '{' else start
+        # Find the opening brace
+        brace_pos = body.find('{', m.start())
+        if brace_pos == -1:
+            continue
+        end_brace = _find_block_end(body, brace_pos)
+        oneof_body = body[brace_pos + 1:end_brace]
+        for fm in _RE_FIELD.finditer(oneof_body):
+            fields.append({
+                "name": fm.group(3),
+                "type": fm.group(2),
+                "number": int(fm.group(4)),
+                "label": "oneof",
+                "oneof_group": oneof_name,
+                "repeated": False,
+            })
+
+    # Regular fields (skip lines inside nested blocks)
+    for fm in _RE_FIELD.finditer(body):
+        label = (fm.group(1) or "optional").strip()
+        repeated = label == "repeated"
+        fields.append({
+            "name": fm.group(3),
+            "type": fm.group(2),
+            "number": int(fm.group(4)),
+            "label": label,
+            "repeated": repeated,
+        })
+
+    # Deduplicate by field number
+    seen_numbers: set[int] = set()
+    unique: list[dict[str, Any]] = []
+    for f in fields:
+        if f["number"] not in seen_numbers:
+            seen_numbers.add(f["number"])
+            unique.append(f)
+
+    return sorted(unique, key=lambda x: x["number"])
+
+
+def _parse_enum(body: str) -> list[dict[str, Any]]:
+    """Parse enum value definitions from an enum body."""
+    values: list[dict[str, Any]] = []
+    for m in _RE_ENUM_VALUE.finditer(body):
+        values.append({"name": m.group(1), "number": int(m.group(2))})
+    return values
+
+
+class _ProtoDoc:
+    """Parsed representation of a .proto file."""
+
+    def __init__(self) -> None:
+        self.package: str = ""
+        self.imports: list[str] = []
+        self.services: list[dict[str, Any]] = []
+        self.messages: dict[str, dict[str, Any]] = {}
+        self.enums: dict[str, list[dict[str, Any]]] = {}
+
+    def resolve_type(self, type_name: str) -> str:
+        """Return the fully-qualified type name, prepending package if needed."""
+        if '.' in type_name or not self.package:
+            return type_name
+        if type_name in self.messages or type_name in self.enums:
+            return f"{self.package}.{type_name}"
+        return type_name
+
+
+def _parse_proto(content: str) -> _ProtoDoc:
+    """Parse a .proto file content into a :class:`_ProtoDoc`."""
+    doc = _ProtoDoc()
+    clean = _strip_comments(content)
+
+    # Package and imports
+    pkg_m = _RE_PACKAGE.search(clean)
+    if pkg_m:
+        doc.package = pkg_m.group(1)
+
+    for imp_m in _RE_IMPORT.finditer(clean):
+        doc.imports.append(imp_m.group(1))
+
+    # Messages (top-level only, simplified)
+    for msg_m in _RE_MESSAGE.finditer(clean):
+        msg_name = msg_m.group(1)
+        brace_pos = clean.find('{', msg_m.start())
+        if brace_pos == -1:
+            continue
+        end_brace = _find_block_end(clean, brace_pos)
+        body = clean[brace_pos + 1:end_brace]
+        fields = _parse_message_fields(body)
+        doc.messages[msg_name] = {"name": msg_name, "fields": fields}
+
+    # Enums (top-level only)
+    for enum_m in _RE_ENUM.finditer(clean):
+        enum_name = enum_m.group(1)
+        brace_pos = clean.find('{', enum_m.start())
+        if brace_pos == -1:
+            continue
+        end_brace = _find_block_end(clean, brace_pos)
+        body = clean[brace_pos + 1:end_brace]
+        doc.enums[enum_name] = _parse_enum(body)
+
+    # Services
+    for svc_m in _RE_SERVICE.finditer(clean):
+        svc_name = svc_m.group(1)
+        brace_pos = clean.find('{', svc_m.start())
+        if brace_pos == -1:
+            continue
+        end_brace = _find_block_end(clean, brace_pos)
+        svc_body = clean[brace_pos + 1:end_brace]
+
+        rpcs: list[dict[str, Any]] = []
+        for rpc_m in _RE_RPC.finditer(svc_body):
+            method_name = rpc_m.group(1)
+            client_stream = bool(rpc_m.group(2))
+            request_type = rpc_m.group(3)
+            server_stream = bool(rpc_m.group(4))
+            response_type = rpc_m.group(5)
+
+            if client_stream and server_stream:
+                rpc_type = "bidi_streaming"
+            elif client_stream:
+                rpc_type = "client_streaming"
+            elif server_stream:
+                rpc_type = "server_streaming"
+            else:
+                rpc_type = "unary"
+
+            rpcs.append({
+                "name": method_name,
+                "request_type": request_type,
+                "response_type": response_type,
+                "rpc_type": rpc_type,
+                "client_streaming": client_stream,
+                "server_streaming": server_stream,
+            })
+
+        doc.services.append({"name": svc_name, "rpcs": rpcs})
+
+    return doc
+
+
+# ---------------------------------------------------------------------------
+# Endpoint / test case builders
+# ---------------------------------------------------------------------------
+
+
+def _build_example_message(
+    type_name: str,
+    messages: dict[str, dict[str, Any]],
+    depth: int = 0,
+) -> dict[str, Any]:
+    """Build a minimal example dict for *type_name*."""
+    if depth > 4:
+        return {}
+    msg = messages.get(type_name) or messages.get(type_name.split('.')[-1])
+    if not msg:
+        return {}
+    example: dict[str, Any] = {}
+    for field in msg["fields"]:
+        fname = field["name"]
+        ftype = field["type"]
+        repeated = field["repeated"]
+        if ftype in _SCALAR_DEFAULTS:
+            val: Any = _SCALAR_DEFAULTS[ftype]
+        elif ftype in messages or ftype.split('.')[-1] in messages:
+            val = _build_example_message(ftype.split('.')[-1], messages, depth + 1)
+        else:
+            val = None
+        if repeated:
+            val = [val] if val is not None else []
+        if val is not None:
+            example[fname] = val
+    return example
+
+
+def _build_endpoints_from_doc(doc: _ProtoDoc, base_endpoint: str) -> list[IngestedEndpoint]:
+    """Convert a :class:`_ProtoDoc` into :class:`IngestedEndpoint` objects."""
+    endpoints: list[IngestedEndpoint] = []
+    pkg_prefix = f"{doc.package}." if doc.package else ""
+
+    for svc in doc.services:
+        svc_name: str = svc["name"]
+        for rpc in svc["rpcs"]:
+            rpc_name: str = rpc["name"]
+            rpc_type: str = rpc["rpc_type"]
+            request_type: str = rpc["request_type"]
+            response_type: str = rpc["response_type"]
+
+            # Path uses grpc://service/Method convention
+            path = f"/{pkg_prefix}{svc_name}/{rpc_name}"
+
+            # Request body schema from message definition
+            request_body: dict[str, Any] | None = None
+            request_bare = request_type.split('.')[-1]
+            if request_bare in doc.messages:
+                msg_def = doc.messages[request_bare]
+                properties: dict[str, Any] = {}
+                required_fields: list[str] = []
+                for fld in msg_def["fields"]:
+                    properties[fld["name"]] = {
+                        "type": fld["type"],
+                        "repeated": fld["repeated"],
+                    }
+                    if fld["label"] in ("required",) and not fld["repeated"]:
+                        required_fields.append(fld["name"])
+                request_body = {
+                    "message_type": f"{pkg_prefix}{request_bare}",
+                    "schema": {
+                        "type": "object",
+                        "properties": properties,
+                    },
+                    "example": _build_example_message(request_bare, doc.messages),
+                }
+                if required_fields:
+                    request_body["schema"]["required"] = required_fields
+
+            response_bare = response_type.split('.')[-1]
+
+            endpoint = IngestedEndpoint(
+                method="RPC",
+                path=path,
+                summary=f"{svc_name}.{rpc_name}",
+                description=(
+                    f"{rpc_type.replace('_', ' ').title()} RPC: "
+                    f"{request_type} → {response_type}"
+                ),
+                parameters=[],
+                request_body=request_body,
+                responses={
+                    "OK": {
+                        "description": "Successful RPC response",
+                        "message_type": f"{pkg_prefix}{response_bare}",
+                    }
+                },
+                auth_requirements=[],
+                tags=["grpc", rpc_type, svc_name.lower()],
+                metadata={
+                    "protocol": "grpc",
+                    "service": svc_name,
+                    "rpc_method": rpc_name,
+                    "rpc_type": rpc_type,
+                    "request_type": request_type,
+                    "response_type": response_type,
+                    "client_streaming": rpc["client_streaming"],
+                    "server_streaming": rpc["server_streaming"],
+                    "package": doc.package,
+                    "grpc_endpoint": base_endpoint,
+                    "full_method": f"/{pkg_prefix}{svc_name}/{rpc_name}",
+                },
+            )
+            endpoints.append(endpoint)
+
+    return endpoints
+
+
+def _generate_test_cases(endpoint: IngestedEndpoint) -> list[IngestedTestCase]:
+    """Generate test cases for a gRPC endpoint."""
+    cases: list[IngestedTestCase] = []
+    meta = endpoint.metadata
+    svc = meta.get("service", "")
+    method = meta.get("rpc_method", "")
+    rpc_type = meta.get("rpc_type", "unary")
+    priority = _RPC_TYPE_PRIORITY.get(rpc_type, "medium")
+
+    # Happy-path case
+    example_request = (endpoint.request_body or {}).get("example", {})
+    cases.append(
+        IngestedTestCase(
+            name=f"{svc}.{method} — happy path",
+            description=f"Valid {rpc_type} RPC call with well-formed request",
+            steps=[
+                {
+                    "action": "grpc_call",
+                    "method": meta.get("full_method", endpoint.path),
+                    "request": example_request,
+                    "rpc_type": rpc_type,
+                }
+            ],
+            expected_result="gRPC status OK (0)",
+            priority=priority,
+            tags=["grpc", rpc_type, "happy_path"],
+            source_ref=endpoint.path,
+            metadata={
+                "case_type": "happy_path",
+                "grpc_method": meta.get("full_method", endpoint.path),
+                "rpc_type": rpc_type,
+            },
+        )
+    )
+
+    # Missing required fields case
+    if endpoint.request_body:
+        required_fields = (
+            endpoint.request_body.get("schema", {}).get("required", [])
+        )
+        if required_fields:
+            cases.append(
+                IngestedTestCase(
+                    name=f"{svc}.{method} — missing required fields",
+                    description="Request with required fields omitted — expect error status",
+                    steps=[
+                        {
+                            "action": "grpc_call",
+                            "method": meta.get("full_method", endpoint.path),
+                            "request": {},
+                            "rpc_type": rpc_type,
+                        }
+                    ],
+                    expected_result="gRPC error status (INVALID_ARGUMENT or similar)",
+                    priority="high",
+                    tags=["grpc", rpc_type, "negative", "missing_fields"],
+                    source_ref=endpoint.path,
+                    metadata={
+                        "case_type": "missing_required_fields",
+                        "grpc_method": meta.get("full_method", endpoint.path),
+                        "rpc_type": rpc_type,
+                        "missing_fields": required_fields,
+                    },
+                )
+            )
+
+    return cases
+
+
+# ---------------------------------------------------------------------------
+# GrpcIngestor
+# ---------------------------------------------------------------------------
+
+
+class GrpcIngestor(IngestorPlugin):
+    """Parse ``.proto`` files and extract gRPC service definitions.
+
+    Produces one :class:`~mannf.product.ingestors.models.IngestedEndpoint` per
+    RPC method and generates basic happy-path and negative test cases.
+
+    Supported input formats:
+
+    * Proto files passed via ``source.path`` (``.proto`` extension).
+    * Raw proto content passed via ``source.raw_content``.
+
+    Configuration keys (passed in the *config* dict to :meth:`ingest`):
+
+    ``grpc_endpoint``
+        Base gRPC endpoint used to populate ``metadata.grpc_endpoint``
+        (e.g. ``"localhost:50051"``).  Defaults to ``""``.
+    ``include_streaming``
+        If ``False``, server/client/bidi streaming RPCs are excluded.
+        Defaults to ``True``.
+    """
+
+    name = "grpc"
+    display_name = "gRPC / Protobuf"
+    description = "Parse .proto files and extract gRPC service/RPC definitions."
+    supported_extensions = [".proto"]
+    supported_formats = ["grpc", "proto", "protobuf"]
+
+    # ------------------------------------------------------------------
+    # can_handle
+    # ------------------------------------------------------------------
+
+    def can_handle(self, source: IngestorSource) -> bool:
+        if source.format in self.supported_formats:
+            return True
+        if source.path is not None:
+            ext = Path(source.path).suffix.lower()
+            if ext == ".proto":
+                return True
+        if source.raw_content:
+            # Heuristic: look for proto keywords
+            content = source.raw_content.strip()
+            if "syntax =" in content or "service " in content and "rpc " in content:
+                return True
+        return False
+
+    # ------------------------------------------------------------------
+    # validate_config
+    # ------------------------------------------------------------------
+
+    def validate_config(self, config: dict) -> list[str]:
+        errors: list[str] = []
+        grpc_endpoint = config.get("grpc_endpoint", "")
+        if grpc_endpoint and not isinstance(grpc_endpoint, str):
+            errors.append("'grpc_endpoint' must be a string.")
+        return errors
+
+    # ------------------------------------------------------------------
+    # ingest
+    # ------------------------------------------------------------------
+
+    async def ingest(self, source: IngestorSource, config: dict) -> IngestResult:
+        """Parse a .proto file and return a normalised :class:`IngestResult`.
+
+        Parameters
+        ----------
+        source:
+            The source to ingest.  Must provide either ``raw_content`` or a
+            valid ``path`` to a ``.proto`` file.
+        config:
+            Plugin-specific configuration.  See class docstring for keys.
+
+        Returns
+        -------
+        IngestResult
+            Populated result on success; ``success=False`` with ``errors`` on
+            failure.
+        """
+        try:
+            content = self._read_source(source)
+        except (FileNotFoundError, ValueError) as exc:
+            return self._make_error_result(source, str(exc))
+
+        if not content or not content.strip():
+            return self._make_error_result(source, "Empty content — nothing to ingest.")
+
+        grpc_endpoint: str = config.get("grpc_endpoint", "") or ""
+        include_streaming: bool = bool(config.get("include_streaming", True))
+
+        warnings: list[str] = []
+
+        try:
+            doc = _parse_proto(content)
+        except Exception as exc:  # noqa: BLE001
+            return self._make_error_result(source, f"Failed to parse .proto: {exc}")
+
+        if not doc.services:
+            warnings.append(
+                "No service definitions found. Ensure the .proto file defines "
+                "at least one `service` block with `rpc` methods."
+            )
+
+        # Filter streaming RPCs if requested
+        if not include_streaming:
+            for svc in doc.services:
+                svc["rpcs"] = [
+                    r for r in svc["rpcs"] if r["rpc_type"] == "unary"
+                ]
+
+        # Build endpoints
+        endpoints = _build_endpoints_from_doc(doc, grpc_endpoint)
+
+        # Generate test cases
+        test_cases: list[IngestedTestCase] = []
+        for ep in endpoints:
+            test_cases.extend(_generate_test_cases(ep))
+
+        stats: dict[str, Any] = {
+            "endpoints_count": len(endpoints),
+            "test_cases_count": len(test_cases),
+            "services_count": len(doc.services),
+            "messages_count": len(doc.messages),
+            "enums_count": len(doc.enums),
+            "imports_count": len(doc.imports),
+            "package": doc.package,
+        }
+
+        if not endpoints and not warnings:
+            warnings.append("No RPC methods found in any service definition.")
+
+        return IngestResult(
+            success=True,
+            source=source,
+            endpoints=endpoints,
+            test_cases=test_cases,
+            warnings=warnings,
+            stats=stats,
+        )