nat-engine 1__py3-none-any.whl

mannf/product/ingestors/loader.py

@@ -0,0 +1,284 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Plugin loader for custom IngestorPlugin implementations.
+
+Provides two discovery mechanisms:
+  - Directory scanning: load .py files from a directory path.
+  - Entry-point discovery: use importlib.metadata to find registered ingestors.
+
+Both mechanisms validate that each discovered class has the required attributes
+and callable methods before instantiating it.  Invalid ingestors are skipped
+with a warning — they will never cause the scanner to crash.
+"""
+
+from __future__ import annotations
+
+import importlib
+import importlib.util
+import inspect
+import logging
+import sys
+from importlib.metadata import entry_points
+from pathlib import Path
+
+from mannf.product.ingestors.base import IngestorPlugin
+from mannf.product.ingestors.models import IngestorSource
+
+logger = logging.getLogger(__name__)
+
+_REQUIRED_ATTRS = ("name", "display_name", "description", "supported_extensions", "supported_formats")
+
+
+def _is_valid_ingestor(cls: type) -> bool:
+    """Return True if *cls* is a valid, concrete IngestorPlugin subclass."""
+    if not (isinstance(cls, type) and issubclass(cls, IngestorPlugin) and cls is not IngestorPlugin):
+        return False
+    for attr in _REQUIRED_ATTRS:
+        if not hasattr(cls, attr):
+            logger.warning(
+                "Ingestor %s is missing required attribute '%s' — skipping.",
+                cls.__name__,
+                attr,
+            )
+            return False
+    if not callable(getattr(cls, "ingest", None)):
+        logger.warning(
+            "Ingestor %s has no callable 'ingest' method — skipping.",
+            cls.__name__,
+        )
+        return False
+    if not callable(getattr(cls, "validate_config", None)):
+        logger.warning(
+            "Ingestor %s has no callable 'validate_config' method — skipping.",
+            cls.__name__,
+        )
+        return False
+    if not callable(getattr(cls, "can_handle", None)):
+        logger.warning(
+            "Ingestor %s has no callable 'can_handle' method — skipping.",
+            cls.__name__,
+        )
+        return False
+    return True
+
+
+def _instantiate(cls: type) -> IngestorPlugin | None:
+    """Try to instantiate *cls*; log a warning and return None on failure."""
+    try:
+        return cls()
+    except Exception as exc:  # noqa: BLE001
+        logger.warning(
+            "Ingestor %s raised %s during __init__ — skipping.", cls.__name__, exc
+        )
+        return None
+
+
+def load_ingestors_from_directory(directory: Path) -> list[IngestorPlugin]:
+    """Discover and load IngestorPlugin subclasses from all ``.py`` files in *directory*.
+
+    Parameters
+    ----------
+    directory:
+        Path to a directory containing ingestor ``.py`` files.
+
+    Returns
+    -------
+    list[IngestorPlugin]
+        Instantiated ingestor objects for every valid ingestor found.
+    """
+    ingestors: list[IngestorPlugin] = []
+
+    if not directory.exists():
+        logger.warning(
+            "Ingestor directory '%s' does not exist — no ingestors loaded.", directory
+        )
+        return ingestors
+
+    if not directory.is_dir():
+        logger.warning(
+            "Ingestor path '%s' is not a directory — no ingestors loaded.", directory
+        )
+        return ingestors
+
+    for py_file in sorted(directory.glob("*.py")):
+        module_name = f"_nat_ingestor_{py_file.stem}"
+        try:
+            spec = importlib.util.spec_from_file_location(module_name, py_file)
+            if spec is None or spec.loader is None:
+                logger.warning(
+                    "Could not create module spec for '%s' — skipping.", py_file
+                )
+                continue
+            module = importlib.util.module_from_spec(spec)
+            sys.modules[module_name] = module
+            spec.loader.exec_module(module)  # type: ignore[union-attr]
+        except Exception as exc:  # noqa: BLE001
+            logger.warning(
+                "Failed to import ingestor file '%s': %s — skipping.", py_file, exc
+            )
+            continue
+
+        for _name, obj in inspect.getmembers(module, inspect.isclass):
+            if obj.__module__ != module_name:
+                # Avoid re-processing imported base classes
+                continue
+            if _is_valid_ingestor(obj):
+                instance = _instantiate(obj)
+                if instance is not None:
+                    ingestors.append(instance)
+
+    return ingestors
+
+
+def load_ingestors_from_entry_points(
+    group: str = "nat.ingestors",
+) -> list[IngestorPlugin]:
+    """Discover and load IngestorPlugin subclasses registered as package entry points.
+
+    Parameters
+    ----------
+    group:
+        The entry-point group name to scan (default ``"nat.ingestors"``).
+
+    Returns
+    -------
+    list[IngestorPlugin]
+        Instantiated ingestor objects for every valid entry point found.
+    """
+    ingestors: list[IngestorPlugin] = []
+
+    try:
+        eps = entry_points(group=group)
+    except Exception as exc:  # noqa: BLE001
+        logger.warning(
+            "Failed to query entry points for group '%s': %s", group, exc
+        )
+        return ingestors
+
+    for ep in eps:
+        try:
+            obj = ep.load()
+        except Exception as exc:  # noqa: BLE001
+            logger.warning(
+                "Failed to load entry point '%s': %s — skipping.", ep.name, exc
+            )
+            continue
+
+        if _is_valid_ingestor(obj):
+            instance = _instantiate(obj)
+            if instance is not None:
+                ingestors.append(instance)
+        else:
+            logger.warning(
+                "Entry point '%s' does not point to a valid IngestorPlugin subclass — skipping.",
+                ep.name,
+            )
+
+    return ingestors
+
+
+def load_all_ingestors(
+    ingestors_dir: Path | None = None,
+    entry_point_group: str = "nat.ingestors",
+) -> list[IngestorPlugin]:
+    """Load ingestors from built-ins, a directory, and registered entry points.
+
+    Results are deduplicated by class name; built-in ingestors take precedence.
+
+    Parameters
+    ----------
+    ingestors_dir:
+        Optional path to a directory of ingestor ``.py`` files.
+    entry_point_group:
+        Entry-point group to scan (default ``"nat.ingestors"``).
+
+    Returns
+    -------
+    list[IngestorPlugin]
+        Merged, deduplicated list of instantiated ingestor objects.
+    """
+    from mannf.product.ingestors import BUILTIN_INGESTORS  # avoid circular import
+
+    seen_class_names: set[str] = set()
+    combined: list[IngestorPlugin] = []
+
+    dir_ingestors = (
+        load_ingestors_from_directory(ingestors_dir) if ingestors_dir else []
+    )
+    ep_ingestors = load_ingestors_from_entry_points(entry_point_group)
+
+    for ingestor in BUILTIN_INGESTORS + dir_ingestors + ep_ingestors:
+        class_name = type(ingestor).__name__
+        if class_name not in seen_class_names:
+            seen_class_names.add(class_name)
+            combined.append(ingestor)
+
+    return combined
+
+
+def get_ingestor_by_name(
+    name: str,
+    ingestors: list[IngestorPlugin] | None = None,
+) -> IngestorPlugin | None:
+    """Look up an ingestor by its ``name`` attribute.
+
+    Parameters
+    ----------
+    name:
+        The ingestor name to search for (e.g., ``"openapi"``, ``"postman"``).
+    ingestors:
+        List of instantiated ingestor objects to search.  Defaults to
+        :func:`load_all_ingestors` if not provided.
+
+    Returns
+    -------
+    IngestorPlugin | None
+        The first matching ingestor, or ``None`` if not found.
+    """
+    if ingestors is None:
+        ingestors = load_all_ingestors()
+    for ingestor in ingestors:
+        if ingestor.name == name:
+            return ingestor
+    return None
+
+
+def get_ingestor_for_source(
+    source: IngestorSource,
+    ingestors: list[IngestorPlugin] | None = None,
+) -> IngestorPlugin | None:
+    """Auto-detect and return the appropriate ingestor for *source*.
+
+    Iterates through the available ingestors and returns the first one whose
+    :meth:`~IngestorPlugin.can_handle` method returns True.
+
+    Parameters
+    ----------
+    source:
+        The source to find an ingestor for.
+    ingestors:
+        List of instantiated ingestor objects to search.  Defaults to
+        :func:`load_all_ingestors` if not provided.
+
+    Returns
+    -------
+    IngestorPlugin | None
+        The first matching ingestor, or ``None`` if no ingestor can handle
+        the source.
+    """
+    if ingestors is None:
+        ingestors = load_all_ingestors()
+    for ingestor in ingestors:
+        try:
+            if ingestor.can_handle(source):
+                return ingestor
+        except Exception as exc:  # noqa: BLE001
+            logger.warning(
+                "Ingestor '%s' raised an error in can_handle: %s — skipping.",
+                ingestor.name,
+                exc,
+            )
+    return None

mannf/product/ingestors/models.py

@@ -0,0 +1,146 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""Dataclass models for the ingestor plugin architecture.
+
+These models represent the unified internal format that all ingestor plugins
+produce when consuming specs, test cases, and other artifacts.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class IngestorSource:
+    """Metadata about the source being ingested.
+
+    Attributes
+    ----------
+    format:
+        Source format identifier (e.g., ``"openapi"``, ``"postman"``,
+        ``"har"``, ``"graphql"``, ``"gherkin"``, ``"bgstm"``, ``"curl"``).
+    path:
+        File path if the source is a local file.
+    url:
+        URL if the source is a remote resource.
+    raw_content:
+        Raw content string if provided inline.
+    metadata:
+        Arbitrary metadata from the source.
+    """
+
+    format: str
+    path: str | None = None
+    url: str | None = None
+    raw_content: str | None = None
+    metadata: dict = field(default_factory=dict)
+
+
+@dataclass
+class IngestedEndpoint:
+    """A single API endpoint/operation extracted from an ingested source.
+
+    Attributes
+    ----------
+    method:
+        HTTP method (GET, POST, PUT, DELETE, etc.).
+    path:
+        URL path template (e.g., ``"/api/v1/users/{id}"``).
+    summary:
+        Human-readable summary of the operation.
+    description:
+        Detailed description of the operation.
+    parameters:
+        Parameter definitions (query, path, header, cookie).
+    request_body:
+        Request body schema or example, if applicable.
+    responses:
+        Expected response schemas keyed by HTTP status code.
+    auth_requirements:
+        Authentication schemes required for this endpoint.
+    tags:
+        Categorization tags.
+    metadata:
+        Format-specific extra data.
+    """
+
+    method: str
+    path: str
+    summary: str = ""
+    description: str = ""
+    parameters: list[dict] = field(default_factory=list)
+    request_body: dict | None = None
+    responses: dict = field(default_factory=dict)
+    auth_requirements: list[str] = field(default_factory=list)
+    tags: list[str] = field(default_factory=list)
+    metadata: dict = field(default_factory=dict)
+
+
+@dataclass
+class IngestedTestCase:
+    """A test case extracted from an ingested source (e.g., BDD/Gherkin/BGSTM).
+
+    Attributes
+    ----------
+    name:
+        Test case name.
+    description:
+        Test case description.
+    steps:
+        Ordered list of test steps.
+    expected_result:
+        Expected outcome of the test.
+    priority:
+        Priority level (``"critical"``, ``"high"``, ``"medium"``, ``"low"``).
+    tags:
+        Categorization tags.
+    source_ref:
+        Reference back to the source (file:line, scenario name, etc.).
+    metadata:
+        Format-specific extra data.
+    """
+
+    name: str
+    description: str = ""
+    steps: list[dict] = field(default_factory=list)
+    expected_result: str = ""
+    priority: str = "medium"
+    tags: list[str] = field(default_factory=list)
+    source_ref: str = ""
+    metadata: dict = field(default_factory=dict)
+
+
+@dataclass
+class IngestResult:
+    """The complete output of an ingestion operation.
+
+    Attributes
+    ----------
+    success:
+        Whether ingestion succeeded.
+    source:
+        The source that was ingested.
+    endpoints:
+        Extracted API endpoints.
+    test_cases:
+        Extracted test cases.
+    errors:
+        Any errors encountered during ingestion.
+    warnings:
+        Any warnings (non-fatal issues).
+    stats:
+        Summary statistics (e.g., ``endpoint_count``, ``test_case_count``,
+        ``duration_ms``).
+    """
+
+    success: bool
+    source: IngestorSource
+    endpoints: list[IngestedEndpoint] = field(default_factory=list)
+    test_cases: list[IngestedTestCase] = field(default_factory=list)
+    errors: list[str] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+    stats: dict = field(default_factory=dict)