nat-engine 1__py3-none-any.whl

mannf/product/ingestors/bgstm_ingestor.py

@@ -0,0 +1,764 @@
+# Copyright (C) 2026 Brad Guider
+# This file is part of NAT (Neural Agent Testing Framework).
+# Licensed under the AGPL-3.0. See LICENSE for details.
+# Commercial licensing available — see COMMERCIAL_LICENSE.md.
+
+"""BGSTM Test Plan ingestor plugin.
+
+Connects to a running BGSTM instance's REST API and translates its test plans,
+requirements, test cases, and traceability links into NAT's normalised
+:class:`~mannf.product.ingestors.models.IngestedEndpoint` and
+:class:`~mannf.product.ingestors.models.IngestedTestCase` dataclasses.
+
+This is **Phase 1** — a read-only API client that pulls structured data from
+BGSTM and feeds it into the NAT ingestor pipeline.  No bidirectional sync or
+methodology-aware intelligence yet.
+
+Authentication
+--------------
+Two modes are supported:
+
+* **API token** — supply ``api_token`` in the config.  The token is sent as
+  ``Authorization: Bearer <token>`` on every request.
+* **Username/password** — supply ``username`` and ``password``.  The ingestor
+  first calls ``POST /api/v1/auth/login`` to exchange credentials for a JWT,
+  then uses that JWT as a Bearer token.
+
+BGSTM API endpoints consumed
+-----------------------------
+* ``POST /api/v1/auth/login``         — obtain JWT token
+* ``GET  /api/v1/requirements``       — paginated list of requirements
+* ``GET  /api/v1/test-cases``         — paginated list of test cases
+* ``GET  /api/v1/traceability``       — full traceability matrix
+* ``GET  /api/v1/links``              — paginated requirement↔test-case links
+* ``GET  /api/v1/suggestions``        — AI-generated link suggestions (optional)
+
+Mapping
+-------
+* BGSTM **Requirement** → :class:`~mannf.product.ingestors.models.IngestedEndpoint`
+  (requirements carry endpoint context such as source URLs, acceptance criteria,
+  and methodology metadata).
+* BGSTM **TestCase** → :class:`~mannf.product.ingestors.models.IngestedTestCase`
+  (nearly 1-to-1 field mapping).
+* BGSTM **Link** → associates test cases with their parent requirements via
+  ``metadata["requirement_ids"]``.
+* BGSTM **Suggestion** → additional
+  :class:`~mannf.product.ingestors.models.IngestedTestCase` entries tagged with
+  ``source_hint="ai-suggestion"``.
+
+Priority mapping
+----------------
+BGSTM ``PriorityLevel`` values are mapped to NAT string priorities:
+
+========  =======  ===========
+BGSTM     NAT str  NAT float
+========  =======  ===========
+critical  critical 1.0
+high      high     0.8
+medium    medium   0.5
+low       low      0.2
+========  =======  ===========
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import urllib.error
+import urllib.parse
+import urllib.request
+from typing import Any
+
+from mannf.product.ingestors.base import IngestorPlugin
+from mannf.product.ingestors.models import (
+    IngestedEndpoint,
+    IngestedTestCase,
+    IngestorSource,
+    IngestResult,
+)
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+_API_PREFIX = "/api/v1"
+
+# BGSTM priority string → NAT string priority
+_PRIORITY_MAP: dict[str, str] = {
+    "critical": "critical",
+    "high": "high",
+    "medium": "medium",
+    "low": "low",
+}
+
+# BGSTM priority string → NAT float priority (stored in metadata)
+_PRIORITY_FLOAT_MAP: dict[str, float] = {
+    "critical": 1.0,
+    "high": 0.8,
+    "medium": 0.5,
+    "low": 0.2,
+}
+
+# Default page size for paginated BGSTM endpoints
+_PAGE_SIZE = 200
+
+
+# ---------------------------------------------------------------------------
+# Low-level HTTP helpers
+# ---------------------------------------------------------------------------
+
+
+def _json_request(
+    url: str,
+    headers: dict[str, str],
+    method: str = "GET",
+    body: dict[str, Any] | None = None,
+) -> Any:
+    """Make an HTTP request and return the parsed JSON response body.
+
+    Parameters
+    ----------
+    url:
+        Fully-qualified URL to request.
+    headers:
+        HTTP headers to include.
+    method:
+        HTTP method (default ``"GET"``).
+    body:
+        Optional JSON-serialisable request body (triggers POST encoding).
+
+    Returns
+    -------
+    Any
+        Parsed JSON response.
+
+    Raises
+    ------
+    urllib.error.HTTPError
+        On non-2xx HTTP responses.
+    urllib.error.URLError
+        On connection-level errors.
+    json.JSONDecodeError
+        If the response body cannot be parsed as JSON.
+    """
+    data: bytes | None = None
+    if body is not None:
+        data = json.dumps(body).encode("utf-8")
+        headers = dict(headers)
+        headers["Content-Type"] = "application/json"
+
+    req = urllib.request.Request(url, data=data, headers=headers, method=method)
+    with urllib.request.urlopen(req) as resp:  # noqa: S310
+        raw = resp.read()
+    return json.loads(raw)
+
+
+def _authenticate(base_url: str, config: dict) -> str:
+    """Return a Bearer token for use with subsequent BGSTM API calls.
+
+    Supports two modes:
+
+    * If ``api_token`` is present in *config*, it is returned directly.
+    * Otherwise ``username`` and ``password`` are exchanged for a JWT via
+      ``POST /api/v1/auth/login``.
+
+    Raises
+    ------
+    ValueError
+        If the login response is missing ``access_token``.
+    urllib.error.HTTPError / urllib.error.URLError
+        On HTTP / connection errors during login.
+    """
+    if config.get("api_token"):
+        return str(config["api_token"])
+
+    login_url = base_url.rstrip("/") + _API_PREFIX + "/auth/login"
+    payload = {"email": config["username"], "password": config["password"]}
+    resp = _json_request(login_url, headers={}, method="POST", body=payload)
+    token = resp.get("access_token") if isinstance(resp, dict) else None
+    if not token:
+        raise ValueError(
+            "BGSTM login response did not contain 'access_token'. "
+            f"Response: {resp!r}"
+        )
+    return str(token)
+
+
+def _fetch_paginated(base_url: str, path: str, headers: dict[str, str]) -> list[Any]:
+    """Fetch all pages of a BGSTM paginated endpoint.
+
+    Parameters
+    ----------
+    base_url:
+        BGSTM instance base URL (without trailing slash).
+    path:
+        API path relative to *base_url* (e.g. ``"/api/v1/requirements"``).
+    headers:
+        Auth headers to include on every request.
+
+    Returns
+    -------
+    list[Any]
+        Concatenated ``items`` from all pages.
+    """
+    items: list[Any] = []
+    page = 1
+    while True:
+        params = urllib.parse.urlencode({"page": page, "page_size": _PAGE_SIZE})
+        url = base_url.rstrip("/") + path + "?" + params
+        resp = _json_request(url, headers=headers)
+        if not isinstance(resp, dict):
+            break
+        page_items = resp.get("items", [])
+        if not isinstance(page_items, list):
+            break
+        items.extend(page_items)
+        total_pages = resp.get("pages", 1)
+        if page >= total_pages:
+            break
+        page += 1
+    return items
+
+
+# ---------------------------------------------------------------------------
+# Mapping helpers
+# ---------------------------------------------------------------------------
+
+
+def _map_priority(bgstm_priority: str | None) -> str:
+    """Convert a BGSTM priority string to a NAT priority string."""
+    if not bgstm_priority:
+        return "medium"
+    return _PRIORITY_MAP.get(bgstm_priority.lower(), "medium")
+
+
+def _priority_float(bgstm_priority: str | None) -> float:
+    """Return the NAT float priority score for a BGSTM priority string."""
+    if not bgstm_priority:
+        return 0.5
+    return _PRIORITY_FLOAT_MAP.get(bgstm_priority.lower(), 0.5)
+
+
+def _methodology_tags(metadata: dict[str, Any] | None) -> list[str]:
+    """Extract methodology/phase tags from BGSTM custom_metadata.
+
+    BGSTM requirements and test cases may carry arbitrary ``custom_metadata``
+    including keys such as ``methodology``, ``sprint``, and ``phase``.  This
+    function converts those well-known keys into NAT tag strings of the form
+    ``key:value`` so that NAT's belief-guided prioritiser can use them later.
+    """
+    if not metadata or not isinstance(metadata, dict):
+        return []
+    tags: list[str] = []
+    for key in ("methodology", "sprint", "phase"):
+        if key in metadata:
+            tags.append(f"{key}:{metadata[key]}")
+    return tags
+
+
+def _requirement_to_endpoint(req: dict[str, Any]) -> IngestedEndpoint:
+    """Map a BGSTM RequirementResponse dict to an :class:`IngestedEndpoint`.
+
+    Requirements in BGSTM carry rich context (acceptance criteria, source URL,
+    module, priority) that maps naturally to the ``metadata`` of an
+    :class:`IngestedEndpoint`.  The ``path`` is derived from the requirement's
+    ``source_url`` when present, or set to a synthetic ``/requirement/{id}``
+    fallback so that every endpoint has a non-empty path.
+    """
+    req_id = str(req.get("id", ""))
+    title = req.get("title", "")
+    description = req.get("description", "")
+    priority = req.get("priority", "medium")
+    status = req.get("status", "")
+    module = req.get("module", "")
+    tags_raw: list[str] = req.get("tags") or []
+    custom_metadata: dict[str, Any] = req.get("custom_metadata") or {}
+    source_url: str = req.get("source_url") or ""
+    source_system: str = req.get("source_system") or ""
+
+    # Build path from source_url if it looks like a URL path component
+    if source_url:
+        parsed = urllib.parse.urlparse(source_url)
+        path = parsed.path or source_url
+    else:
+        path = f"/requirement/{req_id}" if req_id else "/requirement/unknown"
+
+    tags: list[str] = list(tags_raw) + _methodology_tags(custom_metadata)
+    if module:
+        tags.append(f"module:{module}")
+    if status:
+        tags.append(f"status:{status}")
+    tags.append("bgstm:requirement")
+
+    metadata: dict[str, Any] = {
+        "bgstm_id": req_id,
+        "bgstm_type": "requirement",
+        "requirement_type": req.get("type", ""),
+        "priority_float": _priority_float(priority),
+        "status": status,
+        "module": module,
+        "source_system": source_system,
+        "source_url": source_url,
+        "external_id": req.get("external_id") or "",
+        "custom_metadata": custom_metadata,
+        "version": req.get("version"),
+    }
+
+    return IngestedEndpoint(
+        method="GET",
+        path=path,
+        summary=title,
+        description=description,
+        tags=tags,
+        metadata=metadata,
+    )
+
+
+def _test_case_to_ingest(
+    tc: dict[str, Any],
+    requirement_ids: list[str] | None = None,
+) -> IngestedTestCase:
+    """Map a BGSTM TestCaseResponse dict to an :class:`IngestedTestCase`.
+
+    Parameters
+    ----------
+    tc:
+        Raw BGSTM test case dict from the API.
+    requirement_ids:
+        List of requirement UUIDs that are linked to this test case via
+        traceability links.  Stored in ``metadata["requirement_ids"]`` so
+        that NAT's prioritiser can use the traceability context.
+    """
+    tc_id = str(tc.get("id", ""))
+    title = tc.get("title", "")
+    description = tc.get("description", "")
+    priority = tc.get("priority", "medium")
+    status = tc.get("status", "")
+    module = tc.get("module", "")
+    tags_raw: list[str] = tc.get("tags") or []
+    custom_metadata: dict[str, Any] = tc.get("custom_metadata") or {}
+    steps_raw: dict[str, Any] | None = tc.get("steps")
+    preconditions: str = tc.get("preconditions") or ""
+    postconditions: str = tc.get("postconditions") or ""
+
+    # Normalise steps from BGSTM's flexible dict format
+    steps: list[dict] = []
+    if steps_raw and isinstance(steps_raw, dict):
+        # Common BGSTM shape: {"1": "Step one", "2": "Step two", ...}
+        # or {"steps": [...]} or any other shape — we do a best-effort parse
+        if "steps" in steps_raw and isinstance(steps_raw["steps"], list):
+            for i, s in enumerate(steps_raw["steps"], 1):
+                if isinstance(s, str):
+                    steps.append({"step": i, "action": s})
+                elif isinstance(s, dict):
+                    steps.append(s)
+        else:
+            for key in sorted(str(k) for k in steps_raw.keys()):
+                steps.append({"step": key, "action": str(steps_raw[key])})
+    elif steps_raw and isinstance(steps_raw, list):
+        for i, s in enumerate(steps_raw, 1):
+            if isinstance(s, str):
+                steps.append({"step": i, "action": s})
+            elif isinstance(s, dict):
+                steps.append(s)
+
+    tags: list[str] = list(tags_raw) + _methodology_tags(custom_metadata)
+    if module:
+        tags.append(f"module:{module}")
+    if status:
+        tags.append(f"status:{status}")
+    tags.append("bgstm:test-case")
+
+    metadata: dict[str, Any] = {
+        "bgstm_id": tc_id,
+        "bgstm_type": "test_case",
+        "test_case_type": tc.get("type", ""),
+        "priority_float": _priority_float(priority),
+        "status": status,
+        "module": module,
+        "automation_status": tc.get("automation_status", ""),
+        "execution_time_minutes": tc.get("execution_time_minutes"),
+        "preconditions": preconditions,
+        "postconditions": postconditions,
+        "test_data": tc.get("test_data"),
+        "source_system": tc.get("source_system") or "",
+        "source_url": tc.get("source_url") or "",
+        "external_id": tc.get("external_id") or "",
+        "custom_metadata": custom_metadata,
+        "requirement_ids": requirement_ids or [],
+        "version": tc.get("version"),
+    }
+
+    return IngestedTestCase(
+        name=title,
+        description=description,
+        steps=steps,
+        expected_result=postconditions,
+        priority=_map_priority(priority),
+        tags=tags,
+        source_ref=f"bgstm:test-case:{tc_id}",
+        metadata=metadata,
+    )
+
+
+def _suggestion_to_ingest(sugg: dict[str, Any]) -> IngestedTestCase:
+    """Map a BGSTM SuggestionResponse dict to an :class:`IngestedTestCase`.
+
+    Suggestions are AI-generated traceability hints; they do not have their
+    own title or description but they do carry a ``similarity_score`` and a
+    ``suggestion_reason``.  We synthesise a name and mark the entry with a
+    ``source_hint="ai-suggestion"`` tag so that consumers can distinguish them
+    from hard test cases.
+    """
+    sugg_id = str(sugg.get("id", ""))
+    req_id = str(sugg.get("requirement_id", ""))
+    tc_id = str(sugg.get("test_case_id", ""))
+    score: float = float(sugg.get("similarity_score") or 0.0)
+    method: str = str(sugg.get("suggestion_method") or "unknown")
+    reason: str = sugg.get("suggestion_reason") or ""
+    status_val: str = str(sugg.get("status") or "pending")
+
+    name = f"AI Suggestion: req={req_id[:8]} ↔ tc={tc_id[:8]}"
+    description = reason or f"AI-generated link suggestion ({method}), score={score:.3f}"
+
+    tags = [
+        "bgstm:suggestion",
+        "source_hint:ai-suggestion",
+        f"suggestion_method:{method}",
+        f"suggestion_status:{status_val}",
+    ]
+
+    metadata: dict[str, Any] = {
+        "bgstm_id": sugg_id,
+        "bgstm_type": "suggestion",
+        "requirement_id": req_id,
+        "test_case_id": tc_id,
+        "similarity_score": score,
+        "suggestion_method": method,
+        "suggestion_reason": reason,
+        "status": status_val,
+        "reviewed_by": sugg.get("reviewed_by"),
+        "feedback": sugg.get("feedback"),
+    }
+
+    return IngestedTestCase(
+        name=name,
+        description=description,
+        priority="low",
+        tags=tags,
+        source_ref=f"bgstm:suggestion:{sugg_id}",
+        metadata=metadata,
+    )
+
+
+# ---------------------------------------------------------------------------
+# Build a requirement-id → test-case-id mapping from the traceability data
+# ---------------------------------------------------------------------------
+
+
+def _build_tc_to_req_map(
+    traceability: dict[str, Any] | None,
+    links: list[dict[str, Any]],
+) -> dict[str, list[str]]:
+    """Return a dict mapping test-case UUID → list of requirement UUIDs.
+
+    Data from both the ``/api/v1/traceability`` matrix and the ``/api/v1/links``
+    list are merged so that partial availability of either endpoint still
+    produces a useful result.
+    """
+    tc_to_reqs: dict[str, list[str]] = {}
+
+    # 1. From the traceability matrix
+    if traceability and isinstance(traceability, dict):
+        matrix = traceability.get("matrix") or []
+        for row in matrix:
+            req_id = str(row.get("requirement_id", ""))
+            for ltc in row.get("linked_test_cases") or []:
+                tc_id = str(ltc.get("test_case_id", ""))
+                if tc_id:
+                    tc_to_reqs.setdefault(tc_id, [])
+                    if req_id and req_id not in tc_to_reqs[tc_id]:
+                        tc_to_reqs[tc_id].append(req_id)
+
+    # 2. From the links list (may contain additional accepted/pending links)
+    for link in links:
+        req_id = str(link.get("requirement_id", ""))
+        tc_id = str(link.get("test_case_id", ""))
+        if tc_id and req_id:
+            tc_to_reqs.setdefault(tc_id, [])
+            if req_id not in tc_to_reqs[tc_id]:
+                tc_to_reqs[tc_id].append(req_id)
+
+    return tc_to_reqs
+
+
+# ---------------------------------------------------------------------------
+# BGSTMIngestor
+# ---------------------------------------------------------------------------
+
+
+class BGSTMIngestor(IngestorPlugin):
+    """Ingest test plans, requirements, and test cases from a BGSTM instance.
+
+    The ingestor connects to BGSTM's FastAPI REST API, authenticates, fetches
+    all available data, and maps it to NAT's unified
+    :class:`~mannf.product.ingestors.models.IngestResult`.
+
+    Config keys
+    -----------
+    base_url : str (required)
+        BGSTM instance base URL, e.g. ``"http://localhost:8000"``.
+    api_token : str (optional)
+        Pre-issued API/JWT token.  If supplied, ``username``/``password`` are
+        not required.
+    username : str (optional)
+        BGSTM user email.  Required when ``api_token`` is not supplied.
+    password : str (optional)
+        BGSTM user password.  Required when ``api_token`` is not supplied.
+    include_suggestions : bool (optional, default False)
+        Whether to fetch AI-generated link suggestions and include them as
+        extra :class:`~mannf.product.ingestors.models.IngestedTestCase` entries.
+    """
+
+    name = "bgstm"
+    display_name = "BGSTM Test Plan"
+    description = (
+        "Ingest requirements, test cases, and traceability links from a "
+        "BGSTM instance via its REST API."
+    )
+    supported_extensions: list[str] = []
+    supported_formats: list[str] = ["bgstm"]
+
+    # ------------------------------------------------------------------
+    # can_handle
+    # ------------------------------------------------------------------
+
+    def can_handle(self, source: IngestorSource) -> bool:
+        """Return ``True`` for BGSTM sources.
+
+        Recognises sources where:
+
+        * ``source.format`` is ``"bgstm"``, or
+        * ``source.metadata.get("format_hint")`` is ``"bgstm"``, or
+        * ``source.url`` contains a path segment matching the BGSTM API
+          prefix ``/api/v1/``.
+        """
+        if source.format in self.supported_formats:
+            return True
+        if source.metadata.get("format_hint") == "bgstm":
+            return True
+        if source.url:
+            return "/api/v1/" in source.url or source.url.rstrip("/").endswith("/api/v1")
+        return False
+
+    # ------------------------------------------------------------------
+    # validate_config
+    # ------------------------------------------------------------------
+
+    def validate_config(self, config: dict) -> list[str]:
+        """Validate BGSTMIngestor configuration.
+
+        Returns a list of human-readable error strings; empty means valid.
+        """
+        errors: list[str] = []
+        if not config.get("base_url"):
+            errors.append("'base_url' is required")
+        has_token = bool(config.get("api_token"))
+        has_creds = bool(config.get("username")) and bool(config.get("password"))
+        if not has_token and not has_creds:
+            errors.append(
+                "Either 'api_token' or both 'username' and 'password' are required"
+            )
+        return errors
+
+    # ------------------------------------------------------------------
+    # ingest
+    # ------------------------------------------------------------------
+
+    async def ingest(self, source: IngestorSource, config: dict) -> IngestResult:  # noqa: C901
+        """Ingest from a BGSTM instance and return a unified :class:`IngestResult`.
+
+        Steps
+        -----
+        1. Authenticate (API token or JWT login).
+        2. Fetch all requirements (paginated).
+        3. Fetch all test cases (paginated).
+        4. Fetch traceability matrix (best-effort).
+        5. Fetch links (paginated, best-effort).
+        6. Optionally fetch AI suggestions.
+        7. Map everything to NAT dataclasses and return.
+        """
+        errors: list[str] = []
+        warnings: list[str] = []
+
+        base_url: str = (config.get("base_url") or "").rstrip("/")
+        include_suggestions: bool = bool(config.get("include_suggestions", False))
+
+        # --- Step 1: Authenticate ---
+        try:
+            token = _authenticate(base_url, config)
+        except urllib.error.HTTPError as exc:
+            if exc.code == 401:
+                return self._make_error_result(
+                    source,
+                    "BGSTM authentication failed: invalid credentials (HTTP 401).",
+                )
+            return self._make_error_result(
+                source,
+                f"BGSTM authentication failed: HTTP {exc.code} {exc.reason}.",
+            )
+        except urllib.error.URLError as exc:
+            return self._make_error_result(
+                source,
+                f"Cannot connect to BGSTM at '{base_url}': {exc.reason}.",
+            )
+        except Exception as exc:  # noqa: BLE001
+            return self._make_error_result(
+                source,
+                f"BGSTM authentication error: {exc}",
+            )
+
+        auth_headers = {"Authorization": f"Bearer {token}"}
+
+        # --- Step 2: Fetch requirements ---
+        raw_requirements: list[dict] = []
+        try:
+            raw_requirements = _fetch_paginated(
+                base_url, _API_PREFIX + "/requirements", auth_headers
+            )
+        except urllib.error.HTTPError as exc:
+            if exc.code == 401:
+                return self._make_error_result(
+                    source, "BGSTM token rejected when fetching requirements (HTTP 401)."
+                )
+            warnings.append(
+                f"Could not fetch requirements: HTTP {exc.code} {exc.reason}. "
+                "Proceeding without requirements."
+            )
+        except urllib.error.URLError as exc:
+            warnings.append(
+                f"Could not fetch requirements: {exc.reason}. "
+                "Proceeding without requirements."
+            )
+        except Exception as exc:  # noqa: BLE001
+            warnings.append(f"Could not fetch requirements: {exc}. Proceeding without requirements.")
+
+        # --- Step 3: Fetch test cases ---
+        raw_test_cases: list[dict] = []
+        try:
+            raw_test_cases = _fetch_paginated(
+                base_url, _API_PREFIX + "/test-cases", auth_headers
+            )
+        except urllib.error.HTTPError as exc:
+            if exc.code == 401:
+                return self._make_error_result(
+                    source, "BGSTM token rejected when fetching test cases (HTTP 401)."
+                )
+            warnings.append(
+                f"Could not fetch test cases: HTTP {exc.code} {exc.reason}. "
+                "Proceeding without test cases."
+            )
+        except urllib.error.URLError as exc:
+            warnings.append(
+                f"Could not fetch test cases: {exc.reason}. "
+                "Proceeding without test cases."
+            )
+        except Exception as exc:  # noqa: BLE001
+            warnings.append(f"Could not fetch test cases: {exc}. Proceeding without test cases.")
+
+        # --- Step 4: Fetch traceability matrix (best-effort) ---
+        traceability_data: dict[str, Any] | None = None
+        try:
+            trace_url = base_url + _API_PREFIX + "/traceability"
+            traceability_data = _json_request(trace_url, headers=auth_headers)
+        except urllib.error.HTTPError as exc:
+            warnings.append(
+                f"Traceability endpoint unavailable: HTTP {exc.code} {exc.reason}. "
+                "Test cases will not have requirement associations from traceability."
+            )
+        except Exception as exc:  # noqa: BLE001
+            warnings.append(
+                f"Could not fetch traceability matrix: {exc}. "
+                "Test cases will not have requirement associations from traceability."
+            )
+
+        # --- Step 5: Fetch links (best-effort) ---
+        raw_links: list[dict] = []
+        try:
+            raw_links = _fetch_paginated(
+                base_url, _API_PREFIX + "/links", auth_headers
+            )
+        except Exception as exc:  # noqa: BLE001
+            warnings.append(
+                f"Could not fetch links: {exc}. "
+                "Test cases will not have requirement associations from links."
+            )
+
+        # --- Step 6: Optionally fetch AI suggestions ---
+        raw_suggestions: list[dict] = []
+        if include_suggestions:
+            try:
+                raw_suggestions = _fetch_paginated(
+                    base_url, _API_PREFIX + "/suggestions", auth_headers
+                )
+            except Exception as exc:  # noqa: BLE001
+                warnings.append(
+                    f"Could not fetch AI suggestions: {exc}. "
+                    "Proceeding without suggestions."
+                )
+
+        # --- Step 7: Build traceability map ---
+        tc_to_reqs = _build_tc_to_req_map(traceability_data, raw_links)
+
+        # --- Step 8: Map requirements → IngestedEndpoints ---
+        endpoints: list[IngestedEndpoint] = []
+        for req in raw_requirements:
+            try:
+                endpoints.append(_requirement_to_endpoint(req))
+            except Exception as exc:  # noqa: BLE001
+                req_id = req.get("id", "?")
+                warnings.append(f"Skipped requirement {req_id}: {exc}")
+
+        # --- Step 9: Map test cases → IngestedTestCases ---
+        test_cases: list[IngestedTestCase] = []
+        for tc in raw_test_cases:
+            try:
+                tc_id = str(tc.get("id", ""))
+                req_ids = tc_to_reqs.get(tc_id, [])
+                test_cases.append(_test_case_to_ingest(tc, requirement_ids=req_ids))
+            except Exception as exc:  # noqa: BLE001
+                tc_id_s = tc.get("id", "?")
+                warnings.append(f"Skipped test case {tc_id_s}: {exc}")
+
+        # --- Step 10: Map suggestions → IngestedTestCases ---
+        suggestion_count = 0
+        for sugg in raw_suggestions:
+            try:
+                test_cases.append(_suggestion_to_ingest(sugg))
+                suggestion_count += 1
+            except Exception as exc:  # noqa: BLE001
+                sugg_id = sugg.get("id", "?")
+                warnings.append(f"Skipped suggestion {sugg_id}: {exc}")
+
+        # --- Stats ---
+        stats: dict[str, Any] = {
+            "requirements_count": len(raw_requirements),
+            "endpoints_count": len(endpoints),
+            "test_cases_count": len(raw_test_cases),
+            "suggestions_count": suggestion_count,
+            "total_test_cases_count": len(test_cases),
+            "links_count": len(raw_links),
+            "base_url": base_url,
+        }
+
+        return IngestResult(
+            success=True,
+            source=source,
+            endpoints=endpoints,
+            test_cases=test_cases,
+            errors=errors,
+            warnings=warnings,
+            stats=stats,
+        )