cortexdb-connectors 0.2.0__py3-none-any.whl

cortexdb_connectors/servicenow/__init__.py

@@ -0,0 +1,460 @@
+"""
+ServiceNow connector for CortexDB.
+
+Polls ServiceNow Table API for Incidents, Change Requests, Problems,
+Knowledge Articles, and Tasks, converting them into CortexDB Episodes.
+Uses ``httpx`` for direct HTTP access with offset-based pagination
+(``sysparm_offset``).
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+import httpx
+
+from cortexdb_connectors.base import (
+    Actor,
+    CortexConnector,
+    EntityRef,
+    Episode,
+    EpisodeType,
+    RawEvent,
+    Source,
+    Visibility,
+    VisibilityLevel,
+)
+
+logger = logging.getLogger(__name__)
+
+_SOURCE = Source(system="servicenow")
+
+_DEFAULT_TABLES = frozenset(
+    ["incident", "change_request", "problem", "kb_knowledge", "sc_task"]
+)
+
+_TABLE_TYPE_MAP: dict[str, EpisodeType] = {
+    "incident": EpisodeType.INCIDENT,
+    "change_request": EpisodeType.DEPLOYMENT,
+    "problem": EpisodeType.ISSUE,
+    "kb_knowledge": EpisodeType.DOCUMENT,
+    "sc_task": EpisodeType.ISSUE,
+}
+
+_MAX_RETRIES = 3
+_PAGE_SIZE = 100
+
+
+class ServiceNowConnector(CortexConnector):
+    """Connector that syncs ServiceNow records into CortexDB.
+
+    Parameters
+    ----------
+    cortex_url:
+        Base URL of the CortexDB API.
+    cortex_api_key:
+        Bearer token for CortexDB authentication.
+    instance:
+        ServiceNow instance name (e.g. ``mycompany`` for
+        mycompany.service-now.com) or full URL.
+    username:
+        ServiceNow username for basic auth.
+    password:
+        ServiceNow password for basic auth.
+    tables:
+        ServiceNow table names to ingest.  Defaults to incident,
+        change_request, problem, kb_knowledge, sc_task.
+    namespace:
+        CortexDB namespace for ingested episodes.
+    backfill_days:
+        Number of days of historical data to backfill on first sync.
+    tenant_id:
+        CortexDB tenant identifier.
+    """
+
+    connector_name: str = "servicenow"
+
+    def __init__(
+        self,
+        cortex_url: str,
+        cortex_api_key: str,
+        instance: str,
+        username: str,
+        password: str,
+        tables: list[str] | None = None,
+        namespace: str = "servicenow",
+        backfill_days: int = 90,
+        tenant_id: str = "default",
+    ) -> None:
+        super().__init__(cortex_url, cortex_api_key, tenant_id)
+        # Accept either "mycompany" or "https://mycompany.service-now.com".
+        if instance.startswith("http"):
+            self._base_url = instance.rstrip("/")
+        else:
+            self._base_url = f"https://{instance}.service-now.com"
+        self.username = username
+        self.password = password
+        self.table_filter = set(tables) if tables else set(_DEFAULT_TABLES)
+        self.namespace = namespace
+        self.backfill_days = backfill_days
+
+    # -- HTTP helpers --------------------------------------------------------
+
+    def _auth(self) -> tuple[str, str]:
+        return self.username, self.password
+
+    async def _request(
+        self,
+        client: httpx.AsyncClient,
+        url: str,
+        params: dict[str, Any] | None = None,
+    ) -> httpx.Response:
+        """GET request with rate-limit retry."""
+        for attempt in range(_MAX_RETRIES):
+            resp = await client.get(url, params=params)
+            if resp.status_code == 429:
+                import asyncio
+
+                retry_after = int(resp.headers.get("Retry-After", "10"))
+                logger.warning(
+                    "ServiceNow rate limited, retrying in %ds (attempt %d/%d)",
+                    retry_after,
+                    attempt + 1,
+                    _MAX_RETRIES,
+                )
+                await asyncio.sleep(retry_after)
+                continue
+            resp.raise_for_status()
+            return resp
+        raise httpx.HTTPStatusError(
+            "Rate limit retries exhausted",
+            request=resp.request,  # type: ignore[possibly-undefined]
+            response=resp,  # type: ignore[possibly-undefined]
+        )
+
+    # -- CortexConnector interface -------------------------------------------
+
+    async def fetch_events(
+        self, since: datetime | None = None
+    ) -> list[RawEvent]:
+        """Fetch records from configured ServiceNow tables.
+
+        Uses the Table API with ``sysparm_query`` for incremental sync
+        and ``sysparm_offset`` / ``sysparm_limit`` for pagination.
+        """
+        cutoff = since or datetime.now(timezone.utc) - timedelta(
+            days=self.backfill_days
+        )
+        raw_events: list[RawEvent] = []
+
+        async with httpx.AsyncClient(
+            auth=self._auth(),
+            headers={"Accept": "application/json"},
+            timeout=30.0,
+        ) as client:
+            for table in self.table_filter:
+                try:
+                    raw_events.extend(
+                        await self._fetch_table(client, table, cutoff)
+                    )
+                except Exception:
+                    logger.exception(
+                        "Failed to fetch ServiceNow table %s", table
+                    )
+                    continue
+
+        return raw_events
+
+    def normalize(self, raw: RawEvent) -> Episode:
+        """Convert a ServiceNow ``RawEvent`` into a CortexDB ``Episode``."""
+        data = raw.data
+        table = data.get("table", "unknown")
+        record = data.get("record", {})
+        episode_type = _TABLE_TYPE_MAP.get(table, EpisodeType.CUSTOM)
+
+        actor = self._extract_actor(record)
+        content = self._build_content(table, record)
+        entities = self._extract_entities(table, record)
+        tags = ["servicenow", table]
+
+        state = record.get("state", "")
+        priority = record.get("priority", "")
+
+        metadata: dict[str, Any] = {
+            "servicenow_table": table,
+            "sys_id": record.get("sys_id", ""),
+            "number": record.get("number", ""),
+        }
+        if state:
+            metadata["state"] = state
+        if priority:
+            metadata["priority"] = priority
+        if record.get("category"):
+            metadata["category"] = record["category"]
+        if record.get("impact"):
+            metadata["impact"] = record["impact"]
+        if record.get("urgency"):
+            metadata["urgency"] = record["urgency"]
+
+        thread_id = self._derive_thread_id(table, record)
+
+        return Episode(
+            actor=actor,
+            source=_SOURCE,
+            episode_type=episode_type,
+            occurred_at=raw.timestamp,
+            content=content,
+            raw_payload=data,
+            tenant_id=self.tenant_id,
+            namespace=self.namespace,
+            entities=entities,
+            tags=tags,
+            metadata=metadata,
+            thread_id=thread_id,
+            idempotency_key=f"snow:{table}:{record.get('sys_id', raw.external_id)}",
+        )
+
+    def map_permissions(self, raw: RawEvent) -> Visibility:
+        """Map ServiceNow record visibility.
+
+        Knowledge articles marked as public are ``Organization``;
+        everything else is ``Restricted`` by default since ServiceNow
+        typically contains internal operational data.
+        """
+        data = raw.data
+        table = data.get("table", "")
+        record = data.get("record", {})
+
+        if table == "kb_knowledge":
+            workflow_state = record.get("workflow_state", "")
+            if workflow_state == "published":
+                return Visibility(level=VisibilityLevel.ORGANIZATION)
+            return Visibility(level=VisibilityLevel.RESTRICTED)
+
+        return Visibility(level=VisibilityLevel.ORGANIZATION)
+
+    # -- fetchers ------------------------------------------------------------
+
+    async def _fetch_table(
+        self,
+        client: httpx.AsyncClient,
+        table: str,
+        cutoff: datetime,
+    ) -> list[RawEvent]:
+        """Fetch all records from a ServiceNow table since cutoff."""
+        events: list[RawEvent] = []
+        cutoff_str = cutoff.strftime("%Y-%m-%d %H:%M:%S")
+        fields = self._fields_for(table)
+        offset = 0
+
+        while True:
+            params: dict[str, Any] = {
+                "sysparm_query": (
+                    f"sys_updated_on>{cutoff_str}^ORDERBYDESCsys_updated_on"
+                ),
+                "sysparm_fields": fields,
+                "sysparm_limit": _PAGE_SIZE,
+                "sysparm_offset": offset,
+                "sysparm_display_value": "false",
+            }
+
+            try:
+                resp = await self._request(
+                    client,
+                    f"{self._base_url}/api/now/table/{table}",
+                    params=params,
+                )
+            except Exception:
+                logger.exception(
+                    "Failed to fetch page %d for table %s", offset, table
+                )
+                break
+
+            body = resp.json()
+            records = body.get("result", [])
+            if not records:
+                break
+
+            for rec in records:
+                ts = self._parse_snow_ts(rec.get("sys_updated_on"))
+                events.append(
+                    RawEvent(
+                        source="servicenow",
+                        external_id=rec.get("sys_id", ""),
+                        timestamp=ts,
+                        data={
+                            "table": table,
+                            "record": rec,
+                        },
+                        permissions={
+                            "assigned_to": rec.get("assigned_to", ""),
+                            "assignment_group": rec.get("assignment_group", ""),
+                        },
+                    )
+                )
+
+            if len(records) < _PAGE_SIZE:
+                break
+            offset += _PAGE_SIZE
+
+        return events
+
+    # -- helpers -------------------------------------------------------------
+
+    @staticmethod
+    def _fields_for(table: str) -> str:
+        """Return the field list for a ServiceNow table query."""
+        base = "sys_id,sys_updated_on,sys_created_on,opened_by,assigned_to,assignment_group"
+        table_fields: dict[str, str] = {
+            "incident": (
+                f"{base},number,short_description,description,state,"
+                "priority,urgency,impact,category,subcategory,"
+                "cmdb_ci,caller_id,resolved_by,close_code"
+            ),
+            "change_request": (
+                f"{base},number,short_description,description,state,"
+                "priority,risk,impact,type,category,"
+                "start_date,end_date,cmdb_ci"
+            ),
+            "problem": (
+                f"{base},number,short_description,description,state,"
+                "priority,urgency,impact,category,"
+                "known_error,cause_notes,fix_notes"
+            ),
+            "kb_knowledge": (
+                "sys_id,sys_updated_on,sys_created_on,"
+                "number,short_description,text,workflow_state,"
+                "kb_category,author,published"
+            ),
+            "sc_task": (
+                f"{base},number,short_description,description,state,"
+                "priority,request_item,due_date"
+            ),
+        }
+        return table_fields.get(table, base + ",number,short_description,description,state")
+
+    @staticmethod
+    def _build_content(table: str, record: dict[str, Any]) -> str:
+        """Build human-readable content from a ServiceNow record."""
+        number = record.get("number", "")
+        short_desc = record.get("short_description", "Untitled")
+        desc = (record.get("description") or record.get("text") or "")[:1000]
+        state = record.get("state", "")
+
+        label_map: dict[str, str] = {
+            "incident": "Incident",
+            "change_request": "Change Request",
+            "problem": "Problem",
+            "kb_knowledge": "Knowledge Article",
+            "sc_task": "Task",
+        }
+        label = label_map.get(table, table.title())
+
+        if table == "kb_knowledge":
+            workflow = record.get("workflow_state", "")
+            return f"{label} [{workflow}]: {short_desc}\n{desc}".strip()
+
+        return f"{label} {number} [{state}]: {short_desc}\n{desc}".strip()
+
+    @staticmethod
+    def _extract_actor(record: dict[str, Any]) -> Actor:
+        """Extract actor from a ServiceNow record."""
+        opened_by = record.get("opened_by") or record.get("author") or ""
+        assigned_to = record.get("assigned_to") or ""
+        actor_id = opened_by or assigned_to or "system"
+        return Actor(id=str(actor_id), name=str(actor_id))
+
+    @staticmethod
+    def _extract_entities(
+        table: str, record: dict[str, Any]
+    ) -> list[EntityRef]:
+        """Extract entity references from a ServiceNow record."""
+        entities: list[EntityRef] = []
+
+        sys_id = record.get("sys_id", "")
+        number = record.get("number", "")
+        if sys_id:
+            entities.append(
+                EntityRef(
+                    entity_type=table,
+                    entity_id=sys_id,
+                    display_name=number or sys_id,
+                )
+            )
+
+        cmdb_ci = record.get("cmdb_ci")
+        if cmdb_ci:
+            entities.append(
+                EntityRef(
+                    entity_type="configuration_item",
+                    entity_id=str(cmdb_ci),
+                    display_name=str(cmdb_ci),
+                )
+            )
+
+        assigned_to = record.get("assigned_to")
+        if assigned_to:
+            entities.append(
+                EntityRef(
+                    entity_type="user",
+                    entity_id=str(assigned_to),
+                    display_name=str(assigned_to),
+                )
+            )
+
+        group = record.get("assignment_group")
+        if group:
+            entities.append(
+                EntityRef(
+                    entity_type="group",
+                    entity_id=str(group),
+                    display_name=str(group),
+                )
+            )
+
+        category = record.get("category") or record.get("kb_category")
+        if category:
+            entities.append(
+                EntityRef(
+                    entity_type="category",
+                    entity_id=str(category),
+                    display_name=str(category),
+                )
+            )
+
+        return entities
+
+    @staticmethod
+    def _derive_thread_id(
+        table: str, record: dict[str, Any]
+    ) -> str | None:
+        """Derive a thread_id for grouping related records."""
+        sys_id = record.get("sys_id", "")
+        if not sys_id:
+            return None
+
+        # Tasks thread to their parent request item.
+        if table == "sc_task":
+            request_item = record.get("request_item")
+            if request_item:
+                return f"snow:request:{request_item}"
+
+        return f"snow:{table}:{sys_id}"
+
+    @staticmethod
+    def _parse_snow_ts(value: str | None) -> datetime:
+        """Parse a ServiceNow datetime string (``YYYY-MM-DD HH:MM:SS``)."""
+        if not value:
+            return datetime.now(timezone.utc)
+        try:
+            return datetime.strptime(value, "%Y-%m-%d %H:%M:%S").replace(
+                tzinfo=timezone.utc
+            )
+        except (ValueError, TypeError):
+            pass
+        try:
+            return datetime.fromisoformat(value.replace("Z", "+00:00"))
+        except (ValueError, TypeError):
+            return datetime.now(timezone.utc)