icdev 0.0.3__py3-none-any.whl

tools/databridge/connectors/telegram.py

@@ -0,0 +1,192 @@
+# CUI // SP-CTI
+"""Telegram Bot API connector — paid tier messaging (D-DB-29).
+
+Uses Telegram Bot API (https://core.telegram.org/bots/api).
+Auth: BOT_TOKEN (from BotFather). Connection mode: long_poll (getUpdates).
+"""
+
+import logging
+import time
+from typing import Any, Callable, Dict, List, Optional
+
+from tools.databridge.connector import ConnectorCapabilities
+from tools.databridge.connectors.messaging_base import MessagingBaseConnector
+from tools.databridge.messaging.message_envelope import MessageEnvelope
+from tools.databridge.registry import register_connector
+
+logger = logging.getLogger("databridge.connectors.telegram")
+
+try:
+    import requests as _requests
+    HAS_REQUESTS = True
+except ImportError:
+    _requests = None  # type: ignore[assignment]
+    HAS_REQUESTS = False
+
+
+@register_connector
+class TelegramConnector(MessagingBaseConnector):
+    """Telegram Bot API connector (paid tier)."""
+
+    _connector_name = "telegram"
+    _capabilities = ConnectorCapabilities(
+        supports_read=True,
+        supports_write=True,
+        supports_streaming=True,
+        supports_schema_inference=False,
+        max_batch_size=100,
+        supported_formats=["json"],
+    )
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._bot_token: str = ""
+        self._api_base: str = ""
+        self._update_offset: int = 0
+        self._stop: bool = False
+
+    @property
+    def connector_name(self) -> str:
+        return "telegram"
+
+    def _base_url(self, config: Dict[str, Any]) -> str:
+        token = config.get("_resolved_secret") or config.get("bot_token", "")
+        return f"https://api.telegram.org/bot{token}"
+
+    def _auth_headers(self, config: Dict[str, Any]) -> Dict[str, str]:
+        return {}  # Telegram uses token in URL, not headers
+
+    def _read_endpoint(self, table_name: str) -> str:
+        return f"{self._base}/getUpdates"
+
+    def _parse_response(self, resp_json: Any) -> tuple:
+        results = resp_json.get("result", [])
+        return results, None
+
+    def connect(self, config: Dict[str, Any]) -> bool:
+        self._config = config
+        self._bot_token = config.get("_resolved_secret") or config.get("bot_token", "")
+        self._api_base = f"https://api.telegram.org/bot{self._bot_token}"
+        if HAS_REQUESTS:
+            resp = _requests.get(f"{self._api_base}/getMe", timeout=10)
+            return resp.status_code == 200
+        return False
+
+    def disconnect(self) -> None:
+        self._stop = True
+
+    def health_check(self) -> Dict[str, Any]:
+        if not HAS_REQUESTS:
+            return {"status": "unhealthy", "message": "requests not installed"}
+        start = time.time()
+        try:
+            resp = _requests.get(f"{self._api_base}/getMe", timeout=10)
+            latency = int((time.time() - start) * 1000)
+            if resp.status_code == 200 and resp.json().get("ok"):
+                bot_info = resp.json().get("result", {})
+                return {"status": "healthy", "latency_ms": latency,
+                        "bot_username": bot_info.get("username", "")}
+            return {"status": "unhealthy", "latency_ms": latency}
+        except Exception as exc:
+            return {"status": "unhealthy", "message": str(exc)}
+
+    def send_message(self, channel_id: str, text: str,
+                     thread_id: str = "",
+                     attachments: Optional[List[Dict[str, Any]]] = None) -> MessageEnvelope:
+        payload: Dict[str, Any] = {"chat_id": channel_id, "text": text}
+        if thread_id:
+            payload["reply_to_message_id"] = thread_id
+        resp = _requests.post(f"{self._api_base}/sendMessage", json=payload, timeout=15)
+        data = resp.json()
+        msg = data.get("result", {})
+        return MessageEnvelope(
+            platform="telegram",
+            direction="outbound",
+            channel_id=str(channel_id),
+            author_id=str(msg.get("from", {}).get("id", "")),
+            author_name=msg.get("from", {}).get("username", "bot"),
+            content=text,
+            platform_message_id=str(msg.get("message_id", "")),
+        )
+
+    def receive_messages(self, channel_id: str, limit: int = 50,
+                         since: Optional[str] = None) -> List[MessageEnvelope]:
+        params: Dict[str, Any] = {"limit": min(limit, 100), "offset": self._update_offset}
+        resp = _requests.get(f"{self._api_base}/getUpdates", params=params, timeout=15)
+        data = resp.json()
+        updates = data.get("result", [])
+        envelopes = []
+        for update in updates:
+            msg = update.get("message") or update.get("channel_post", {})
+            if not msg:
+                continue
+            self._update_offset = update["update_id"] + 1
+            chat = msg.get("chat", {})
+            if channel_id and str(chat.get("id")) != str(channel_id):
+                continue
+            sender = msg.get("from", {})
+            envelopes.append(MessageEnvelope(
+                platform="telegram",
+                direction="inbound",
+                channel_id=str(chat.get("id", "")),
+                channel_name=chat.get("title") or chat.get("username", ""),
+                author_id=str(sender.get("id", "")),
+                author_name=sender.get("username") or sender.get("first_name", ""),
+                content=msg.get("text", ""),
+                platform_message_id=str(msg.get("message_id", "")),
+                thread_id=str(msg.get("reply_to_message", {}).get("message_id", ""))
+                if msg.get("reply_to_message") else "",
+            ))
+        return envelopes
+
+    def list_channels(self) -> List[Dict[str, Any]]:
+        # Telegram bots discover chats via getUpdates
+        envelopes = self.receive_messages("", limit=100)
+        seen: Dict[str, Dict[str, Any]] = {}
+        for e in envelopes:
+            if e.channel_id not in seen:
+                seen[e.channel_id] = {
+                    "id": e.channel_id, "name": e.channel_name, "type": "chat",
+                }
+        return list(seen.values())
+
+    def get_connection_mode(self) -> str:
+        return self._config.get("connection_mode", "long_poll")
+
+    def start_persistent_listener(self, callback: Callable[[MessageEnvelope], None]) -> None:
+        self._stop = False
+        poll_timeout = self._config.get("poll_timeout", 30)
+        while not self._stop:
+            try:
+                params = {"limit": 100, "offset": self._update_offset, "timeout": poll_timeout}
+                request_timeout = poll_timeout + 10
+                resp = _requests.get(  # nosec B113
+                    f"{self._api_base}/getUpdates", params=params,
+                    timeout=request_timeout,
+                )
+                data = resp.json()
+                for update in data.get("result", []):
+                    self._update_offset = update["update_id"] + 1
+                    msg = update.get("message") or update.get("channel_post", {})
+                    if not msg:
+                        continue
+                    chat = msg.get("chat", {})
+                    sender = msg.get("from", {})
+                    envelope = MessageEnvelope(
+                        platform="telegram",
+                        direction="inbound",
+                        channel_id=str(chat.get("id", "")),
+                        channel_name=chat.get("title") or chat.get("username", ""),
+                        author_id=str(sender.get("id", "")),
+                        author_name=sender.get("username") or sender.get("first_name", ""),
+                        content=msg.get("text", ""),
+                        platform_message_id=str(msg.get("message_id", "")),
+                    )
+                    callback(envelope)
+            except Exception as exc:
+                if not self._stop:
+                    logger.warning("Telegram poll error: %s", exc)
+                    time.sleep(5)
+
+    def stop_persistent_listener(self) -> None:
+        self._stop = True

tools/databridge/connectors/whatsapp.py

@@ -0,0 +1,137 @@
+# CUI // SP-CTI
+"""WhatsApp Business Cloud API connector — paid tier messaging (D-DB-29).
+
+Auth: API_KEY (permanent token from Meta Business).
+Connection mode: webhook_only (Meta sends to our webhook).
+"""
+
+import logging
+from typing import Any, Dict, List, Optional
+
+from tools.databridge.connector import ConnectorCapabilities
+from tools.databridge.connectors.messaging_base import MessagingBaseConnector
+from tools.databridge.messaging.message_envelope import MessageEnvelope
+from tools.databridge.registry import register_connector
+
+logger = logging.getLogger("databridge.connectors.whatsapp")
+
+try:
+    import requests as _requests
+    HAS_REQUESTS = True
+except ImportError:
+    _requests = None  # type: ignore[assignment]
+    HAS_REQUESTS = False
+
+GRAPH_API = "https://graph.facebook.com/v18.0"
+
+
+@register_connector
+class WhatsAppConnector(MessagingBaseConnector):
+    """WhatsApp Business Cloud API connector (paid tier)."""
+
+    _connector_name = "whatsapp"
+    _capabilities = ConnectorCapabilities(
+        supports_read=True,
+        supports_write=True,
+        supports_streaming=False,
+        supports_schema_inference=False,
+        max_batch_size=50,
+        supported_formats=["json"],
+    )
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._phone_number_id: str = ""
+
+    @property
+    def connector_name(self) -> str:
+        return "whatsapp"
+
+    def _base_url(self, config: Dict[str, Any]) -> str:
+        return GRAPH_API
+
+    def _auth_headers(self, config: Dict[str, Any]) -> Dict[str, str]:
+        token = config.get("_resolved_secret") or config.get("access_token", "")
+        return {"Authorization": f"Bearer {token}"}
+
+    def _read_endpoint(self, table_name: str) -> str:
+        return f"{GRAPH_API}/{self._phone_number_id}/messages"
+
+    def _parse_response(self, resp_json: Any) -> tuple:
+        data = resp_json.get("data", [])
+        return data, None
+
+    def connect(self, config: Dict[str, Any]) -> bool:
+        if not HAS_REQUESTS:
+            raise ImportError("requests required")
+        self._config = config
+        self._phone_number_id = config.get("phone_number_id", "")
+        self._base = GRAPH_API
+        self._session = _requests.Session()
+        self._session.headers.update(self._auth_headers(config))
+        return bool(self._phone_number_id)
+
+    def send_message(self, channel_id: str, text: str,
+                     thread_id: str = "",
+                     attachments: Optional[List[Dict[str, Any]]] = None) -> MessageEnvelope:
+        payload = {
+            "messaging_product": "whatsapp",
+            "to": channel_id,
+            "type": "text",
+            "text": {"body": text},
+        }
+        resp = self._session.post(
+            f"{GRAPH_API}/{self._phone_number_id}/messages",
+            json=payload, timeout=15,
+        )
+        data = resp.json()
+        msg_id = ""
+        if "messages" in data:
+            msg_id = data["messages"][0].get("id", "")
+        return MessageEnvelope(
+            platform="whatsapp",
+            direction="outbound",
+            channel_id=channel_id,
+            author_id=self._phone_number_id,
+            author_name="bot",
+            content=text,
+            platform_message_id=msg_id,
+        )
+
+    def receive_messages(self, channel_id: str, limit: int = 50,
+                         since: Optional[str] = None) -> List[MessageEnvelope]:
+        # WhatsApp uses webhook-only for inbound — messages come via webhook route
+        return []
+
+    def list_channels(self) -> List[Dict[str, Any]]:
+        # WhatsApp doesn't have channel listing — contacts come via webhooks
+        return []
+
+    def get_connection_mode(self) -> str:
+        return "webhook_only"
+
+    @staticmethod
+    def parse_webhook_payload(payload: Dict[str, Any]) -> List[MessageEnvelope]:
+        """Parse inbound WhatsApp webhook payload into MessageEnvelopes."""
+        envelopes = []
+        for entry in payload.get("entry", []):
+            for change in entry.get("changes", []):
+                value = change.get("value", {})
+                for msg in value.get("messages", []):
+                    contact = {}
+                    for c in value.get("contacts", []):
+                        if c.get("wa_id") == msg.get("from"):
+                            contact = c
+                            break
+                    envelopes.append(MessageEnvelope(
+                        platform="whatsapp",
+                        direction="inbound",
+                        channel_id=msg.get("from", ""),
+                        author_id=msg.get("from", ""),
+                        author_name=contact.get("profile", {}).get("name", ""),
+                        content=msg.get("text", {}).get("body", "")
+                        if msg.get("type") == "text" else f"[{msg.get('type', 'unknown')}]",
+                        platform_message_id=msg.get("id", ""),
+                        timestamp=msg.get("timestamp", ""),
+                    ))
+        return envelopes

tools/databridge/data_profiler.py

@@ -0,0 +1,99 @@
+# CUI // SP-CTI
+"""DataBridge Data Profiler — data quality and statistical profiling (D-DB-5).
+
+Provides column-level statistics, data quality metrics, and anomaly detection.
+Uses DuckDB when available, falls back to Python stdlib.
+"""
+
+import logging
+import math
+from collections import Counter
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger("databridge.data_profiler")
+
+
+def profile_rows(
+    rows: List[Dict[str, Any]],
+    columns: Optional[List[str]] = None,
+) -> Dict[str, Any]:
+    """Profile data from a list of dicts.
+
+    Returns per-column statistics: type, nulls, distinct, min, max, avg, etc.
+    """
+    if not rows:
+        return {"status": "ok", "columns": [], "row_count": 0}
+
+    if not columns:
+        columns = list(rows[0].keys())
+
+    profiles = []
+    for col in columns:
+        values = [row.get(col) for row in rows]
+        non_null = [v for v in values if v is not None]
+        null_count = len(values) - len(non_null)
+        distinct = len(set(str(v) for v in non_null))
+
+        profile: Dict[str, Any] = {
+            "column": col,
+            "total": len(values),
+            "non_null": len(non_null),
+            "null_count": null_count,
+            "null_pct": round(null_count / len(values) * 100, 1) if values else 0,
+            "distinct_count": distinct,
+            "uniqueness_pct": round(distinct / len(values) * 100, 1) if values else 0,
+        }
+
+        # Detect type
+        numeric = [v for v in non_null if isinstance(v, (int, float))]
+        strings = [v for v in non_null if isinstance(v, str)]
+
+        if len(numeric) > len(non_null) * 0.8:
+            profile["inferred_type"] = "numeric"
+            profile["min"] = min(numeric)
+            profile["max"] = max(numeric)
+            profile["avg"] = round(sum(numeric) / len(numeric), 4) if numeric else None
+            profile["sum"] = sum(numeric)
+            if len(numeric) > 1:
+                mean = sum(numeric) / len(numeric)
+                variance = sum((x - mean) ** 2 for x in numeric) / (len(numeric) - 1)
+                profile["stddev"] = round(math.sqrt(variance), 4)
+        elif strings:
+            profile["inferred_type"] = "string"
+            lengths = [len(s) for s in strings]
+            profile["min_length"] = min(lengths)
+            profile["max_length"] = max(lengths)
+            profile["avg_length"] = round(sum(lengths) / len(lengths), 1)
+            # Top values
+            counter = Counter(strings)
+            profile["top_values"] = [
+                {"value": v, "count": c}
+                for v, c in counter.most_common(5)
+            ]
+        else:
+            profile["inferred_type"] = "mixed"
+
+        # Empty string check
+        empty_strings = sum(1 for v in non_null if isinstance(v, str) and v.strip() == "")
+        if empty_strings:
+            profile["empty_string_count"] = empty_strings
+
+        profiles.append(profile)
+
+    # Data quality score
+    quality_issues = 0
+    for p in profiles:
+        if p["null_pct"] > 50:
+            quality_issues += 1
+        if p.get("uniqueness_pct", 100) < 1 and p["distinct_count"] > 1:
+            quality_issues += 1
+
+    quality_score = max(0, 100 - quality_issues * 10)
+
+    return {
+        "status": "ok",
+        "row_count": len(rows),
+        "column_count": len(profiles),
+        "columns": profiles,
+        "quality_score": quality_score,
+    }

tools/databridge/forge/__init__.py

@@ -0,0 +1,6 @@
+# CUI // SP-CTI
+"""Connector Forge — dynamic DataBridge connector generation (D-CF-1).
+
+6-stage pipeline: spec parse → base select → code gen → validate → sandbox → integrate.
+Part of the sparkpilot multi-agent architecture (Agent 12, port 9458).
+"""

tools/databridge/forge/base_selector.py

@@ -0,0 +1,150 @@
+# CUI // SP-CTI
+"""Base Selector — Stage 2 of Connector Forge pipeline (D-CF-1).
+
+Deterministic protocol → base class mapping.
+No LLM, no I/O. Input: ForgeApiManifest. Output: BaseClassSelection.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Tuple
+
+from tools.databridge.forge.spec_parser import ForgeApiManifest
+
+
+@dataclass
+class BaseClassSelection:
+    """Result of base class selection for a connector."""
+
+    base_class: str  # e.g. 'SaaSBaseConnector'
+    base_module: str  # e.g. 'tools.databridge.connectors.saas_base'
+    template_name: str  # e.g. 'rest_connector'
+    connector_type: str  # e.g. 'saas_api'
+    rationale: str = ""
+    confidence: float = 1.0  # 1.0 = deterministic, <1.0 = heuristic
+
+
+# Protocol → (base_class, base_module, template_name, connector_type)
+_PROTOCOL_MAP: dict[str, Tuple[str, str, str, str]] = {
+    "rest": (
+        "SaaSBaseConnector",
+        "tools.databridge.connectors.saas_base",
+        "rest_connector",
+        "saas_api",
+    ),
+    "graphql": (
+        "SaaSBaseConnector",
+        "tools.databridge.connectors.saas_base",
+        "graphql_connector",
+        "saas_api",
+    ),
+    "soap": (
+        "SoapBaseConnector",
+        "tools.databridge.connectors.soap_base",
+        "soap_connector",
+        "soap",
+    ),
+    "sql": (
+        "SQLBaseConnector",
+        "tools.databridge.connectors.sql_base",
+        "sql_connector",
+        "database",
+    ),
+    "odbc": (
+        "SQLBaseConnector",
+        "tools.databridge.connectors.sql_base",
+        "sql_connector",
+        "database",
+    ),
+    "ftp": (
+        "FsspecBaseConnector",
+        "tools.databridge.connectors.fsspec_base",
+        "ftp_connector",
+        "cloud_storage",
+    ),
+    "sftp": (
+        "FsspecBaseConnector",
+        "tools.databridge.connectors.fsspec_base",
+        "ftp_connector",
+        "cloud_storage",
+    ),
+    "messaging": (
+        "MessagingBaseConnector",
+        "tools.databridge.connectors.messaging_base",
+        "messaging_connector",
+        "messaging",
+    ),
+    "hl7": (
+        "HealthBaseConnector",
+        "tools.databridge.connectors.health_base",
+        "health_connector",
+        "health",
+    ),
+    "fhir": (
+        "HealthBaseConnector",
+        "tools.databridge.connectors.health_base",
+        "health_connector",
+        "health",
+    ),
+    "tcp": (
+        "DataConnector",
+        "tools.databridge.connector",
+        "tcp_connector",
+        "on_prem",
+    ),
+}
+
+
+def select_base_class(manifest: ForgeApiManifest) -> BaseClassSelection:
+    """Select the appropriate base class for a connector manifest.
+
+    Uses deterministic protocol mapping with heuristic fallback.
+    """
+    proto = manifest.protocol.lower()
+
+    # Direct protocol match
+    if proto in _PROTOCOL_MAP:
+        base_class, base_module, template, conn_type = _PROTOCOL_MAP[proto]
+        return BaseClassSelection(
+            base_class=base_class,
+            base_module=base_module,
+            template_name=template,
+            connector_type=conn_type,
+            rationale=f"Protocol '{proto}' maps deterministically to {base_class}",
+            confidence=1.0,
+        )
+
+    # Heuristic: WSDL/operations hint
+    if manifest.wsdl_url or manifest.soap_operations:
+        base_class, base_module, template, conn_type = _PROTOCOL_MAP["soap"]
+        return BaseClassSelection(
+            base_class=base_class,
+            base_module=base_module,
+            template_name=template,
+            connector_type=conn_type,
+            rationale="WSDL URL or SOAP operations detected — selecting SOAP",
+            confidence=0.9,
+        )
+
+    # Heuristic: FHIR resources hint
+    if manifest.fhir_resources:
+        base_class, base_module, template, conn_type = _PROTOCOL_MAP["fhir"]
+        return BaseClassSelection(
+            base_class=base_class,
+            base_module=base_module,
+            template_name=template,
+            connector_type=conn_type,
+            rationale="FHIR resources detected — selecting Health",
+            confidence=0.9,
+        )
+
+    # Default: REST/SaaS
+    base_class, base_module, template, conn_type = _PROTOCOL_MAP["rest"]
+    return BaseClassSelection(
+        base_class=base_class,
+        base_module=base_module,
+        template_name=template,
+        connector_type=conn_type,
+        rationale=f"Unknown protocol '{proto}' — defaulting to REST/SaaS",
+        confidence=0.6,
+    )