icdev 0.0.3__py3-none-any.whl

tools/databridge/connectors/excel_connector.py

@@ -0,0 +1,175 @@
+# CUI // SP-CTI
+"""Excel File Connector — paid tier file connector (D-DB-11).
+
+Requires: openpyxl
+Supports: read, write, schema inference
+"""
+
+import logging
+import os
+import time
+from typing import Any, Dict, List
+
+from tools.databridge.connector import (
+    ConnectorCapabilities,
+    ConnectorRequest,
+    ConnectorResponse,
+    ConnectorType,
+    DataConnector,
+    SchemaDefinition,
+    SchemaField,
+)
+from tools.databridge.registry import register_connector
+from tools.databridge.schema_engine import infer_schema_from_dicts
+
+logger = logging.getLogger("databridge.connectors.excel")
+
+try:
+    import openpyxl
+
+    HAS_OPENPYXL = True
+except ImportError:
+    openpyxl = None  # type: ignore[assignment]
+    HAS_OPENPYXL = False
+
+try:
+    import pyarrow as pa
+
+    HAS_ARROW_LOCAL = True
+except ImportError:
+    pa = None  # type: ignore[assignment]
+    HAS_ARROW_LOCAL = False
+
+
+@register_connector
+class ExcelConnector(DataConnector):
+    """Excel (XLSX) file connector (paid tier)."""
+
+    _connector_name = "excel"
+    _connector_type = ConnectorType.FILE
+    _capabilities = ConnectorCapabilities(
+        supports_read=True,
+        supports_write=True,
+        supports_schema_inference=True,
+        max_batch_size=50_000,
+        supported_formats=["arrow", "xlsx"],
+    )
+
+    def __init__(self) -> None:
+        self._config: Dict[str, Any] = {}
+        self._connected = False
+
+    @property
+    def connector_type(self) -> ConnectorType:
+        return self._connector_type
+
+    @property
+    def connector_name(self) -> str:
+        return "excel"
+
+    @property
+    def capabilities(self) -> ConnectorCapabilities:
+        return self._capabilities
+
+    def connect(self, config: Dict[str, Any]) -> bool:
+        if not HAS_OPENPYXL:
+            raise ImportError("openpyxl required for Excel connector")
+        self._config = config
+        self._connected = True
+        return True
+
+    def disconnect(self) -> None:
+        self._connected = False
+
+    def health_check(self) -> Dict[str, Any]:
+        return {"status": "healthy" if self._connected and HAS_OPENPYXL else "unhealthy"}
+
+    def read(self, request: ConnectorRequest) -> ConnectorResponse:
+        start = time.time()
+        try:
+            path = request.table_name or self._config.get("file_path", "")
+            if not path or not os.path.exists(path):
+                return ConnectorResponse(status="error", errors=[f"File not found: {path}"])
+
+            sheet_name = self._config.get("sheet_name")
+            wb = openpyxl.load_workbook(path, read_only=True, data_only=True)
+            ws = wb[sheet_name] if sheet_name else wb.active
+
+            rows_iter = ws.iter_rows(values_only=True)
+            headers = [str(h) if h else f"col_{i}" for i, h in enumerate(next(rows_iter))]
+            rows = []
+            for row_vals in rows_iter:
+                rows.append(dict(zip(headers, row_vals)))
+                if request.limit and len(rows) >= request.limit:
+                    break
+            wb.close()
+
+            if HAS_ARROW_LOCAL and rows:
+                table = pa.Table.from_pylist(rows)
+                data = table
+                schema = table.schema
+            else:
+                data = rows
+                schema = None
+
+            duration_ms = int((time.time() - start) * 1000)
+            return ConnectorResponse(
+                status="ok", data=data, schema=schema,
+                row_count=len(rows), duration_ms=duration_ms,
+                bytes_transferred=os.path.getsize(path),
+            )
+        except Exception as exc:
+            return ConnectorResponse(
+                status="error", errors=[str(exc)],
+                duration_ms=int((time.time() - start) * 1000),
+            )
+
+    def write(self, request: ConnectorRequest, data: Any) -> ConnectorResponse:
+        start = time.time()
+        try:
+            path = request.table_name or self._config.get("file_path", "")
+            if HAS_ARROW_LOCAL and isinstance(data, (pa.RecordBatch, pa.Table)):
+                if isinstance(data, pa.RecordBatch):
+                    data = pa.Table.from_batches([data])
+                rows = data.to_pylist()
+                headers = data.column_names
+                row_count = data.num_rows
+            elif isinstance(data, list) and data:
+                headers = list(data[0].keys())
+                rows = data
+                row_count = len(data)
+            else:
+                return ConnectorResponse(status="error", errors=["Unsupported data type"])
+
+            wb = openpyxl.Workbook()
+            ws = wb.active
+            ws.append(headers)
+            for row in rows:
+                ws.append([row.get(h) for h in headers])
+            wb.save(path)
+
+            duration_ms = int((time.time() - start) * 1000)
+            return ConnectorResponse(status="ok", row_count=row_count, duration_ms=duration_ms)
+        except Exception as exc:
+            return ConnectorResponse(
+                status="error", errors=[str(exc)],
+                duration_ms=int((time.time() - start) * 1000),
+            )
+
+    def infer_schema(self, table_name: str) -> SchemaDefinition:
+        path = table_name or self._config.get("file_path", "")
+        wb = openpyxl.load_workbook(path, read_only=True, data_only=True)
+        ws = wb.active
+        rows_iter = ws.iter_rows(values_only=True)
+        headers = [str(h) if h else f"col_{i}" for i, h in enumerate(next(rows_iter))]
+        sample = []
+        for i, row_vals in enumerate(rows_iter):
+            if i >= 1000:
+                break
+            sample.append(dict(zip(headers, row_vals)))
+        wb.close()
+        return infer_schema_from_dicts(sample)
+
+    def list_tables(self) -> List[str]:
+        root = self._config.get("root_path", ".")
+        return [f for f in os.listdir(root) if f.endswith((".xlsx", ".xls"))]

tools/databridge/connectors/fsspec_base.py

@@ -0,0 +1,300 @@
+# CUI // SP-CTI
+"""FsspecBaseConnector — shared filesystem abstraction for cloud storage (D-DB-4).
+
+Uses fsspec as a universal filesystem interface (S3, Azure, GCS, HDFS, local).
+Supports glob patterns, auto-format detection, and Arrow conversion.
+"""
+
+import io
+import logging
+import os
+import time
+from typing import Any, Dict, List, Optional
+
+from tools.databridge.connector import (
+    ConnectorCapabilities,
+    ConnectorRequest,
+    ConnectorResponse,
+    ConnectorType,
+    DataConnector,
+    HAS_ARROW,
+    SchemaDefinition,
+    SchemaField,
+)
+
+logger = logging.getLogger("databridge.connectors.fsspec_base")
+
+# Graceful imports (D-DB-18)
+try:
+    import fsspec
+
+    HAS_FSSPEC = True
+except ImportError:
+    fsspec = None  # type: ignore[assignment]
+    HAS_FSSPEC = False
+
+try:
+    import pyarrow as pa
+    import pyarrow.parquet as pq
+    import pyarrow.csv as pac
+    import pyarrow.json as paj
+
+    HAS_ARROW_LOCAL = True
+except ImportError:
+    pa = None  # type: ignore[assignment]
+    HAS_ARROW_LOCAL = False
+
+
+# Extension → format mapping
+_FORMAT_MAP = {
+    ".parquet": "parquet",
+    ".pq": "parquet",
+    ".csv": "csv",
+    ".tsv": "csv",
+    ".json": "json",
+    ".jsonl": "jsonl",
+    ".ndjson": "jsonl",
+    ".arrow": "ipc",
+    ".ipc": "ipc",
+    ".feather": "ipc",
+    ".avro": "avro",
+    ".xlsx": "excel",
+    ".xls": "excel",
+    ".orc": "orc",
+}
+
+
+def _detect_format(path: str) -> str:
+    """Detect file format from extension."""
+    ext = os.path.splitext(path)[1].lower()
+    return _FORMAT_MAP.get(ext, "binary")
+
+
+def _read_file_to_arrow(fs: Any, path: str, fmt: str, limit: Optional[int] = None) -> Any:
+    """Read a file into an Arrow Table using the appropriate reader."""
+    if not HAS_ARROW_LOCAL:
+        raise ImportError("pyarrow required for Arrow conversion")
+
+    if fmt == "parquet":
+        with fs.open(path, "rb") as f:
+            table = pq.read_table(f)
+    elif fmt == "csv":
+        with fs.open(path, "rb") as f:
+            table = pac.read_csv(f)
+    elif fmt in ("json", "jsonl"):
+        with fs.open(path, "rb") as f:
+            table = paj.read_json(f)
+    elif fmt == "ipc":
+        with fs.open(path, "rb") as f:
+            reader = pa.ipc.open_file(f)
+            table = reader.read_all()
+    else:
+        # Binary: read raw bytes
+        with fs.open(path, "rb") as f:
+            data = f.read()
+        table = pa.table({"path": [path], "data": [data], "size": [len(data)]})
+
+    if limit and table.num_rows > limit:
+        table = table.slice(0, limit)
+    return table
+
+
+class FsspecBaseConnector(DataConnector):
+    """Base class for filesystem-based storage connectors.
+
+    Subclasses override:
+      - connector_name property
+      - _get_fs_protocol() -> str  (e.g. "s3", "az", "gcs", "file")
+      - _build_storage_options(config) -> dict
+    """
+
+    _connector_type = ConnectorType.CLOUD_STORAGE
+    _capabilities = ConnectorCapabilities(
+        supports_read=True,
+        supports_write=True,
+        supports_schema_inference=True,
+        supports_incremental=False,
+        max_batch_size=100_000,
+        supported_formats=["parquet", "csv", "json", "ipc", "arrow"],
+    )
+
+    def __init__(self) -> None:
+        self._fs: Any = None
+        self._config: Dict[str, Any] = {}
+        self._protocol: str = "file"
+
+    @property
+    def connector_type(self) -> ConnectorType:
+        return self._connector_type
+
+    @property
+    def capabilities(self) -> ConnectorCapabilities:
+        return self._capabilities
+
+    def _get_fs_protocol(self) -> str:
+        """Return fsspec protocol. Override in subclasses."""
+        return "file"
+
+    def _build_storage_options(self, config: Dict[str, Any]) -> Dict[str, Any]:
+        """Build storage_options for fsspec. Override in subclasses."""
+        return {}
+
+    def connect(self, config: Dict[str, Any]) -> bool:
+        if not HAS_FSSPEC:
+            raise ImportError("fsspec required for storage connectors")
+        self._config = config
+        self._protocol = self._get_fs_protocol()
+        try:
+            storage_options = self._build_storage_options(config)
+            self._fs = fsspec.filesystem(self._protocol, **storage_options)
+            return True
+        except Exception as exc:
+            logger.error("Storage connect failed: %s", exc)
+            self._fs = None
+            return False
+
+    def disconnect(self) -> None:
+        self._fs = None
+
+    def health_check(self) -> Dict[str, Any]:
+        if not self._fs:
+            return {"status": "unhealthy", "message": "Not connected"}
+        start = time.time()
+        try:
+            # Try listing root or configured path
+            root = self._config.get("root_path", self._config.get("bucket", "/"))
+            self._fs.ls(root, detail=False)
+            latency_ms = int((time.time() - start) * 1000)
+            return {"status": "healthy", "latency_ms": latency_ms, "protocol": self._protocol}
+        except Exception as exc:
+            return {"status": "unhealthy", "message": str(exc)}
+
+    def read(self, request: ConnectorRequest) -> ConnectorResponse:
+        if not self._fs:
+            return ConnectorResponse(status="error", errors=["Not connected"])
+
+        start = time.time()
+        try:
+            path = request.table_name  # For storage, table_name = path/glob
+            fmt = request.output_format if request.output_format != "arrow" else None
+
+            # Support glob patterns
+            if "*" in path or "?" in path:
+                paths = self._fs.glob(path)
+            else:
+                paths = [path]
+
+            if not paths:
+                return ConnectorResponse(status="ok", row_count=0, data=None)
+
+            tables = []
+            for p in paths:
+                file_fmt = fmt or _detect_format(p)
+                table = _read_file_to_arrow(self._fs, p, file_fmt, request.limit)
+                tables.append(table)
+
+            if HAS_ARROW_LOCAL and tables:
+                combined = pa.concat_tables(tables)
+                if request.limit and combined.num_rows > request.limit:
+                    combined = combined.slice(0, request.limit)
+                data = combined
+                row_count = combined.num_rows
+                schema = combined.schema
+            else:
+                data = tables
+                row_count = sum(t.num_rows for t in tables) if tables else 0
+                schema = None
+
+            duration_ms = int((time.time() - start) * 1000)
+            return ConnectorResponse(
+                status="ok",
+                data=data,
+                schema=schema,
+                row_count=row_count,
+                duration_ms=duration_ms,
+            )
+        except Exception as exc:
+            duration_ms = int((time.time() - start) * 1000)
+            return ConnectorResponse(
+                status="error", errors=[str(exc)], duration_ms=duration_ms,
+            )
+
+    def write(self, request: ConnectorRequest, data: Any) -> ConnectorResponse:
+        if not self._fs:
+            return ConnectorResponse(status="error", errors=["Not connected"])
+
+        start = time.time()
+        try:
+            path = request.table_name
+            fmt = _detect_format(path)
+
+            if HAS_ARROW_LOCAL:
+                if isinstance(data, pa.RecordBatch):
+                    data = pa.Table.from_batches([data])
+
+                if fmt == "parquet":
+                    with self._fs.open(path, "wb") as f:
+                        pq.write_table(data, f)
+                elif fmt == "csv":
+                    with self._fs.open(path, "wb") as f:
+                        pac.write_csv(data, f)
+                elif fmt in ("json", "jsonl"):
+                    rows = data.to_pydict()
+                    import json
+                    content = json.dumps(rows).encode()
+                    with self._fs.open(path, "wb") as f:
+                        f.write(content)
+                elif fmt == "ipc":
+                    with self._fs.open(path, "wb") as f:
+                        writer = pa.ipc.new_file(f, data.schema)
+                        writer.write_table(data)
+                        writer.close()
+                else:
+                    return ConnectorResponse(status="error", errors=[f"Unsupported format: {fmt}"])
+
+                row_count = data.num_rows
+            else:
+                return ConnectorResponse(status="error", errors=["pyarrow required for write"])
+
+            duration_ms = int((time.time() - start) * 1000)
+            return ConnectorResponse(status="ok", row_count=row_count, duration_ms=duration_ms)
+        except Exception as exc:
+            duration_ms = int((time.time() - start) * 1000)
+            return ConnectorResponse(status="error", errors=[str(exc)], duration_ms=duration_ms)
+
+    def infer_schema(self, table_name: str) -> SchemaDefinition:
+        if not self._fs:
+            raise RuntimeError("Not connected")
+
+        fmt = _detect_format(table_name)
+        if not HAS_ARROW_LOCAL:
+            raise ImportError("pyarrow required for schema inference")
+
+        if fmt == "parquet":
+            with self._fs.open(table_name, "rb") as f:
+                schema = pq.read_schema(f)
+        else:
+            # Read small sample for other formats
+            table = _read_file_to_arrow(self._fs, table_name, fmt, limit=100)
+            schema = table.schema
+
+        fields = []
+        for i in range(len(schema)):
+            af = schema.field(i)
+            fields.append(SchemaField(
+                name=af.name,
+                data_type=str(af.type),
+                nullable=af.nullable,
+            ))
+        return SchemaDefinition(fields=fields, metadata={"source": self.connector_name, "path": table_name})
+
+    def list_tables(self) -> List[str]:
+        """List files in configured root path."""
+        if not self._fs:
+            return []
+        try:
+            root = self._config.get("root_path", self._config.get("bucket", "/"))
+            return self._fs.ls(root, detail=False)
+        except Exception as exc:
+            logger.error("list_tables failed: %s", exc)
+            return []

tools/databridge/connectors/gcs.py

@@ -0,0 +1,53 @@
+# CUI // SP-CTI
+"""Google Cloud Storage Connector — paid tier cloud storage connector (D-DB-11).
+
+Requires: gcsfs + fsspec
+Supports: read, write, schema inference, glob patterns
+"""
+
+import logging
+from typing import Any, Dict
+
+from tools.databridge.connectors.fsspec_base import FsspecBaseConnector
+from tools.databridge.connector import ConnectorCapabilities
+from tools.databridge.registry import register_connector
+
+logger = logging.getLogger("databridge.connectors.gcs")
+
+
+@register_connector
+class GCSConnector(FsspecBaseConnector):
+    """Google Cloud Storage connector (paid tier)."""
+
+    _connector_name = "gcs"
+
+    _capabilities = ConnectorCapabilities(
+        supports_read=True,
+        supports_write=True,
+        supports_schema_inference=True,
+        max_batch_size=100_000,
+        supported_formats=["parquet", "csv", "json", "ipc"],
+    )
+
+    @property
+    def connector_name(self) -> str:
+        return "gcs"
+
+    def _get_fs_protocol(self) -> str:
+        return "gcs"
+
+    def _build_storage_options(self, config: Dict[str, Any]) -> Dict[str, Any]:
+        """Build GCS storage options.
+
+        Config keys:
+          - token: path to service account JSON, or 'anon', or 'google_default'
+          - project: GCP project ID
+          - _resolved_secret: path to service account JSON
+        """
+        opts: Dict[str, Any] = {}
+        token = config.get("token") or config.get("_resolved_secret")
+        if token:
+            opts["token"] = token
+        if config.get("project"):
+            opts["project"] = config["project"]
+        return opts

tools/databridge/connectors/github_connector.py

@@ -0,0 +1,138 @@
+# CUI // SP-CTI
+"""GitHub connector — paid tier DevOps (D-DB-21).
+
+Auth: PAT or OAUTH2.
+API: REST v3 + GraphQL, Link header pagination.
+"""
+
+import logging
+from typing import Any, Dict, List, Optional
+
+from tools.databridge.connector import ConnectorCapabilities, ConnectorType
+from tools.databridge.connectors.saas_base import SaaSBaseConnector
+from tools.databridge.registry import register_connector
+
+logger = logging.getLogger("databridge.connectors.github")
+
+try:
+    import requests as _requests
+    HAS_REQUESTS = True
+except ImportError:
+    _requests = None  # type: ignore[assignment]
+    HAS_REQUESTS = False
+
+GITHUB_API = "https://api.github.com"
+
+
+@register_connector
+class GitHubConnector(SaaSBaseConnector):
+    """GitHub REST + GraphQL connector (paid tier)."""
+
+    _connector_name = "github"
+    _connector_type = ConnectorType.DEVOPS
+    _capabilities = ConnectorCapabilities(
+        supports_read=True,
+        supports_write=True,
+        supports_streaming=False,
+        supports_schema_inference=True,
+        max_batch_size=100,
+        supported_formats=["json"],
+    )
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._owner: str = ""
+        self._repo: str = ""
+
+    @property
+    def connector_name(self) -> str:
+        return "github"
+
+    def _base_url(self, config: Dict[str, Any]) -> str:
+        return config.get("api_url", GITHUB_API)
+
+    def _auth_headers(self, config: Dict[str, Any]) -> Dict[str, str]:
+        token = config.get("_resolved_secret") or config.get("token", "")
+        return {
+            "Authorization": f"Bearer {token}",
+            "X-GitHub-Api-Version": "2022-11-28",
+        }
+
+    def _read_endpoint(self, table_name: str) -> str:
+        # table_name: issues, pulls, commits, releases, actions/runs
+        resource = table_name or "repos"
+        if self._owner and self._repo:
+            return f"{self._base}/repos/{self._owner}/{self._repo}/{resource}"
+        return f"{self._base}/user/{resource}"
+
+    def _parse_response(self, resp_json: Any) -> tuple:
+        if isinstance(resp_json, list):
+            return resp_json, None
+        items = resp_json.get("items", resp_json.get("workflow_runs", []))
+        return items, None
+
+    def connect(self, config: Dict[str, Any]) -> bool:
+        if not HAS_REQUESTS:
+            raise ImportError("requests required")
+        self._config = config
+        self._base = self._base_url(config)
+        self._owner = config.get("owner", "")
+        self._repo = config.get("repo", "")
+        self._session = _requests.Session()
+        self._session.headers.update(self._auth_headers(config))
+        self._session.headers["Accept"] = "application/vnd.github+json"
+        resp = self._session.get(f"{self._base}/user", timeout=10)
+        return resp.status_code == 200
+
+    def list_repos(self, org: str = "") -> List[Dict[str, Any]]:
+        """List repositories for user or org."""
+        if org:
+            url = f"{self._base}/orgs/{org}/repos"
+        else:
+            url = f"{self._base}/user/repos"
+        resp = self._session.get(url, params={"per_page": 100}, timeout=15)
+        return [
+            {"id": r["id"], "name": r["full_name"], "private": r["private"],
+             "language": r.get("language", ""), "default_branch": r.get("default_branch", "")}
+            for r in resp.json() if isinstance(resp.json(), list)
+        ] if isinstance(resp.json(), list) else []
+
+    def get_issues(self, state: str = "open", limit: int = 30) -> List[Dict[str, Any]]:
+        """Get issues for the configured repo."""
+        url = f"{self._base}/repos/{self._owner}/{self._repo}/issues"
+        resp = self._session.get(url, params={"state": state, "per_page": limit}, timeout=15)
+        return resp.json() if isinstance(resp.json(), list) else []
+
+    def get_pull_requests(self, state: str = "open", limit: int = 30) -> List[Dict[str, Any]]:
+        """Get pull requests for the configured repo."""
+        url = f"{self._base}/repos/{self._owner}/{self._repo}/pulls"
+        resp = self._session.get(url, params={"state": state, "per_page": limit}, timeout=15)
+        return resp.json() if isinstance(resp.json(), list) else []
+
+    def create_issue(self, title: str, body: str = "",
+                     labels: Optional[List[str]] = None) -> Dict[str, Any]:
+        """Create an issue."""
+        url = f"{self._base}/repos/{self._owner}/{self._repo}/issues"
+        payload: Dict[str, Any] = {"title": title, "body": body}
+        if labels:
+            payload["labels"] = labels
+        resp = self._session.post(url, json=payload, timeout=15)
+        return resp.json()
+
+    def get_workflow_runs(self, limit: int = 10) -> List[Dict[str, Any]]:
+        """Get recent workflow runs."""
+        url = f"{self._base}/repos/{self._owner}/{self._repo}/actions/runs"
+        resp = self._session.get(url, params={"per_page": limit}, timeout=15)
+        return resp.json().get("workflow_runs", [])
+
+    def graphql_query(self, query: str,
+                      variables: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+        """Execute a GitHub GraphQL query."""
+        url = f"{self._base.replace('/api.github.com', '')}/graphql"
+        if "api.github.com" in self._base:
+            url = "https://api.github.com/graphql"
+        payload: Dict[str, Any] = {"query": query}
+        if variables:
+            payload["variables"] = variables
+        resp = self._session.post(url, json=payload, timeout=30)
+        return resp.json()