cortexdb-connectors 0.2.0__py3-none-any.whl

cortexdb_connectors/cli.py

@@ -0,0 +1,476 @@
+"""CortexDB connectors CLI — `cortexdb-sync`.
+
+Commands:
+
+    cortexdb-sync sync slack                   one-shot sync (uses stored cursor)
+    cortexdb-sync sync slack --since 2026-01-01T00:00:00Z
+    cortexdb-sync watch slack --interval 60    poll-and-sleep loop
+    cortexdb-sync status                       cursor table for every connector
+    cortexdb-sync list                         list available connectors + env vars
+    cortexdb-sync auth                         show resolved CortexDB credentials
+
+Reads CortexDB creds in this order:
+
+    1. CLI flags (--api-url, --api-key, --actor, --scope-template)
+    2. Env vars (CORTEXDB_URL, CORTEXDB_API_KEY, CORTEXDB_ACTOR, CORTEXDB_SCOPE_TEMPLATE)
+    3. `~/.cortexdb/state.json` — written by `cortexdb init` from the cortexdb-cli
+       package; same format the cortexdb-mcp 0.3.0 server uses, so you can mint
+       a free-tier identity once and use it from everywhere.
+
+Per-connector secrets (Slack bot token, Jira API token, …) come from env
+vars by default; override with a YAML config file via --config.
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import importlib
+import json
+import logging
+import os
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Callable
+
+from cortexdb_connectors.state import SyncStateStore
+
+logger = logging.getLogger("cortexdb_connectors")
+
+
+# ---------------------------------------------------------------------------
+# Connector registry
+#
+# Each entry maps a CLI slug to:
+#   - module + class to import
+#   - secret env vars (required for the connector to function)
+#   - optional list/scalar config knobs the CLI surfaces
+# ---------------------------------------------------------------------------
+
+CONNECTORS: dict[str, dict[str, Any]] = {
+    "slack": {
+        "import": "cortexdb_connectors.slack:SlackConnector",
+        "secrets": [("slack_bot_token", "SLACK_BOT_TOKEN")],
+        "options": [("channels", "SLACK_CHANNELS", "list")],
+    },
+    "github": {
+        "import": "cortexdb_connectors.github:GitHubConnector",
+        "secrets": [("github_token", "GITHUB_TOKEN")],
+        "options": [
+            ("repos", "GITHUB_REPOS", "list"),
+            ("events", "GITHUB_EVENTS", "list"),
+        ],
+    },
+    "gitlab": {
+        "import": "cortexdb_connectors.gitlab:GitLabConnector",
+        "secrets": [("gitlab_token", "GITLAB_TOKEN")],
+        "options": [
+            ("gitlab_url", "GITLAB_URL", "str"),
+            ("project_ids", "GITLAB_PROJECT_IDS", "list-int"),
+            ("group_ids", "GITLAB_GROUP_IDS", "list-int"),
+            ("events", "GITLAB_EVENTS", "list"),
+        ],
+    },
+    "jira": {
+        "import": "cortexdb_connectors.jira:JiraConnector",
+        "secrets": [
+            ("jira_url", "JIRA_URL"),
+            ("jira_email", "JIRA_EMAIL"),
+            ("jira_api_token", "JIRA_API_TOKEN"),
+        ],
+        "options": [("project_keys", "JIRA_PROJECT_KEYS", "list")],
+    },
+    "linear": {
+        "import": "cortexdb_connectors.linear:LinearConnector",
+        "secrets": [("linear_api_key", "LINEAR_API_KEY")],
+        "options": [("team_ids", "LINEAR_TEAMS", "list")],
+    },
+    "confluence": {
+        "import": "cortexdb_connectors.confluence:ConfluenceConnector",
+        "secrets": [
+            ("confluence_url", "CONFLUENCE_URL"),
+            ("email", "CONFLUENCE_EMAIL"),
+            ("api_token", "CONFLUENCE_API_TOKEN"),
+        ],
+        "options": [("space_keys", "CONFLUENCE_SPACES", "list")],
+    },
+    "notion": {
+        "import": "cortexdb_connectors.notion:NotionConnector",
+        "secrets": [("notion_token", "NOTION_TOKEN")],
+        "options": [
+            ("database_ids", "NOTION_DATABASES", "list"),
+            ("page_ids", "NOTION_PAGES", "list"),
+        ],
+    },
+    "pagerduty": {
+        "import": "cortexdb_connectors.pagerduty:PagerDutyConnector",
+        "secrets": [("pagerduty_api_key", "PAGERDUTY_API_KEY")],
+        "options": [("service_ids", "PAGERDUTY_SERVICES", "list")],
+    },
+    "discord": {
+        "import": "cortexdb_connectors.discord:DiscordConnector",
+        "secrets": [("bot_token", "DISCORD_BOT_TOKEN")],
+        "options": [
+            ("guild_ids", "DISCORD_GUILDS", "list"),
+            ("channel_ids", "DISCORD_CHANNELS", "list"),
+        ],
+    },
+    "teams": {
+        "import": "cortexdb_connectors.teams:TeamsConnector",
+        "secrets": [
+            ("azure_tenant_id", "TEAMS_TENANT_ID"),
+            ("client_id", "TEAMS_CLIENT_ID"),
+            ("client_secret", "TEAMS_CLIENT_SECRET"),
+        ],
+        "options": [
+            ("team_ids", "TEAMS_TEAM_IDS", "list"),
+            ("include_chats", "TEAMS_INCLUDE_CHATS", "bool"),
+        ],
+    },
+    "google-workspace": {
+        "import": "cortexdb_connectors.google_workspace:GoogleWorkspaceConnector",
+        "secrets": [
+            ("service_account_key", "GW_SERVICE_ACCOUNT_KEY"),
+            ("delegated_user", "GW_DELEGATED_USER"),
+        ],
+        "options": [
+            ("gmail_enabled", "GW_GMAIL", "bool"),
+            ("drive_enabled", "GW_DRIVE", "bool"),
+            ("calendar_enabled", "GW_CALENDAR", "bool"),
+            ("shared_drives", "GW_SHARED_DRIVES", "list"),
+            ("calendar_ids", "GW_CALENDARS", "list"),
+        ],
+    },
+    "salesforce": {
+        "import": "cortexdb_connectors.salesforce:SalesforceConnector",
+        "secrets": [
+            ("instance_url", "SF_INSTANCE_URL"),
+            ("client_id", "SF_CLIENT_ID"),
+            ("client_secret", "SF_CLIENT_SECRET"),
+            ("username", "SF_USERNAME"),
+            ("password", "SF_PASSWORD"),
+        ],
+        "options": [("objects", "SF_OBJECTS", "list")],
+    },
+    "hubspot": {
+        "import": "cortexdb_connectors.hubspot:HubSpotConnector",
+        "secrets": [("access_token", "HUBSPOT_TOKEN")],
+        "options": [("objects", "HUBSPOT_OBJECTS", "list")],
+    },
+    "zendesk": {
+        "import": "cortexdb_connectors.zendesk:ZendeskConnector",
+        "secrets": [
+            ("subdomain", "ZENDESK_SUBDOMAIN"),
+            ("email", "ZENDESK_EMAIL"),
+            ("api_token", "ZENDESK_TOKEN"),
+        ],
+        "options": [("include_help_center", "ZENDESK_HELP_CENTER", "bool")],
+    },
+    "intercom": {
+        "import": "cortexdb_connectors.intercom:IntercomConnector",
+        "secrets": [("access_token", "INTERCOM_TOKEN")],
+        "options": [("include_articles", "INTERCOM_ARTICLES", "bool")],
+    },
+    "servicenow": {
+        "import": "cortexdb_connectors.servicenow:ServiceNowConnector",
+        "secrets": [
+            ("instance", "SNOW_INSTANCE"),
+            ("username", "SNOW_USERNAME"),
+            ("password", "SNOW_PASSWORD"),
+        ],
+        "options": [("tables", "SNOW_TABLES", "list")],
+    },
+}
+
+
+# ---------------------------------------------------------------------------
+# CortexDB credential resolution
+# ---------------------------------------------------------------------------
+
+
+def _state_file_path() -> Path:
+    return Path.home() / ".cortexdb" / "state.json"
+
+
+def _load_cortex_creds(args: argparse.Namespace) -> dict[str, str]:
+    """Resolve CortexDB url/api_key/actor/scope_template from flags → env → state.json."""
+    creds: dict[str, str] = {
+        "api_url": "",
+        "api_key": "",
+        "actor": "",
+        "scope_template": "",
+    }
+
+    sf = _state_file_path()
+    if sf.exists():
+        try:
+            data = json.loads(sf.read_text(encoding="utf-8"))
+            creds["api_key"] = data.get("token", "") or creds["api_key"]
+            creds["actor"] = data.get("user_id", "") or creds["actor"]
+            creds["scope_template"] = data.get("scope", "") or creds["scope_template"]
+        except (OSError, json.JSONDecodeError):
+            pass
+
+    env_map = {
+        "api_url": ("CORTEXDB_URL", "CORTEX_URL"),
+        "api_key": ("CORTEXDB_API_KEY", "CORTEX_API_KEY"),
+        "actor": ("CORTEXDB_ACTOR",),
+        "scope_template": ("CORTEXDB_SCOPE_TEMPLATE", "CORTEXDB_SCOPE"),
+    }
+    for key, vars_ in env_map.items():
+        for v in vars_:
+            if os.environ.get(v):
+                creds[key] = os.environ[v]
+                break
+
+    if getattr(args, "api_url", None):
+        creds["api_url"] = args.api_url
+    if getattr(args, "api_key", None):
+        creds["api_key"] = args.api_key
+    if getattr(args, "actor", None):
+        creds["actor"] = args.actor
+    if getattr(args, "scope_template", None):
+        creds["scope_template"] = args.scope_template
+
+    if not creds["api_url"]:
+        creds["api_url"] = "https://api-v1.cortexdb.ai"
+    if not creds["scope_template"] and creds["actor"]:
+        creds["scope_template"] = creds["actor"]
+    return creds
+
+
+# ---------------------------------------------------------------------------
+# Connector factory
+# ---------------------------------------------------------------------------
+
+_OPTION_PARSERS: dict[str, Callable[[str], Any]] = {
+    "str": lambda v: v,
+    "list": lambda v: [x.strip() for x in v.split(",") if x.strip()],
+    "list-int": lambda v: [int(x.strip()) for x in v.split(",") if x.strip()],
+    "bool": lambda v: v.lower() in ("1", "true", "yes", "on"),
+}
+
+
+def _resolve_kwarg(env_var: str, config_value: Any, parser: str) -> Any:
+    """Pull a value from config or env, parse it according to its type."""
+    if config_value is not None:
+        return config_value
+    raw = os.environ.get(env_var, "")
+    if not raw:
+        return None
+    return _OPTION_PARSERS[parser](raw)
+
+
+def _build_connector(
+    slug: str,
+    creds: dict[str, str],
+    config: dict[str, Any],
+    tenant_id: str,
+) -> Any:
+    if slug not in CONNECTORS:
+        raise ValueError(f"Unknown connector: {slug!r}. Try `cortexdb-sync list`.")
+    spec = CONNECTORS[slug]
+    module_name, class_name = spec["import"].split(":")
+    module = importlib.import_module(module_name)
+    cls = getattr(module, class_name)
+
+    connector_cfg = (config or {}).get(slug, {})
+
+    kwargs: dict[str, Any] = {
+        "cortex_url": creds["api_url"],
+        "cortex_api_key": creds["api_key"],
+        "tenant_id": tenant_id,
+        "actor": creds["actor"],
+        "scope_template": creds["scope_template"],
+    }
+
+    for kw_name, env_var in spec.get("secrets", []):
+        value = _resolve_kwarg(env_var, connector_cfg.get(kw_name), "str")
+        if not value:
+            raise SystemExit(
+                f"Missing required secret for {slug}: set --{slug}-{kw_name} via "
+                f"config or env {env_var}."
+            )
+        kwargs[kw_name] = value
+
+    for kw_name, env_var, parser in spec.get("options", []):
+        value = _resolve_kwarg(env_var, connector_cfg.get(kw_name), parser)
+        if value is not None:
+            kwargs[kw_name] = value
+
+    return cls(**kwargs)
+
+
+# ---------------------------------------------------------------------------
+# Commands
+# ---------------------------------------------------------------------------
+
+
+def _cmd_sync(args: argparse.Namespace) -> None:
+    creds = _load_cortex_creds(args)
+    config = _load_config(args.config)
+    state_store = SyncStateStore(path=args.state_file)
+
+    since: datetime | None = None
+    if args.since:
+        since = datetime.fromisoformat(args.since)
+        if since.tzinfo is None:
+            since = since.replace(tzinfo=timezone.utc)
+    else:
+        st = state_store.get(args.connector, args.tenant_id)
+        since = st.last_synced_at
+
+    connector = _build_connector(args.connector, creds, config, args.tenant_id)
+
+    logger.info(
+        "sync start  connector=%s  tenant=%s  scope=%s  since=%s",
+        args.connector,
+        args.tenant_id,
+        creds["scope_template"],
+        since,
+    )
+
+    result = asyncio.run(connector.sync(since=since))
+
+    state_store.update_last_synced(args.connector, args.tenant_id)
+
+    logger.info(
+        "sync done   fetched=%d ingested=%d errors=%d duration=%.1fs",
+        result.episodes_fetched,
+        result.episodes_ingested,
+        len(result.errors),
+        result.duration_seconds,
+    )
+    for err in result.errors:
+        logger.error("  %s", err)
+    if result.errors:
+        sys.exit(1)
+
+
+def _cmd_watch(args: argparse.Namespace) -> None:
+    """Repeat sync forever with a sleep between cycles."""
+    while True:
+        try:
+            _cmd_sync(args)
+        except SystemExit as exc:
+            # Don't exit the watch loop on per-cycle errors.
+            logger.error("cycle failed: %s", exc)
+        logger.info("sleeping %ds before next cycle", args.interval)
+        time.sleep(args.interval)
+
+
+def _cmd_status(args: argparse.Namespace) -> None:
+    state_store = SyncStateStore(path=args.state_file)
+    states = state_store.list_all()
+    if not states:
+        print("No sync state recorded yet.")
+        return
+    print(f"{'Connector':<20} {'Tenant':<20} {'Last synced':<30} {'Cursor'}")
+    print("-" * 90)
+    for s in states:
+        last = s.last_synced_at.isoformat() if s.last_synced_at else "never"
+        cursor = s.cursor or "-"
+        print(f"{s.connector_name:<20} {s.tenant_id:<20} {last:<30} {cursor}")
+
+
+def _cmd_list(_: argparse.Namespace) -> None:
+    print("Available connectors:\n")
+    for slug, spec in CONNECTORS.items():
+        env_vars = [v for _, v in spec.get("secrets", [])]
+        print(f"  {slug:<20} required env: {', '.join(env_vars)}")
+
+
+def _cmd_auth(args: argparse.Namespace) -> None:
+    creds = _load_cortex_creds(args)
+    redacted = {
+        "api_url": creds["api_url"],
+        "api_key": (creds["api_key"][:8] + "..." + creds["api_key"][-4:]) if creds["api_key"] else "(unset)",
+        "actor": creds["actor"] or "(unset)",
+        "scope_template": creds["scope_template"] or "(unset)",
+    }
+    print(json.dumps(redacted, indent=2))
+
+
+def _load_config(path: str | None) -> dict[str, Any]:
+    if path is None:
+        candidates = [
+            Path.cwd() / "cortexdb-connectors.yaml",
+            Path.cwd() / "cortexdb-connectors.yml",
+            Path.home() / ".cortexdb" / "connectors.yaml",
+        ]
+        for candidate in candidates:
+            if candidate.exists():
+                path = str(candidate)
+                break
+    if path is None:
+        return {}
+    try:
+        import yaml  # type: ignore[import-untyped]
+    except ImportError:
+        logger.warning("PyYAML is not installed; ignoring config file %s", path)
+        return {}
+    with open(path, "r", encoding="utf-8") as fh:
+        data = yaml.safe_load(fh)
+    return data if isinstance(data, dict) else {}
+
+
+# ---------------------------------------------------------------------------
+# Entry point
+# ---------------------------------------------------------------------------
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(
+        prog="cortexdb-sync",
+        description="CortexDB v1 data connector CLI",
+    )
+    parser.add_argument("--config", type=str, default=None, help="YAML config file path.")
+    parser.add_argument("--state-file", type=str, default=None, help="Sync state JSON file path.")
+    parser.add_argument("--api-url", type=str, help="CortexDB v1 API base URL.")
+    parser.add_argument("--api-key", type=str, help="PASETO bearer token.")
+    parser.add_argument("--actor", type=str, help="X-Cortex-Actor value.")
+    parser.add_argument("--scope-template", type=str, help='Scope template, e.g. "org:acme/source:slack".')
+    parser.add_argument("-v", "--verbose", action="store_true", help="Debug logging.")
+    sub = parser.add_subparsers(dest="command")
+
+    sync_p = sub.add_parser("sync", help="One-shot sync for a connector.")
+    sync_p.add_argument("connector", choices=list(CONNECTORS.keys()))
+    sync_p.add_argument("--since", type=str, default=None)
+    sync_p.add_argument("--tenant-id", type=str, default="default")
+
+    watch_p = sub.add_parser("watch", help="Sync forever with a sleep between cycles.")
+    watch_p.add_argument("connector", choices=list(CONNECTORS.keys()))
+    watch_p.add_argument("--since", type=str, default=None)
+    watch_p.add_argument("--tenant-id", type=str, default="default")
+    watch_p.add_argument("--interval", type=int, default=300, help="Seconds between cycles (default 300).")
+
+    sub.add_parser("status", help="Show cursor state for every connector.")
+    sub.add_parser("list", help="List available connectors and required env vars.")
+    sub.add_parser("auth", help="Show resolved CortexDB credentials (redacted).")
+
+    args = parser.parse_args(argv)
+
+    logging.basicConfig(
+        level=logging.DEBUG if args.verbose else logging.INFO,
+        format="%(asctime)s %(levelname)-8s %(name)s: %(message)s",
+    )
+
+    if args.command == "sync":
+        _cmd_sync(args)
+    elif args.command == "watch":
+        _cmd_watch(args)
+    elif args.command == "status":
+        _cmd_status(args)
+    elif args.command == "list":
+        _cmd_list(args)
+    elif args.command == "auth":
+        _cmd_auth(args)
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()