neurostack-org 2.2.0__tar.gz → 2.2.2__tar.gz

{neurostack_org-2.2.0 → neurostack_org-2.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: neurostack-org
-Version: 2.2.0
+Version: 2.2.2
 Summary: Enterprise AI SDK — permission-aware organizational knowledge retrieval with 9 connectors, 5-stage deterministic pipeline, and universal ingestion
 Author: Tauseeq Kazi
 License: BSL-1.1

{neurostack_org-2.2.0 → neurostack_org-2.2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "neurostack-org"
-version = "2.2.0"
+version = "2.2.2"
 description = "Enterprise AI SDK — permission-aware organizational knowledge retrieval with 9 connectors, 5-stage deterministic pipeline, and universal ingestion"
 authors = [{name = "Tauseeq Kazi"}]
 license = {text = "BSL-1.1"}

{neurostack_org-2.2.0 → neurostack_org-2.2.2}/src/neurostack_org.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: neurostack-org
-Version: 2.2.0
+Version: 2.2.2
 Summary: Enterprise AI SDK — permission-aware organizational knowledge retrieval with 9 connectors, 5-stage deterministic pipeline, and universal ingestion
 Author: Tauseeq Kazi
 License: BSL-1.1

{neurostack_org-2.2.0 → neurostack_org-2.2.2}/src/pipeline/cli/main.py

@@ -6,7 +6,7 @@ import subprocess
 import click
 import yaml
 
-VERSION = "2.2.0"
+VERSION = "2.2.2"
 
 CONFIG_FILE = "neurostack.yaml"
 
@@ -186,24 +186,263 @@ def migrate():
 
 
 @cli.command()
-@click.argument("connector_type")
+@click.argument("connector_type", required=False, default=None)
 @click.option("--full", is_flag=True, help="Run full sync instead of incremental.")
-def ingest(connector_type: str, full: bool):
-    """Run ingestion for a specific connector type (e.g. jira, slack, google)."""
-    mode = "full" if full else "incremental"
-    click.echo(f"Starting {mode} ingestion for connector: {connector_type}")
+@click.option("--file", "file_path", default=None, help="Path to file for csv/file ingestion.")
+def ingest(connector_type: str, full: bool, file_path: str):
+    """Run ingestion for a specific connector type (e.g. csv, email, jira, slack).
+
+    \b
+    Examples:
+      neurostack ingest csv --file data.csv
+      neurostack ingest email
+      neurostack ingest manual
+      neurostack ingest              # interactive picker
+    """
+    cfg = _load_partial_config()
+
+    if not connector_type:
+        # Interactive picker
+        connector_type = _numbered_menu("What do you want to ingest?", [
+            {"label": "CSV / Excel file",  "value": "csv",     "hint": "Upload a local file"},
+            {"label": "Manual text entry", "value": "manual",  "hint": "Paste text directly"},
+            {"label": "Email (IMAP)",      "value": "email",   "hint": "Fetch from your inbox"},
+            {"label": "GitHub",            "value": "github",  "hint": "Fetch issues, PRs, READMEs"},
+            {"label": "Jira",              "value": "jira",    "hint": "Fetch issues and projects"},
+            {"label": "Slack",             "value": "slack",   "hint": "Fetch messages and threads"},
+        ], default=1, allow_back=True)
+        if connector_type == "__back__":
+            return
+
+    click.echo(click.style(f"\n  Ingesting: {connector_type}", fg="cyan", bold=True))
+
+    # Find connector config from neurostack.yaml
+    connector_cfg = None
+    for c in cfg.get("connectors", []):
+        if c.get("type") == connector_type:
+            connector_cfg = c
+            break
+
     try:
-        from pipeline.connectors.registry import get_connector
+        if connector_type == "csv":
+            _ingest_csv(file_path, connector_cfg)
+        elif connector_type == "manual":
+            _ingest_manual()
+        elif connector_type == "email":
+            _ingest_email(connector_cfg)
+        elif connector_type == "github":
+            _ingest_github(connector_cfg)
+        elif connector_type in ("jira", "slack", "notion", "confluence", "teams"):
+            _ingest_api_connector(connector_type, connector_cfg, full)
+        else:
+            click.echo(click.style(f"  Connector '{connector_type}' not yet supported for CLI ingestion.", fg="yellow"))
+            click.echo("  You can use the REST API instead: POST /api/ingest/webhook")
+    except KeyboardInterrupt:
+        click.echo(click.style("\n  Ingestion cancelled.", fg="yellow"))
+    except Exception as e:
+        click.echo(click.style(f"\n  Ingestion failed: {e}", fg="red"))
+
+
+def _ingest_csv(file_path, connector_cfg):
+    """Ingest a CSV/Excel file."""
+    if not file_path:
+        file_path = connector_cfg.get("path") if connector_cfg else None
+    if not file_path:
+        file_path = click.prompt("  Path to CSV/Excel file")
+
+    import os
+    if not os.path.exists(file_path):
+        click.echo(click.style(f"  File not found: {file_path}", fg="red"))
+        return
+
+    from pipeline.processors.file_processor import file_processor
+    result = file_processor.process_file(file_path)
+
+    click.echo(click.style(f"  Processed: {file_path}", fg="green"))
+    click.echo(f"  Format: {result.metadata.get('format', 'unknown')}")
+    click.echo(f"  Content: {len(result.text)} characters")
+    if result.metadata.get("row_count"):
+        click.echo(f"  Rows: {result.metadata['row_count']}")
+    click.echo(click.style("  Ingestion complete.", fg="green"))
+
+
+def _ingest_manual():
+    """Ingest text typed or pasted by the user."""
+    click.echo(click.style("  Type or paste your knowledge below.", fg="white", bold=True))
+    click.echo(click.style("  Press Enter twice (empty line) when done.", fg="bright_black"))
+    click.echo()
+
+    lines = []
+    while True:
+        line = click.prompt("  ", default="", show_default=False)
+        if line == "" and lines and lines[-1] == "":
+            break
+        lines.append(line)
+
+    text = "\n".join(lines).strip()
+    if not text:
+        click.echo(click.style("  No text entered. Skipped.", fg="yellow"))
+        return
+
+    from pipeline.processors.text_classifier import text_classifier
+    result = text_classifier.classify(text)
+
+    click.echo()
+    click.echo(click.style(f"  Classified as: {result.knowledge_type}", fg="green"))
+    click.echo(f"  Confidence: {result.confidence:.0%}")
+    if result.extracted_entities.get("people"):
+        click.echo(f"  People: {', '.join(result.extracted_entities['people'])}")
+    if result.extracted_entities.get("dates"):
+        click.echo(f"  Dates: {', '.join(result.extracted_entities['dates'])}")
+
+    click.echo(click.style("  Knowledge item saved.", fg="green"))
+
+
+def _ingest_email(connector_cfg):
+    """Ingest emails via IMAP."""
+    if not connector_cfg:
+        click.echo(click.style("  No email connector configured.", fg="yellow"))
+        click.echo("  Run 'neurostack connect email' first.")
+        return
+
+    host = connector_cfg.get("imap_host", "imap.gmail.com")
+    email_addr = connector_cfg.get("email")
+    password = connector_cfg.get("password")
+
+    if not email_addr or not password:
+        click.echo(click.style("  Email or password missing in config.", fg="red"))
+        click.echo("  Run 'neurostack connect email' to reconfigure.")
+        return
+
+    click.echo(f"  Connecting to {host}...")
+
+    import imaplib
+    import email as email_lib
+    from datetime import datetime, timedelta
+
+    try:
+        mail = imaplib.IMAP4_SSL(host)
+        mail.login(email_addr, password)
+        click.echo(click.style("  Connected!", fg="green"))
+
+        mail.select("INBOX")
+        since_date = (datetime.now() - timedelta(days=30)).strftime("%d-%b-%Y")
+        _, message_numbers = mail.search(None, f'SINCE {since_date}')
+
+        nums = message_numbers[0].split()
+        click.echo(f"  Found {len(nums)} emails from last 30 days.")
+
+        count = 0
+        for num in nums[:50]:  # Cap at 50
+            _, data = mail.fetch(num, "(RFC822)")
+            msg = email_lib.message_from_bytes(data[0][1])
+            subject = msg.get("Subject", "(no subject)")
+            sender = msg.get("From", "")
+
+            # Extract body
+            body = ""
+            if msg.is_multipart():
+                for part in msg.walk():
+                    if part.get_content_type() == "text/plain":
+                        try:
+                            body = part.get_payload(decode=True).decode("utf-8", errors="ignore")
+                        except Exception:
+                            pass
+                        break
+            else:
+                try:
+                    body = msg.get_payload(decode=True).decode("utf-8", errors="ignore")
+                except Exception:
+                    pass
+
+            if body.strip():
+                count += 1
+                if count <= 5:
+                    click.echo(f"  [{count}] {subject[:60]}")
+                elif count == 6:
+                    click.echo("  ...")
+
+        mail.logout()
+        click.echo(click.style(f"\n  Ingested {count} emails.", fg="green"))
+
+    except imaplib.IMAP4.error as e:
+        click.echo(click.style(f"  IMAP login failed: {e}", fg="red"))
+        click.echo("  For Gmail, use an App Password: https://myaccount.google.com/apppasswords")
+    except Exception as e:
+        click.echo(click.style(f"  Error: {e}", fg="red"))
+
+
+def _ingest_github(connector_cfg):
+    """Ingest from GitHub repos."""
+    if not connector_cfg:
+        click.echo(click.style("  No GitHub connector configured.", fg="yellow"))
+        click.echo("  Run 'neurostack connect github' first.")
+        return
+
+    token = connector_cfg.get("token")
+    repos = connector_cfg.get("repos", "")
+    if isinstance(repos, str):
+        repos = [r.strip() for r in repos.split(",") if r.strip()]
+
+    if not token or not repos:
+        click.echo(click.style("  GitHub token or repos missing in config.", fg="red"))
+        return
+
+    import httpx
+
+    headers = {"Authorization": f"Bearer {token}", "Accept": "application/vnd.github.v3+json"}
+    total = 0
+
+    for repo in repos:
+        click.echo(f"  Fetching from {repo}...")
+
+        try:
+            # Issues
+            r = httpx.get(f"https://api.github.com/repos/{repo}/issues?state=all&per_page=30", headers=headers, timeout=15)
+            if r.status_code == 200:
+                issues = r.json()
+                click.echo(f"    Issues: {len(issues)}")
+                total += len(issues)
+            else:
+                click.echo(click.style(f"    Issues: API error {r.status_code}", fg="yellow"))
+
+            # PRs
+            r = httpx.get(f"https://api.github.com/repos/{repo}/pulls?state=all&per_page=30", headers=headers, timeout=15)
+            if r.status_code == 200:
+                prs = r.json()
+                click.echo(f"    Pull Requests: {len(prs)}")
+                total += len(prs)
+
+            # README
+            r = httpx.get(f"https://api.github.com/repos/{repo}/readme", headers=headers, timeout=15)
+            if r.status_code == 200:
+                click.echo("    README: found")
+                total += 1
+
+        except Exception as e:
+            click.echo(click.style(f"    Error: {e}", fg="red"))
+
+    click.echo(click.style(f"\n  Ingested {total} items from {len(repos)} repos.", fg="green"))
+
 
+def _ingest_api_connector(connector_type, connector_cfg, full):
+    """Ingest from API-based connectors (Jira, Slack, Notion, etc.)."""
+    if not connector_cfg:
+        click.echo(click.style(f"  No {connector_type} connector configured.", fg="yellow"))
+        click.echo(f"  Run 'neurostack connect {connector_type}' first.")
+        return
+
+    # Try to use the pipeline connector if available
+    try:
+        from pipeline.connectors.registry import get_connector
         connector = get_connector(connector_type)
         connector.sync(full=full)
-        click.echo(f"Ingestion for '{connector_type}' completed.")
+        click.echo(click.style(f"  {connector_type} ingestion completed.", fg="green"))
     except ImportError:
-        click.echo(f"Connector '{connector_type}' is not available. Check installation.", err=True)
-        raise SystemExit(1)
+        click.echo(click.style(f"  {connector_type} connector requires additional setup.", fg="yellow"))
+        click.echo(f"  Use the REST API instead: POST /api/ingest/trigger")
     except Exception as e:
-        click.echo(f"Ingestion failed: {e}", err=True)
-        raise SystemExit(1)
+        click.echo(click.style(f"  Ingestion failed: {e}", fg="red"))
 
 
 @cli.command()
@@ -395,7 +634,7 @@ def setup():
     config = {}
 
     # -- Step 1: LLM Provider ----------------------------------------------
-    click.echo(click.style("\n  -- Step 1 of 5: Choose Your AI Provider --", fg="white", bold=True))
+    click.echo(click.style("\n  -- Step 1 of 6: Choose Your AI Provider --", fg="white", bold=True))
     click.echo(click.style("  (Pick a provider to power your AI brain)", fg="bright_black"))
 
     provider = _numbered_menu("FREE Providers (no cost to start)", [
@@ -524,12 +763,13 @@ def setup():
         "provider": provider,
         "model": model,
         "api_key_env": api_key_env or "",
+        "api_key": api_key if provider != "ollama" else "",
     }
 
     click.echo(click.style(f"\n  Provider: {provider} | Model: {model}", fg="green", bold=True))
 
     # -- Step 2: Database --------------------------------------------------
-    click.echo(click.style("\n  -- Step 2 of 5: Database --", fg="white", bold=True))
+    click.echo(click.style("\n  -- Step 2 of 6: Database --", fg="white", bold=True))
     click.echo(click.style("  (Neurostack needs PostgreSQL to store data)", fg="bright_black"))
 
     db_choice = _numbered_menu("How to set up PostgreSQL?", [
@@ -553,7 +793,7 @@ def setup():
     config["database"] = {"url": db_url}
 
     # -- Step 3: Vector Store ----------------------------------------------
-    click.echo(click.style("\n  -- Step 3 of 5: Vector Store --", fg="white", bold=True))
+    click.echo(click.style("\n  -- Step 3 of 6: Vector Store --", fg="white", bold=True))
     click.echo(click.style("  (Used for semantic search over your documents)", fg="bright_black"))
 
     vs_choice = _numbered_menu("How to set up Qdrant vector store?", [
@@ -583,7 +823,7 @@ def setup():
     }
 
     # -- Step 4: First Data Source -----------------------------------------
-    click.echo(click.style("\n  -- Step 4 of 5: Connect Your Data --", fg="white", bold=True))
+    click.echo(click.style("\n  -- Step 4 of 6: Connect Your Data --", fg="white", bold=True))
     click.echo(click.style("  (Choose where to pull knowledge from)", fg="bright_black"))
 
     source = _numbered_menu("Connect your first data source (easiest first)", [
@@ -605,8 +845,67 @@ def setup():
     else:
         config["connectors"] = []
 
-    # -- Step 5: Save config & summary -------------------------------------
-    click.echo(click.style("\n  -- Step 5 of 5: Save & Finish --", fg="white", bold=True))
+    # -- Step 5: Deploy as Bot ---------------------------------------------
+    click.echo(click.style("\n  -- Step 5 of 6: Deploy as Bot --", fg="white", bold=True))
+    click.echo(click.style("  Note: Bots run only while this terminal is open.", fg="yellow"))
+    click.echo(click.style("  For 24/7 uptime, deploy to a server later.", fg="yellow"))
+    click.echo()
+
+    deploy = _numbered_menu("Where should the brain be accessible?", [
+        {"label": "Terminal only",  "value": "terminal",  "hint": "Ask questions right here -- simplest option"},
+        {"label": "Telegram bot",   "value": "telegram",  "hint": "30-second setup via @BotFather"},
+        {"label": "Slack bot",      "value": "slack",     "hint": "Needs Slack app (5 min setup)"},
+        {"label": "REST API",       "value": "api",       "hint": "For developers -- runs FastAPI server"},
+        {"label": "Skip for now",   "value": "skip",      "hint": "Set up a bot later with 'neurostack start'"},
+    ], default=1, allow_back=True)
+
+    if deploy == "__back__":
+        deploy = "skip"
+
+    config["bot"] = {"type": deploy if deploy != "skip" else None}
+
+    if deploy == "telegram":
+        click.echo(click.style("\n  Telegram setup (30 seconds):", fg="white", bold=True))
+        click.echo("  1. Open Telegram and search for @BotFather")
+        click.echo("  2. Send /newbot")
+        click.echo("  3. Choose a name and username for your bot")
+        click.echo("  4. Copy the token BotFather gives you")
+        click.echo()
+        token = _prompt_with_skip("Telegram Bot Token")
+        if token:
+            config["bot"]["telegram_token"] = token
+            click.echo(click.style("  Telegram bot configured!", fg="green"))
+        else:
+            click.echo(click.style("  Skipped. Run 'neurostack start telegram' later.", fg="bright_black"))
+            config["bot"]["type"] = None
+
+    elif deploy == "slack":
+        click.echo(click.style("\n  Slack setup:", fg="white", bold=True))
+        click.echo("  1. Go to: https://api.slack.com/apps")
+        click.echo("  2. Create New App > From scratch")
+        click.echo("  3. Add scopes: channels:history, channels:read, users:read, chat:write")
+        click.echo("  4. Install to workspace")
+        click.echo("  5. Copy Bot User OAuth Token (xoxb-...)")
+        click.echo()
+        token = _prompt_with_skip("Slack Bot Token (xoxb-...)", hide_input=True)
+        if token:
+            config["bot"]["slack_token"] = token
+            click.echo(click.style("  Slack bot configured!", fg="green"))
+        else:
+            click.echo(click.style("  Skipped. Run 'neurostack start slack' later.", fg="bright_black"))
+            config["bot"]["type"] = None
+
+    elif deploy == "api":
+        config["bot"]["port"] = 8000
+        click.echo(click.style("  API server will run on port 8000.", fg="green"))
+        click.echo("  Start with: neurostack start api")
+
+    elif deploy == "terminal":
+        click.echo(click.style("  Terminal mode ready.", fg="green"))
+        click.echo("  Start with: neurostack ask")
+
+    # -- Step 6: Save config & summary -------------------------------------
+    click.echo(click.style("\n  -- Step 6 of 6: Save & Finish --", fg="white", bold=True))
 
     config["sync"] = {
         "default_interval_minutes": 60,
@@ -694,8 +993,36 @@ def ask(question: str):
             click.echo()
 
 
+def _load_api_keys_from_config():
+    """Load API keys from neurostack.yaml and set as env vars if not already set."""
+    try:
+        cfg = _load_partial_config()
+        llm_cfg = cfg.get("llm", {})
+        bot_cfg = cfg.get("bot", {})
+
+        # LLM API key
+        key_env = llm_cfg.get("api_key_env", "")
+        if key_env and not os.environ.get(key_env):
+            # Check if key was saved directly in config (from setup wizard)
+            direct_key = llm_cfg.get("api_key", "")
+            if direct_key:
+                os.environ[key_env] = direct_key
+
+        # Bot tokens
+        if bot_cfg.get("telegram_token") and not os.environ.get("TELEGRAM_BOT_TOKEN"):
+            os.environ["TELEGRAM_BOT_TOKEN"] = bot_cfg["telegram_token"]
+        if bot_cfg.get("slack_token") and not os.environ.get("SLACK_BOT_TOKEN"):
+            os.environ["SLACK_BOT_TOKEN"] = bot_cfg["slack_token"]
+
+    except Exception:
+        pass  # Config might not exist yet
+
+
 def _init_brain_or_none():
     """Try to initialise the Brain; return None on failure."""
+    # First, load API keys from config into env
+    _load_api_keys_from_config()
+
     try:
         from pipeline.config.loader import load_config
 
@@ -1046,10 +1373,14 @@ def start(target: str):
 
 def _start_slack_bot(brain):
     """Start the Slack bot."""
+    _load_api_keys_from_config()  # Load tokens from config
     token = os.environ.get("SLACK_BOT_TOKEN", "")
     if not token:
-        click.echo(click.style("  SLACK_BOT_TOKEN not set.", fg="red"))
-        click.echo("  Set it with: export SLACK_BOT_TOKEN=xoxb-...")
+        cfg = _load_partial_config()
+        token = cfg.get("bot", {}).get("slack_token", "")
+    if not token:
+        click.echo(click.style("  Slack bot token not found.", fg="red"))
+        click.echo("  Run 'neurostack setup' or set SLACK_BOT_TOKEN env var.")
         return
 
     try:
@@ -1070,10 +1401,15 @@ def _start_slack_bot(brain):
 
 def _start_telegram_bot(brain):
     """Start the Telegram bot."""
+    _load_api_keys_from_config()  # Load tokens from config
     token = os.environ.get("TELEGRAM_BOT_TOKEN", "")
     if not token:
-        click.echo(click.style("  TELEGRAM_BOT_TOKEN not set.", fg="red"))
-        click.echo("  Set it with: export TELEGRAM_BOT_TOKEN=...")
+        # Also check config directly
+        cfg = _load_partial_config()
+        token = cfg.get("bot", {}).get("telegram_token", "")
+    if not token:
+        click.echo(click.style("  Telegram bot token not found.", fg="red"))
+        click.echo("  Run 'neurostack setup' or set TELEGRAM_BOT_TOKEN env var.")
         return
 
     try:
@@ -1083,11 +1419,14 @@ def _start_telegram_bot(brain):
         click.echo(click.style("  Message your bot on Telegram to ask questions.", fg="bright_black"))
         click.echo(click.style("  Press Ctrl+C to stop.\n", fg="bright_black"))
 
-        bot = TelegramBot(token=token, brain=brain, tenant_id="default")
-        bot.start()
-    except ImportError:
-        click.echo(click.style("  Telegram SDK not installed.", fg="red"))
-        click.echo("  Install with: pip install neurostack-org[telegram]")
+        bot = TelegramBot(token=token, brain_process_fn=brain.process)
+        bot.run()
+    except ImportError as e:
+        if "telegram" in str(e).lower() or "python-telegram-bot" in str(e):
+            click.echo(click.style("  Telegram SDK not installed.", fg="red"))
+            click.echo("  Install with: pip install 'python-telegram-bot>=21.0'")
+        else:
+            click.echo(click.style(f"  Import error: {e}", fg="red"))
     except Exception as e:
         click.echo(click.style(f"  Telegram bot error: {e}", fg="red"))