icdev 0.0.3__py3-none-any.whl

tools/research/research_manager.py

@@ -0,0 +1,469 @@
+#!/usr/bin/env python3
+# CUI // SP-CTI
+"""Research Manager — session lifecycle, pipeline orchestration, and dossier retrieval.
+
+Tables used:
+  - research_sessions     (read/write — lifecycle management)
+  - research_verticals    (read)
+  - research_dossiers     (read)
+  - audit_trail           (write — append-only)
+
+CLI:
+  python tools/research/research_manager.py --status --json
+  python tools/research/research_manager.py --create --vertical defense --json
+  python tools/research/research_manager.py --resume --session-id <id> --json
+  python tools/research/research_manager.py --run --session-id <id> --json
+  python tools/research/research_manager.py --stage scope --session-id <id> --json
+  python tools/research/research_manager.py --dossier --session-id <id> --json
+  python tools/research/research_manager.py --review --session-id <id> --decision approved --reviewer user --json
+  python tools/research/research_manager.py --list --json
+  python tools/research/research_manager.py --history --session-id <id> --json
+  python tools/research/research_manager.py --archive --session-id <id> --json
+"""
+
+import argparse
+import json
+import logging
+import sqlite3
+import sys
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+BASE_DIR = Path(__file__).resolve().parent.parent.parent
+DB_PATH = BASE_DIR / "data" / "icdev.db"
+
+try:
+    from tools.compat.db_utils import get_db_connection
+except ImportError:
+    get_db_connection = None
+
+PIPELINE_STAGES = [
+    "SCOPE", "LANDSCAPE", "REGULATE", "COMMUNITY",
+    "ACADEMIC", "BUILD_BUY", "SYNTHESIZE", "FORECAST", "DOSSIER",
+]
+
+VALID_STATUSES = [
+    "created", "scoping", "scanning", "synthesizing",
+    "dossier_ready", "reviewed", "child_app_triggered", "archived",
+]
+
+
+def _get_connection(db_path=None):
+    if db_path is None and get_db_connection is not None:
+        try:
+            conn = get_db_connection()
+            conn.row_factory = sqlite3.Row
+            return conn
+        except Exception as exc:
+            logging.debug("db_utils connection failed, falling back to direct: %s", exc)
+    path = Path(db_path) if db_path else DB_PATH
+    if not path.exists():
+        raise FileNotFoundError(f"Database not found: {path}\nRun: python tools/db/init_icdev_db.py")
+    conn = sqlite3.connect(str(path))
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA foreign_keys = ON")
+    return conn
+
+
+def _uid() -> str:
+    return uuid.uuid4().hex[:12]
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _log_audit(conn, project_id, event_type, action, details):
+    try:
+        conn.execute(
+            "INSERT INTO audit_trail "
+            "(project_id, event_type, actor, action, details, classification, created_at) "
+            "VALUES (?, ?, ?, ?, ?, ?, ?)",
+            (project_id, event_type, "icdev-research-engine", action,
+             json.dumps(details) if isinstance(details, dict) else str(details),
+             "CUI", _now()),
+        )
+        conn.commit()
+    except Exception as exc:
+        print(f"Warning: audit log failed: {exc}", file=sys.stderr)
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def get_status(db_path=None) -> dict:
+    """Return overall research engine status."""
+    conn = _get_connection(db_path)
+    try:
+        sessions = conn.execute("SELECT COUNT(*) FROM research_sessions").fetchone()[0]
+        verticals = conn.execute("SELECT COUNT(*) FROM research_verticals").fetchone()[0]
+        signals = conn.execute("SELECT COUNT(*) FROM research_signals").fetchone()[0]
+        challenges = conn.execute("SELECT COUNT(*) FROM research_challenges").fetchone()[0]
+        dossiers = conn.execute("SELECT COUNT(*) FROM research_dossiers").fetchone()[0]
+        active = conn.execute(
+            "SELECT COUNT(*) FROM research_sessions WHERE status NOT IN ('archived','reviewed','child_app_triggered')"
+        ).fetchone()[0]
+        return {
+            "status": "operational",
+            "database": str(DB_PATH),
+            "tables_present": True,
+            "total_sessions": sessions,
+            "active_sessions": active,
+            "total_verticals": verticals,
+            "total_signals": signals,
+            "total_challenges": challenges,
+            "total_dossiers": dossiers,
+        }
+    finally:
+        conn.close()
+
+
+def create_session(vertical: str, name: str = None, description: str = None,
+                   focus_areas: list = None, created_by: str = None,
+                   db_path=None) -> dict:
+    """Create a new research session for a vertical."""
+    conn = _get_connection(db_path)
+    try:
+        vert = conn.execute(
+            "SELECT id, name, slug FROM research_verticals WHERE slug = ? AND active = 1",
+            (vertical,)
+        ).fetchone()
+        if not vert:
+            available = [r["slug"] for r in conn.execute("SELECT slug FROM research_verticals WHERE active=1").fetchall()]
+            raise ValueError(f"Vertical '{vertical}' not found. Available: {available}")
+
+        session_id = f"rsess-{_uid()}"
+        session_name = name or f"Research: {vert['name']}"
+        now = _now()
+
+        conn.execute(
+            "INSERT INTO research_sessions "
+            "(id, name, vertical_id, vertical_name, description, focus_areas, "
+            "status, pipeline_stage, created_by, created_at, updated_at) "
+            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+            (session_id, session_name, vert["id"], vert["name"],
+             description or f"Competitor research for {vert['name']} vertical",
+             json.dumps(focus_areas or []), "created", "SCOPE",
+             created_by or "system", now, now),
+        )
+        conn.execute(
+            "UPDATE research_verticals SET session_count = session_count + 1, updated_at = ? WHERE id = ?",
+            (now, vert["id"]),
+        )
+        conn.commit()
+        _log_audit(conn, session_id, "research_session_created", "create_session",
+                   {"vertical": vertical, "session_id": session_id})
+        return {
+            "session_id": session_id,
+            "name": session_name,
+            "vertical": vert["name"],
+            "vertical_id": vert["id"],
+            "status": "created",
+            "pipeline_stage": "SCOPE",
+        }
+    finally:
+        conn.close()
+
+
+def resume_session(session_id: str, db_path=None) -> dict:
+    """Resume an existing research session."""
+    conn = _get_connection(db_path)
+    try:
+        row = conn.execute("SELECT * FROM research_sessions WHERE id = ?", (session_id,)).fetchone()
+        if not row:
+            raise ValueError(f"Session '{session_id}' not found")
+        return {
+            "session_id": row["id"],
+            "name": row["name"],
+            "vertical": row["vertical_name"],
+            "status": row["status"],
+            "pipeline_stage": row["pipeline_stage"],
+            "signal_count": row["signal_count"],
+            "challenge_count": row["challenge_count"],
+            "dossier_id": row["dossier_id"],
+        }
+    finally:
+        conn.close()
+
+
+def _advance_stage(conn, session_id: str, new_stage: str, new_status: str = None):
+    """Advance session to next pipeline stage."""
+    now = _now()
+    if new_status:
+        conn.execute(
+            "UPDATE research_sessions SET pipeline_stage = ?, status = ?, updated_at = ? WHERE id = ?",
+            (new_stage, new_status, now, session_id),
+        )
+    else:
+        conn.execute(
+            "UPDATE research_sessions SET pipeline_stage = ?, updated_at = ? WHERE id = ?",
+            (new_stage, now, session_id),
+        )
+    conn.commit()
+
+
+def run_pipeline(session_id: str, db_path=None) -> dict:
+    """Run all pipeline stages sequentially."""
+    from tools.research.landscape_scanner import scan_landscape
+    from tools.research.regulatory_scanner import scan_regulatory
+    from tools.research.community_scanner import scan_community
+    from tools.research.academic_scanner import scan_academic
+    from tools.research.build_buy_analyzer import analyze_build_buy
+    from tools.research.challenge_scorer import score_challenges
+    from tools.research.dossier_generator import generate_dossier
+
+    conn = _get_connection(db_path)
+    try:
+        row = conn.execute("SELECT * FROM research_sessions WHERE id = ?", (session_id,)).fetchone()
+        if not row:
+            raise ValueError(f"Session '{session_id}' not found")
+    finally:
+        conn.close()
+
+    results = {}
+    stages = [
+        ("SCOPE", "scoping", None),
+        ("LANDSCAPE", "scanning", lambda: scan_landscape(session_id, db_path=db_path)),
+        ("REGULATE", "scanning", lambda: scan_regulatory(session_id, db_path=db_path)),
+        ("COMMUNITY", "scanning", lambda: scan_community(session_id, db_path=db_path)),
+        ("ACADEMIC", "scanning", lambda: scan_academic(session_id, db_path=db_path)),
+        ("BUILD_BUY", "synthesizing", lambda: analyze_build_buy(session_id, db_path=db_path)),
+        ("SYNTHESIZE", "synthesizing", lambda: score_challenges(session_id, db_path=db_path)),
+        ("DOSSIER", "dossier_ready", lambda: generate_dossier(session_id, db_path=db_path)),
+    ]
+
+    for stage_name, status, func in stages:
+        conn = _get_connection(db_path)
+        try:
+            _advance_stage(conn, session_id, stage_name, status)
+        finally:
+            conn.close()
+        if func:
+            results[stage_name] = func()
+
+    return {"session_id": session_id, "stages_completed": list(results.keys()), "status": "dossier_ready"}
+
+
+def run_stage(session_id: str, stage: str, db_path=None) -> dict:
+    """Run a single pipeline stage."""
+    stage = stage.upper()
+    if stage not in PIPELINE_STAGES:
+        raise ValueError(f"Invalid stage '{stage}'. Must be one of {PIPELINE_STAGES}")
+
+    stage_map = {
+        "LANDSCAPE": lambda: __import__("tools.research.landscape_scanner", fromlist=["scan_landscape"]).scan_landscape(session_id, db_path=db_path),
+        "REGULATE": lambda: __import__("tools.research.regulatory_scanner", fromlist=["scan_regulatory"]).scan_regulatory(session_id, db_path=db_path),
+        "COMMUNITY": lambda: __import__("tools.research.community_scanner", fromlist=["scan_community"]).scan_community(session_id, db_path=db_path),
+        "ACADEMIC": lambda: __import__("tools.research.academic_scanner", fromlist=["scan_academic"]).scan_academic(session_id, db_path=db_path),
+        "BUILD_BUY": lambda: __import__("tools.research.build_buy_analyzer", fromlist=["analyze_build_buy"]).analyze_build_buy(session_id, db_path=db_path),
+        "SYNTHESIZE": lambda: __import__("tools.research.challenge_scorer", fromlist=["score_challenges"]).score_challenges(session_id, db_path=db_path),
+        "DOSSIER": lambda: __import__("tools.research.dossier_generator", fromlist=["generate_dossier"]).generate_dossier(session_id, db_path=db_path),
+    }
+
+    conn = _get_connection(db_path)
+    try:
+        row = conn.execute("SELECT * FROM research_sessions WHERE id = ?", (session_id,)).fetchone()
+        if not row:
+            raise ValueError(f"Session '{session_id}' not found")
+        if stage == "SCOPE":
+            _advance_stage(conn, session_id, "SCOPE", "scoping")
+            return {"session_id": session_id, "stage": "SCOPE", "status": "completed"}
+    finally:
+        conn.close()
+
+    if stage in stage_map:
+        result = stage_map[stage]()
+        return {"session_id": session_id, "stage": stage, "result": result}
+    return {"session_id": session_id, "stage": stage, "status": "no_handler"}
+
+
+def get_dossier(session_id: str, db_path=None) -> dict:
+    """Retrieve the generated dossier for a session."""
+    conn = _get_connection(db_path)
+    try:
+        row = conn.execute(
+            "SELECT * FROM research_dossiers WHERE session_id = ? ORDER BY generated_at DESC LIMIT 1",
+            (session_id,)
+        ).fetchone()
+        if not row:
+            raise ValueError(f"No dossier found for session '{session_id}'")
+        return dict(row)
+    finally:
+        conn.close()
+
+
+def review_dossier(session_id: str, decision: str, reviewer: str,
+                   notes: str = None, db_path=None) -> dict:
+    """Submit HITL review decision on a dossier."""
+    if decision not in ("approved", "rejected"):
+        raise ValueError("Decision must be 'approved' or 'rejected'")
+    conn = _get_connection(db_path)
+    try:
+        dossier = conn.execute(
+            "SELECT id FROM research_dossiers WHERE session_id = ? ORDER BY generated_at DESC LIMIT 1",
+            (session_id,)
+        ).fetchone()
+        if not dossier:
+            raise ValueError(f"No dossier found for session '{session_id}'")
+        now = _now()
+        conn.execute(
+            "UPDATE research_dossiers SET status = ?, reviewer = ?, reviewed_at = ?, review_notes = ? WHERE id = ?",
+            (decision, reviewer, now, notes, dossier["id"]),
+        )
+        conn.execute(
+            "UPDATE research_sessions SET status = 'reviewed', updated_at = ? WHERE id = ?",
+            (now, session_id),
+        )
+        conn.commit()
+        _log_audit(conn, session_id, "research_dossier_reviewed", "review_dossier",
+                   {"decision": decision, "reviewer": reviewer})
+        return {"session_id": session_id, "dossier_id": dossier["id"],
+                "decision": decision, "reviewer": reviewer}
+    finally:
+        conn.close()
+
+
+def list_sessions(state: str = None, db_path=None) -> dict:
+    """List all research sessions, optionally filtered by state."""
+    conn = _get_connection(db_path)
+    try:
+        if state:
+            rows = conn.execute(
+                "SELECT id, name, vertical_name, status, pipeline_stage, signal_count, challenge_count, created_at "
+                "FROM research_sessions WHERE status = ? ORDER BY created_at DESC", (state,)
+            ).fetchall()
+        else:
+            rows = conn.execute(
+                "SELECT id, name, vertical_name, status, pipeline_stage, signal_count, challenge_count, created_at "
+                "FROM research_sessions ORDER BY created_at DESC"
+            ).fetchall()
+        return {"sessions": [dict(r) for r in rows], "count": len(rows)}
+    finally:
+        conn.close()
+
+
+def get_history(session_id: str, db_path=None) -> dict:
+    """Get audit history for a session."""
+    conn = _get_connection(db_path)
+    try:
+        rows = conn.execute(
+            "SELECT event_type, action, details, created_at FROM audit_trail "
+            "WHERE project_id = ? ORDER BY created_at", (session_id,)
+        ).fetchall()
+        return {"session_id": session_id, "events": [dict(r) for r in rows]}
+    finally:
+        conn.close()
+
+
+def archive_session(session_id: str, db_path=None) -> dict:
+    """Archive a research session."""
+    conn = _get_connection(db_path)
+    try:
+        now = _now()
+        conn.execute(
+            "UPDATE research_sessions SET status = 'archived', updated_at = ? WHERE id = ?",
+            (now, session_id),
+        )
+        conn.commit()
+        _log_audit(conn, session_id, "research_session_archived", "archive_session", {})
+        return {"session_id": session_id, "status": "archived"}
+    finally:
+        conn.close()
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+
+def main():
+    parser = argparse.ArgumentParser(description="Research Manager — session lifecycle")
+    parser.add_argument("--json", action="store_true", help="JSON output")
+    parser.add_argument("--db-path", help="Database path override")
+
+    group = parser.add_argument_group("actions")
+    group.add_argument("--status", action="store_true", help="Engine status")
+    group.add_argument("--create", action="store_true", help="Create new session")
+    group.add_argument("--resume", action="store_true", help="Resume session")
+    group.add_argument("--run", action="store_true", help="Run full pipeline")
+    group.add_argument("--stage", help="Run single stage")
+    group.add_argument("--dossier", action="store_true", help="Get dossier")
+    group.add_argument("--review", action="store_true", help="Review dossier")
+    group.add_argument("--list", action="store_true", help="List sessions")
+    group.add_argument("--history", action="store_true", help="Session history")
+    group.add_argument("--archive", action="store_true", help="Archive session")
+
+    parser.add_argument("--vertical", help="Vertical slug for --create")
+    parser.add_argument("--session-id", help="Session ID")
+    parser.add_argument("--name", help="Session name")
+    parser.add_argument("--description", help="Session description")
+    parser.add_argument("--decision", help="Review decision (approved/rejected)")
+    parser.add_argument("--reviewer", help="Reviewer name")
+    parser.add_argument("--notes", help="Review notes")
+    parser.add_argument("--state", help="Filter by state for --list")
+    parser.add_argument("--focus-areas", help="JSON array of focus areas")
+
+    args = parser.parse_args()
+
+    try:
+        if args.status:
+            result = get_status(db_path=args.db_path)
+        elif args.create:
+            if not args.vertical:
+                raise ValueError("--vertical is required for --create")
+            focus = json.loads(args.focus_areas) if args.focus_areas else None
+            result = create_session(args.vertical, name=args.name,
+                                    description=args.description,
+                                    focus_areas=focus, db_path=args.db_path)
+        elif args.resume:
+            if not args.session_id:
+                raise ValueError("--session-id is required for --resume")
+            result = resume_session(args.session_id, db_path=args.db_path)
+        elif args.run:
+            if not args.session_id:
+                raise ValueError("--session-id is required for --run")
+            result = run_pipeline(args.session_id, db_path=args.db_path)
+        elif args.stage:
+            if not args.session_id:
+                raise ValueError("--session-id is required for --stage")
+            result = run_stage(args.session_id, args.stage, db_path=args.db_path)
+        elif args.dossier:
+            if not args.session_id:
+                raise ValueError("--session-id is required for --dossier")
+            result = get_dossier(args.session_id, db_path=args.db_path)
+        elif args.review:
+            if not args.session_id or not args.decision or not args.reviewer:
+                raise ValueError("--session-id, --decision, --reviewer required for --review")
+            result = review_dossier(args.session_id, args.decision, args.reviewer,
+                                    notes=args.notes, db_path=args.db_path)
+        elif args.list:
+            result = list_sessions(state=args.state, db_path=args.db_path)
+        elif args.history:
+            if not args.session_id:
+                raise ValueError("--session-id is required for --history")
+            result = get_history(args.session_id, db_path=args.db_path)
+        elif args.archive:
+            if not args.session_id:
+                raise ValueError("--session-id is required for --archive")
+            result = archive_session(args.session_id, db_path=args.db_path)
+        else:
+            result = get_status(db_path=args.db_path)
+
+        if args.json:
+            print(json.dumps(result, indent=2, default=str))
+        else:
+            for k, v in result.items():
+                print(f"  {k}: {v}")
+    except Exception as exc:
+        err = {"error": str(exc)}
+        if args.json:
+            print(json.dumps(err, indent=2))
+        else:
+            print(f"Error: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

tools/research/source_scanner.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+# CUI // SP-CTI
+"""Source Scanner — base scanning infrastructure for research signals.
+
+Provides common scanning utilities used by landscape, regulatory, community,
+and academic scanners. Handles signal deduplication via content_hash, batch
+insertion, and web search integration.
+
+Tables used:
+  - research_signals      (write — append-only)
+  - research_sessions     (read/write — signal_count)
+"""
+
+import hashlib
+import json
+import logging
+import sqlite3
+import sys
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+BASE_DIR = Path(__file__).resolve().parent.parent.parent
+DB_PATH = BASE_DIR / "data" / "icdev.db"
+
+try:
+    from tools.compat.db_utils import get_db_connection
+except ImportError:
+    get_db_connection = None
+
+
+def _get_connection(db_path=None):
+    if db_path is None and get_db_connection is not None:
+        try:
+            conn = get_db_connection()
+            conn.row_factory = sqlite3.Row
+            return conn
+        except Exception as exc:
+            logging.debug("db_utils connection failed, falling back to direct: %s", exc)
+    path = Path(db_path) if db_path else DB_PATH
+    if not path.exists():
+        raise FileNotFoundError(f"Database not found: {path}")
+    conn = sqlite3.connect(str(path))
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA foreign_keys = ON")
+    return conn
+
+
+def _uid() -> str:
+    return uuid.uuid4().hex[:12]
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _content_hash(title: str, body: str = "") -> str:
+    return hashlib.sha256(f"{title}|{body[:500]}".encode()).hexdigest()[:24]
+
+
+def _log_audit(conn, session_id, event_type, action, details):
+    try:
+        conn.execute(
+            "INSERT INTO audit_trail "
+            "(project_id, event_type, actor, action, details, classification, created_at) "
+            "VALUES (?, ?, ?, ?, ?, ?, ?)",
+            (session_id, event_type, "icdev-research-engine", action,
+             json.dumps(details) if isinstance(details, dict) else str(details),
+             "CUI", _now()),
+        )
+        conn.commit()
+    except Exception as exc:
+        logging.warning("Audit log insert failed for %s/%s: %s", event_type, action, exc)
+
+
+def get_session_vertical(session_id: str, db_path=None) -> dict:
+    """Get session info and vertical keywords."""
+    conn = _get_connection(db_path)
+    try:
+        session = conn.execute(
+            "SELECT s.*, v.keywords, v.regulatory_bodies, v.academic_categories, v.community_sources "
+            "FROM research_sessions s "
+            "JOIN research_verticals v ON s.vertical_id = v.id "
+            "WHERE s.id = ?", (session_id,)
+        ).fetchone()
+        if not session:
+            raise ValueError(f"Session '{session_id}' not found")
+        result = dict(session)
+        for field in ("keywords", "regulatory_bodies", "academic_categories", "community_sources", "focus_areas"):
+            try:
+                result[field] = json.loads(result[field]) if result[field] else []
+            except (json.JSONDecodeError, TypeError):
+                result[field] = []
+        return result
+    finally:
+        conn.close()
+
+
+def insert_signals(session_id: str, signals: List[Dict], db_path=None) -> dict:
+    """Insert signals with deduplication via content_hash."""
+    conn = _get_connection(db_path)
+    try:
+        existing_hashes = set()
+        rows = conn.execute(
+            "SELECT content_hash FROM research_signals WHERE session_id = ?",
+            (session_id,)
+        ).fetchall()
+        for r in rows:
+            existing_hashes.add(r["content_hash"])
+
+        inserted = 0
+        duplicates = 0
+        now = _now()
+
+        for sig in signals:
+            ch = sig.get("content_hash") or _content_hash(sig["title"], sig.get("body", ""))
+            if ch in existing_hashes:
+                duplicates += 1
+                continue
+            existing_hashes.add(ch)
+            sig_id = f"sig-{_uid()}"
+            conn.execute(
+                "INSERT INTO research_signals "
+                "(id, session_id, source, source_type, title, body, url, author, "
+                "upvotes, citations, sentiment, content_hash, keywords, metadata, "
+                "discovered_at, classification) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                (sig_id, session_id, sig["source"], sig["source_type"],
+                 sig["title"], sig.get("body", ""), sig.get("url", ""),
+                 sig.get("author", ""), sig.get("upvotes", 0), sig.get("citations", 0),
+                 sig.get("sentiment"), ch,
+                 json.dumps(sig.get("keywords", [])),
+                 json.dumps(sig.get("metadata", {})),
+                 now, "CUI"),
+            )
+            inserted += 1
+
+        # Update session signal count
+        total = conn.execute(
+            "SELECT COUNT(*) FROM research_signals WHERE session_id = ?", (session_id,)
+        ).fetchone()[0]
+        conn.execute(
+            "UPDATE research_sessions SET signal_count = ?, updated_at = ? WHERE id = ?",
+            (total, now, session_id),
+        )
+        conn.commit()
+        return {"inserted": inserted, "duplicates": duplicates, "total": total}
+    finally:
+        conn.close()