contextsynapse 1.0.0__py3-none-any.whl

contextsynapse/connectors/financial/youtube.py

@@ -0,0 +1,272 @@
+"""YouTube Transcript Connector — extracts transcripts from YouTube videos.
+
+Uses youtube-transcript-api (free, no API key needed).
+Most financial videos have auto-generated captions.
+
+Usage:
+    connector = YouTubeConnector(config)
+    docs = connector.poll()  # fetches transcript for configured video IDs
+
+Or directly:
+    transcript = YouTubeConnector.extract_transcript("https://youtube.com/watch?v=xxxx")
+"""
+from __future__ import annotations
+
+import logging
+import re
+from typing import Dict, List, Optional
+
+from ..base import BaseConnector, ConnectorConfig, ConnectorDocument
+
+logger = logging.getLogger(__name__)
+
+try:
+    from youtube_transcript_api import YouTubeTranscriptApi
+except ImportError:
+    YouTubeTranscriptApi = None
+
+
+def extract_video_id(url: str) -> Optional[str]:
+    """Extract video ID from various YouTube URL formats."""
+    patterns = [
+        r'(?:v=|/v/|youtu\.be/)([a-zA-Z0-9_-]{11})',
+        r'(?:embed/)([a-zA-Z0-9_-]{11})',
+    ]
+    for pattern in patterns:
+        match = re.search(pattern, url)
+        if match:
+            return match.group(1)
+    # Maybe it's already a video ID
+    if re.match(r'^[a-zA-Z0-9_-]{11}$', url):
+        return url
+    return None
+
+
+class YouTubeConnector(BaseConnector):
+    """Extracts transcripts from YouTube videos."""
+
+    def __init__(self, config: ConnectorConfig):
+        super().__init__(config)
+        self._urls: List[str] = config.config.get("urls", [])
+        self._languages: List[str] = config.config.get("languages", ["en", "hi"])
+
+    def poll(self) -> List[ConnectorDocument]:
+        """Fetch transcripts for configured YouTube URLs."""
+        docs = []
+        for url in self._urls:
+            transcript = self.extract_transcript(url)
+            if transcript:
+                docs.append(transcript)
+        return self._dedup(docs)
+
+    def extract_transcript(self, url: str) -> Optional[ConnectorDocument]:
+        """Extract transcript from a single YouTube URL."""
+        video_id = extract_video_id(url)
+        if not video_id:
+            logger.warning("[YOUTUBE] Could not extract video ID from: %s", url)
+            return None
+
+        text = self._get_transcript_text(video_id)
+        if not text:
+            return None
+
+        return ConnectorDocument(
+            title=f"YouTube Transcript: {video_id}",
+            content=text,
+            url=f"https://www.youtube.com/watch?v={video_id}",
+            source="youtube",
+            tags=["youtube", "transcript", "video"],
+            doc_type="transcript",
+            metadata={
+                "video_id": video_id,
+                "original_url": url,
+                "transcript_length": len(text),
+            },
+        )
+
+    def _get_transcript_text(self, video_id: str) -> str:
+        """Get transcript text for a video ID."""
+        if YouTubeTranscriptApi is None:
+            logger.warning("[YOUTUBE] youtube-transcript-api not installed. Install: pip install youtube-transcript-api")
+            return ""
+
+        try:
+            transcript_list = YouTubeTranscriptApi.get_transcript(
+                video_id, languages=self._languages
+            )
+            # Join all segments into one text
+            return " ".join(entry["text"] for entry in transcript_list)
+        except Exception as e:
+            logger.warning("[YOUTUBE] Failed to get transcript for %s: %s", video_id, e)
+            return ""
+
+    def health_check(self) -> bool:
+        if YouTubeTranscriptApi is None:
+            return False
+        return True
+
+
+def transcribe_youtube(url: str, languages: List[str] = None) -> Optional[str]:
+    """Convenience function — extract transcript from a YouTube URL.
+
+    Returns transcript text or None.
+    """
+    if languages is None:
+        languages = ["en", "hi"]
+
+    video_id = extract_video_id(url)
+    if not video_id:
+        return None
+
+    if YouTubeTranscriptApi is None:
+        logger.warning("youtube-transcript-api not installed")
+        return None
+
+    # Try manual captions first, then auto-generated
+    try:
+        transcript_list = YouTubeTranscriptApi.get_transcript(
+            video_id, languages=languages
+        )
+        return " ".join(entry["text"] for entry in transcript_list)
+    except Exception:
+        pass
+
+    # Try auto-generated captions (a]XX format)
+    try:
+        transcript_list = YouTubeTranscriptApi.list_transcripts(video_id)
+        # Try generated (auto) captions in any language
+        for transcript in transcript_list:
+            if transcript.is_generated:
+                entries = transcript.fetch()
+                text = " ".join(entry["text"] for entry in entries)
+                if text.strip():
+                    return text
+        # Try any available transcript
+        for transcript in transcript_list:
+            entries = transcript.fetch()
+            text = " ".join(entry["text"] for entry in entries)
+            if text.strip():
+                return text
+    except Exception as e:
+        logger.warning("YouTube captions unavailable for %s: %s", video_id, e)
+
+    # Fallback: download audio and transcribe with Whisper
+    logger.info("Attempting Whisper audio transcription for %s", video_id)
+    whisper_text = _whisper_fallback(url, video_id)
+    if whisper_text:
+        return whisper_text
+
+    return None
+
+
+def _whisper_fallback(url: str, video_id: str) -> Optional[str]:
+    """Download YouTube audio and transcribe with OpenAI Whisper.
+
+    Requires: pip install yt-dlp openai-whisper (or faster-whisper)
+    Falls back gracefully if not installed.
+    """
+    import tempfile
+    import os
+
+    # Step 1: Download audio with yt-dlp
+    audio_path = None
+    try:
+        import yt_dlp
+        tmp_dir = tempfile.mkdtemp(prefix="yt_whisper_")
+        audio_path = os.path.join(tmp_dir, f"{video_id}.mp3")
+
+        ydl_opts = {
+            "format": "bestaudio/best",
+            "outtmpl": os.path.join(tmp_dir, f"{video_id}.%(ext)s"),
+            "postprocessors": [{
+                "key": "FFmpegExtractAudio",
+                "preferredcodec": "mp3",
+                "preferredquality": "64",  # low quality = smaller file = faster
+            }],
+            "quiet": True,
+            "no_warnings": True,
+            "socket_timeout": 30,
+        }
+
+        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+            ydl.download([url])
+
+        # Find the downloaded file
+        if not os.path.exists(audio_path):
+            for f in os.listdir(tmp_dir):
+                if f.endswith((".mp3", ".m4a", ".wav", ".opus", ".webm")):
+                    audio_path = os.path.join(tmp_dir, f)
+                    break
+
+        if not audio_path or not os.path.exists(audio_path):
+            logger.warning("yt-dlp download produced no audio file for %s", video_id)
+            return None
+
+        logger.info("Audio downloaded: %s (%.1f MB)", audio_path,
+                     os.path.getsize(audio_path) / 1e6)
+
+    except ImportError:
+        logger.info("yt-dlp not installed — Whisper fallback unavailable. Install: pip install yt-dlp")
+        return None
+    except Exception as e:
+        logger.warning("yt-dlp download failed for %s: %s", video_id, e)
+        return None
+
+    # Step 2: Transcribe with Whisper
+    transcript = None
+
+    # Try faster-whisper first (much faster, lower memory)
+    try:
+        from faster_whisper import WhisperModel
+        model = WhisperModel("base", device="cpu", compute_type="int8")
+        segments, info = model.transcribe(audio_path, language="en")
+        transcript = " ".join(seg.text.strip() for seg in segments)
+        logger.info("faster-whisper transcribed %s: %d chars, lang=%s",
+                     video_id, len(transcript), info.language)
+    except ImportError:
+        pass
+    except Exception as e:
+        logger.warning("faster-whisper failed for %s: %s", video_id, e)
+
+    # Try openai-whisper (slower but more common)
+    if not transcript:
+        try:
+            import whisper
+            model = whisper.load_model("base")
+            result = model.transcribe(audio_path, language="en")
+            transcript = result.get("text", "")
+            logger.info("whisper transcribed %s: %d chars", video_id, len(transcript))
+        except ImportError:
+            pass
+        except Exception as e:
+            logger.warning("whisper failed for %s: %s", video_id, e)
+
+    # Try OpenAI API Whisper (cloud, requires API key)
+    if not transcript:
+        try:
+            import openai
+            client = openai.OpenAI()
+            with open(audio_path, "rb") as f:
+                result = client.audio.transcriptions.create(
+                    model="whisper-1",
+                    file=f,
+                    language="en",
+                )
+            transcript = result.text
+            logger.info("OpenAI Whisper API transcribed %s: %d chars", video_id, len(transcript))
+        except ImportError:
+            pass
+        except Exception as e:
+            logger.warning("OpenAI Whisper API failed for %s: %s", video_id, e)
+
+    if not transcript:
+        logger.warning("All Whisper methods failed for %s. Install: pip install faster-whisper yt-dlp", video_id)
+
+    # Cleanup
+    try:
+        import shutil
+        shutil.rmtree(tmp_dir, ignore_errors=True)
+    except Exception:
+        pass
+
+    return transcript if transcript and transcript.strip() else None

contextsynapse/connectors/github.py

@@ -0,0 +1,164 @@
+"""
+GitHub Connector
+=================
+Push code, create PRs, sync issues from the project graph to GitHub.
+
+Usage:
+    gh = GitHubConnector(token="ghp_...", repo="user/my-app")
+    gh.push_file("auth.py", content, message="Implement JWT auth")
+    pr_url = gh.create_pr(title="Add auth", body="Implements JWT auth", branch="feat/auth")
+    gh.create_issue(title="Bug: token expiry", body="...", labels=["bug"])
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import logging
+from typing import Any, Dict, List, Optional
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+class GitHubConnector:
+    """GitHub API connector for code push, PRs, and issues."""
+
+    def __init__(self, token: str, repo: str, branch: str = "main"):
+        """
+        Args:
+            token: GitHub personal access token (ghp_...)
+            repo: Repository in "owner/repo" format
+            branch: Default branch
+        """
+        self.token = token
+        self.repo = repo
+        self.branch = branch
+        self._base = f"https://api.github.com/repos/{repo}"
+        self._headers = {
+            "Authorization": f"Bearer {token}",
+            "Accept": "application/vnd.github+json",
+            "X-GitHub-Api-Version": "2022-11-28",
+        }
+
+    def push_file(
+        self,
+        path: str,
+        content: str,
+        message: str = "Update file",
+        branch: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Push or update a file in the repo.
+
+        Returns commit info or error.
+        """
+        branch = branch or self.branch
+        url = f"{self._base}/contents/{path}"
+
+        # Check if file exists (need SHA for update)
+        sha = None
+        resp = requests.get(url, headers=self._headers, params={"ref": branch})
+        if resp.status_code == 200:
+            sha = resp.json().get("sha")
+
+        payload = {
+            "message": message,
+            "content": base64.b64encode(content.encode()).decode(),
+            "branch": branch,
+        }
+        if sha:
+            payload["sha"] = sha
+
+        resp = requests.put(url, headers=self._headers, json=payload)
+        if resp.status_code in (200, 201):
+            data = resp.json()
+            return {
+                "status": "success",
+                "sha": data.get("commit", {}).get("sha", ""),
+                "url": data.get("content", {}).get("html_url", ""),
+            }
+        return {"status": "error", "code": resp.status_code, "message": resp.text[:200]}
+
+    def create_branch(self, branch_name: str, from_branch: Optional[str] = None) -> Dict[str, Any]:
+        """Create a new branch."""
+        from_branch = from_branch or self.branch
+
+        # Get SHA of source branch
+        resp = requests.get(
+            f"{self._base}/git/refs/heads/{from_branch}",
+            headers=self._headers,
+        )
+        if resp.status_code != 200:
+            return {"status": "error", "message": f"Branch {from_branch} not found"}
+
+        sha = resp.json()["object"]["sha"]
+
+        resp = requests.post(
+            f"{self._base}/git/refs",
+            headers=self._headers,
+            json={"ref": f"refs/heads/{branch_name}", "sha": sha},
+        )
+        if resp.status_code == 201:
+            return {"status": "success", "branch": branch_name, "sha": sha}
+        return {"status": "error", "code": resp.status_code, "message": resp.text[:200]}
+
+    def create_pr(
+        self,
+        title: str,
+        body: str = "",
+        head: str = "",
+        base: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Create a pull request."""
+        resp = requests.post(
+            f"{self._base}/pulls",
+            headers=self._headers,
+            json={
+                "title": title,
+                "body": body,
+                "head": head,
+                "base": base or self.branch,
+            },
+        )
+        if resp.status_code == 201:
+            data = resp.json()
+            return {
+                "status": "success",
+                "pr_number": data["number"],
+                "url": data["html_url"],
+            }
+        return {"status": "error", "code": resp.status_code, "message": resp.text[:200]}
+
+    def create_issue(
+        self,
+        title: str,
+        body: str = "",
+        labels: Optional[List[str]] = None,
+        assignees: Optional[List[str]] = None,
+    ) -> Dict[str, Any]:
+        """Create a GitHub issue."""
+        payload: Dict[str, Any] = {"title": title, "body": body}
+        if labels:
+            payload["labels"] = labels
+        if assignees:
+            payload["assignees"] = assignees
+
+        resp = requests.post(
+            f"{self._base}/issues",
+            headers=self._headers,
+            json=payload,
+        )
+        if resp.status_code == 201:
+            data = resp.json()
+            return {
+                "status": "success",
+                "issue_number": data["number"],
+                "url": data["html_url"],
+            }
+        return {"status": "error", "code": resp.status_code, "message": resp.text[:200]}
+
+    def test_connection(self) -> bool:
+        """Test if the token and repo are valid."""
+        resp = requests.get(self._base, headers=self._headers)
+        return resp.status_code == 200

contextsynapse/connectors/jira.py

@@ -0,0 +1,120 @@
+"""
+Jira Connector
+================
+Create issues, sync status, and track work from the project graph to Jira.
+
+Usage:
+    jira = JiraConnector(url="https://your-org.atlassian.net", email="you@co.com", token="...")
+    jira.create_issue(project="MYAPP", title="Implement JWT auth", description="...", issue_type="Task")
+    jira.update_status(issue_key="MYAPP-42", status="Done")
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any, Dict, List, Optional
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+class JiraConnector:
+    """Jira REST API connector for issue tracking."""
+
+    def __init__(self, url: str, email: str, token: str):
+        """
+        Args:
+            url: Jira instance URL (e.g. https://your-org.atlassian.net)
+            email: Jira account email
+            token: Jira API token
+        """
+        self.url = url.rstrip("/")
+        self._auth = (email, token)
+        self._headers = {
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+        }
+
+    def create_issue(
+        self,
+        project: str,
+        title: str,
+        description: str = "",
+        issue_type: str = "Task",
+        priority: Optional[str] = None,
+        labels: Optional[List[str]] = None,
+        assignee: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Create a Jira issue."""
+        fields: Dict[str, Any] = {
+            "project": {"key": project},
+            "summary": title,
+            "description": description,
+            "issuetype": {"name": issue_type},
+        }
+        if priority:
+            fields["priority"] = {"name": priority.capitalize()}
+        if labels:
+            fields["labels"] = labels
+        if assignee:
+            fields["assignee"] = {"accountId": assignee}
+
+        resp = requests.post(
+            f"{self.url}/rest/api/3/issue",
+            auth=self._auth,
+            headers=self._headers,
+            json={"fields": fields},
+        )
+        if resp.status_code == 201:
+            data = resp.json()
+            return {
+                "status": "success",
+                "key": data["key"],
+                "id": data["id"],
+                "url": f"{self.url}/browse/{data['key']}",
+            }
+        return {"status": "error", "code": resp.status_code, "message": resp.text[:200]}
+
+    def get_issue(self, issue_key: str) -> Dict[str, Any]:
+        """Get issue details."""
+        resp = requests.get(
+            f"{self.url}/rest/api/3/issue/{issue_key}",
+            auth=self._auth,
+            headers=self._headers,
+        )
+        if resp.status_code == 200:
+            data = resp.json()
+            fields = data.get("fields", {})
+            return {
+                "key": data["key"],
+                "summary": fields.get("summary", ""),
+                "status": fields.get("status", {}).get("name", ""),
+                "assignee": (fields.get("assignee") or {}).get("displayName", "Unassigned"),
+                "priority": (fields.get("priority") or {}).get("name", ""),
+            }
+        return {"status": "error", "code": resp.status_code}
+
+    def add_comment(self, issue_key: str, comment: str) -> Dict[str, Any]:
+        """Add a comment to an issue."""
+        resp = requests.post(
+            f"{self.url}/rest/api/3/issue/{issue_key}/comment",
+            auth=self._auth,
+            headers=self._headers,
+            json={"body": {"type": "doc", "version": 1, "content": [
+                {"type": "paragraph", "content": [{"type": "text", "text": comment}]}
+            ]}},
+        )
+        if resp.status_code == 201:
+            return {"status": "success", "comment_id": resp.json().get("id")}
+        return {"status": "error", "code": resp.status_code}
+
+    def test_connection(self) -> bool:
+        """Test if credentials are valid."""
+        resp = requests.get(
+            f"{self.url}/rest/api/3/myself",
+            auth=self._auth,
+            headers=self._headers,
+        )
+        return resp.status_code == 200

contextsynapse/connectors/kafka_connector.py

@@ -0,0 +1,124 @@
+"""Kafka Connector — subscribes to topics and streams ConnectorDocuments."""
+
+from __future__ import annotations
+import json
+import logging
+import threading
+from typing import Callable, List
+from .base import BaseConnector, ConnectorConfig, ConnectorDocument
+
+logger = logging.getLogger(__name__)
+
+
+class KafkaConnector(BaseConnector):
+    """Subscribes to Kafka topics for streaming document ingestion.
+
+    Supports both poll mode (batch consume) and subscribe mode (continuous).
+    """
+
+    def __init__(self, config: ConnectorConfig):
+        config.connector_type = "kafka"
+        super().__init__(config)
+        self._broker = config.config.get("broker", "localhost:9092")
+        self._topic = config.config.get("topic", "")
+        self._group_id = config.config.get("group_id", f"contextcore-{config.connector_id}")
+        self._consumer = None
+        self._running = False
+        self._callback = None
+
+    def _get_consumer(self):
+        if self._consumer is None:
+            try:
+                from kafka import KafkaConsumer
+                self._consumer = KafkaConsumer(
+                    self._topic,
+                    bootstrap_servers=self._broker,
+                    group_id=self._group_id,
+                    auto_offset_reset="latest",
+                    value_deserializer=lambda m: json.loads(m.decode("utf-8")),
+                    consumer_timeout_ms=5000,  # 5s timeout for poll mode
+                )
+            except ImportError:
+                logger.warning("kafka-python not installed: pip install kafka-python")
+                return None
+            except Exception as e:
+                logger.error("[KAFKA] Connection failed: %s", e)
+                return None
+        return self._consumer
+
+    def poll(self) -> List[ConnectorDocument]:
+        """Batch consume — reads available messages (up to 5s timeout)."""
+        consumer = self._get_consumer()
+        if not consumer:
+            return []
+
+        docs = []
+        try:
+            for msg in consumer:
+                payload = msg.value if isinstance(msg.value, dict) else {}
+                docs.append(ConnectorDocument(
+                    title=payload.get("title", ""),
+                    content=payload.get("content", payload.get("text", json.dumps(payload))),
+                    url=payload.get("url", ""),
+                    source=f"kafka:{self._topic}",
+                    published_at=payload.get("timestamp", ""),
+                    tags=payload.get("tags", []),
+                    metadata={"kafka_topic": self._topic, "kafka_offset": msg.offset},
+                    doc_type=payload.get("doc_type", "event"),
+                ))
+                if len(docs) >= 50:  # max 50 per poll
+                    break
+        except Exception as e:
+            logger.error("[KAFKA] Poll error: %s", e)
+
+        return docs
+
+    def subscribe(self, callback: Callable[[ConnectorDocument], None]):
+        """Continuous streaming — calls callback for each message."""
+        self._callback = callback
+        self._running = True
+        thread = threading.Thread(target=self._stream_loop, daemon=True, name=f"kafka-{self._topic}")
+        thread.start()
+
+    def _stream_loop(self):
+        consumer = self._get_consumer()
+        if not consumer:
+            return
+        try:
+            # Reset timeout for streaming mode
+            consumer.config['consumer_timeout_ms'] = float("inf")
+            for msg in consumer:
+                if not self._running:
+                    break
+                payload = msg.value if isinstance(msg.value, dict) else {}
+                doc = ConnectorDocument(
+                    title=payload.get("title", ""),
+                    content=payload.get("content", json.dumps(payload)),
+                    url=payload.get("url", ""),
+                    source=f"kafka:{self._topic}",
+                    doc_type="event",
+                )
+                if self._callback:
+                    self._callback(doc)
+        except Exception as e:
+            logger.error("[KAFKA] Stream error: %s", e)
+        finally:
+            self._running = False
+
+    def health_check(self) -> bool:
+        try:
+            from kafka import KafkaConsumer
+            test = KafkaConsumer(bootstrap_servers=self._broker, consumer_timeout_ms=2000)
+            test.close()
+            return True
+        except Exception:
+            return False
+
+    def stop(self):
+        self._running = False
+        if self._consumer:
+            try:
+                self._consumer.close()
+            except Exception:
+                pass
+            self._consumer = None