sayou-connector 0.3.12__py3-none-any.whl

sayou/connector/interfaces/base_generator.py

@@ -0,0 +1,99 @@
+from abc import abstractmethod
+from typing import Iterator
+
+from sayou.core.base_component import BaseComponent
+from sayou.core.decorators import measure_time
+from sayou.core.schemas import SayouPacket, SayouTask
+
+from ..core.exceptions import GeneratorError
+
+
+class BaseGenerator(BaseComponent):
+    """
+    (Tier 1) Abstract base class for all task generators.
+
+    Generators are responsible for discovering resources (e.g., listing files,
+    crawling links) and creating `SayouTask` objects for the Fetcher.
+    """
+
+    component_name = "BaseGenerator"
+    SUPPORTED_TYPES = []
+
+    @classmethod
+    def can_handle(cls, source: str) -> float:
+        """
+        Evaluates whether this generator can handle the given source.
+
+        Analyzes the source string to determine if it matches the pattern or format
+        supported by this generator. Returns a confidence score between 0.0 and 1.0.
+
+        Args:
+            source (str): The input source string to evaluate.
+
+        Returns:
+            float: A confidence score where 1.0 means full confidence,
+                    0.0 means the source is incompatible, and intermediate values
+                    indicate partial matches or heuristics.
+        """
+        return 0.0
+
+    @measure_time
+    def generate(self, source: str, **kwargs) -> Iterator[SayouTask]:
+        """
+        Execute the generation strategy and yield tasks one by one.
+
+        This method handles the lifecycle of the generation process, including
+        logging and error boundary protection.
+
+        Yields:
+            Iterator[SayouTask]: An iterator of tasks to be processed by Fetchers.
+        """
+        self._emit("on_start", input_data={"component": self.component_name})
+        self._log(f"Starting generation strategy: {self.component_name}")
+        count = 0
+        try:
+            for task in self._do_generate(source, **kwargs):
+                count += 1
+                yield task
+            self._emit("on_finish", result_data={"total_tasks": count}, success=True)
+        except Exception as e:
+            wrapped_error = GeneratorError(
+                f"[{self.component_name}] Strategy crashed: {str(e)}"
+            )
+            self.logger.error(wrapped_error, exc_info=True)
+            self._emit("on_error", error=wrapped_error)
+            raise wrapped_error
+        finally:
+            self._log(f"Generator finished. Total tasks yielded: {count}")
+
+    @abstractmethod
+    def _do_generate(self, source: str, **kwargs) -> Iterator[SayouTask]:
+        """
+        [Abstract Hook] Implement the logic to discover resources.
+
+        Args:
+            source (str): The source string to generate tasks from.
+            **kwargs: Additional keyword arguments.
+
+        Yields:
+            SayouTask: A task object representing a unit of work.
+        """
+        raise NotImplementedError
+
+    def feedback(self, packet: SayouPacket):
+        """
+        Receive feedback from the execution result of a task.
+
+        This allows the generator to adjust its strategy dynamically
+        (e.g., adding new links found in a crawled page).
+
+        Args:
+            packet (SayouPacket): The result packet from the Fetcher.
+        """
+        self._do_feedback(packet)
+
+    def _do_feedback(self, packet: SayouPacket):
+        """
+        [Optional Hook] Override this to handle feedback logic.
+        """
+        pass

sayou/connector/pipeline.py

@@ -0,0 +1,304 @@
+import importlib
+import pkgutil
+from typing import Dict, Iterator, List, Optional, Type
+
+from sayou.core.base_component import BaseComponent
+from sayou.core.decorators import safe_run
+from sayou.core.registry import COMPONENT_REGISTRY
+from sayou.core.schemas import SayouPacket, SayouTask
+
+from .interfaces.base_fetcher import BaseFetcher
+from .interfaces.base_generator import BaseGenerator
+
+
+class ConnectorPipeline(BaseComponent):
+    """
+    Orchestrates the data collection process by connecting Generators and Fetchers.
+    """
+
+    component_name = "ConnectorPipeline"
+
+    def __init__(
+        self,
+        extra_generators: Optional[List[Type[BaseGenerator]]] = None,
+        extra_fetchers: Optional[List[Type[BaseFetcher]]] = None,
+        **kwargs,
+    ):
+        """
+        Initializes the pipeline and discovers available components.
+
+        Sets up the internal storage for generators and fetchers, scans specific
+        package paths to automatically discover plugins, and loads them from the
+        global registry into the local mapping.
+
+        Args:
+            extra_generators: List of custom generator classes to register.
+            extra_fetchers: List of custom fetcher classes to register.
+            **kwargs: Configuration arguments passed to the parent component.
+        """
+        super().__init__()
+
+        self.generator_cls_map: Dict[str, Type[BaseGenerator]] = {}
+        self.fetcher_cls_map: Dict[str, Type[BaseFetcher]] = {}
+
+        self._register("sayou.connector.generator")
+        self._register("sayou.connector.fetcher")
+        self._register("sayou.connector.plugins")
+
+        self._load_from_registry()
+
+        if extra_generators:
+            for cls in extra_generators:
+                self._register_manual(cls)
+
+        if extra_fetchers:
+            for cls in extra_fetchers:
+                self._register_manual(cls)
+
+        self.global_config = kwargs
+
+        self.initialize(**kwargs)
+
+    def _register_manual(self, cls):
+        """
+        Safely registers a user-provided class.
+        """
+        if not isinstance(cls, type):
+            raise TypeError(
+                f"Invalid generator: {cls}. "
+                f"Please pass the CLASS itself (e.g., MyGenerator), not an instance (MyGenerator())."
+            )
+
+        name = getattr(cls, "component_name", cls.__name__)
+        self.generators_cls_map[name] = cls
+
+    @classmethod
+    def process(
+        cls,
+        source: str,
+        strategy: str = "auto",
+        **kwargs,
+    ) -> Iterator[SayouPacket]:
+        """
+        [Facade] 1-Line Execution Method.
+        Creates an instance, runs it, and returns the result immediately.
+        """
+        instance = cls(**kwargs)
+        return instance.run(source, strategy, **kwargs)
+
+    def _register(self, package_name: str):
+        """
+        Automatically discovers and registers plugins from the specified package.
+
+        Scans the directory of the given package name and attempts to import all
+        submodules found. Importing these modules triggers the `@register_component`
+        decorator attached to the classes, effectively registering them into the
+        global `COMPONENT_REGISTRY`.
+
+        Args:
+            package_name (str): The dot-separated Python package path to scan
+                                (e.g., "sayou.connector.generator").
+        """
+        try:
+            package = importlib.import_module(package_name)
+            if hasattr(package, "__path__"):
+                for _, name, _ in pkgutil.iter_modules(package.__path__):
+                    full_name = f"{package_name}.{name}"
+                    try:
+                        importlib.import_module(full_name)
+                        self._log(f"Discovered module: {full_name}", level="debug")
+                    except Exception as e:
+                        self._log(
+                            f"Failed to import module {full_name}: {e}", level="warning"
+                        )
+        except ImportError as e:
+            self._log(f"Package not found: {package_name} ({e})", level="warning")
+
+    def _load_from_registry(self):
+        """
+        Populates local component maps from the global registry.
+
+        Iterates through the global `COMPONENT_REGISTRY` to retrieve registered
+        generator and fetcher classes. It stores references to these classes in
+        `self.generator_cls_map` and instantiates fetchers in `self.fetcher_cls_map`.
+        """
+        for name, cls in COMPONENT_REGISTRY["generator"].items():
+            self.generator_cls_map[name] = cls
+            supported = getattr(cls, "SUPPORTED_TYPES", [])
+            for t in supported:
+                self.generator_cls_map[t] = cls
+
+        for name, cls in COMPONENT_REGISTRY["fetcher"].items():
+            instance = cls()
+            for t in getattr(instance, "SUPPORTED_TYPES", []):
+                self.fetcher_cls_map[t] = instance
+
+    @safe_run(default_return=None)
+    def initialize(self, **kwargs):
+        """
+        Perform global initialization for the pipeline.
+
+        This method is protected by safe_run decorators in the actual implementation
+        to prevent initialization crashes.
+
+        Args:
+            **kwargs: Global configuration parameters.
+        """
+        self.global_config.update(kwargs)
+        self._log("ConnectorPipeline initialized.")
+
+    def run(
+        self,
+        source: str,
+        strategy: str = "auto",
+        **kwargs,
+    ) -> Iterator[SayouPacket]:
+        """
+        Execute the collection pipeline.
+
+        This is the main entry point. It selects a Generator based on the strategy,
+        produces Tasks, routes them to the appropriate Fetcher, and yields the results.
+
+        Args:
+            source (str): The root source (e.g., file path, URL, connection string).
+            strategy (str): The name of the generator strategy to use (default: "auto").
+            **kwargs: Additional arguments passed to the Generator's initialize method.
+
+        Yields:
+            Iterator[SayouPacket]: A stream of packets containing fetched data.
+
+        Raises:
+            ValueError: If the specified strategy is not registered.
+        """
+        self._emit("on_start", input_data={"source": source, "strategy": strategy})
+
+        # 1. Generator 선택
+        generator_cls = self._resolve_generator(source, strategy)
+        generator = generator_cls()
+
+        if hasattr(self, "_callbacks"):
+            for cb in self._callbacks:
+                generator.add_callback(cb)
+
+        # 2. Generator 초기화
+        generator.initialize(source=source, **kwargs)
+        self._log(f"Connector started using strategy '{strategy}' on '{source}'")
+
+        # 3. Execution Loop
+        count = 0
+        success_count = 0
+
+        try:
+            for task in generator.generate(source, **kwargs):
+                if not isinstance(task, SayouTask):
+                    self._log(
+                        f"Invalid task type from generator: {type(task)}",
+                        level="warning",
+                    )
+                    continue
+
+                # 4. Fetcher 라우팅
+                fetcher = self.fetcher_cls_map.get(task.source_type)
+                if not fetcher:
+                    self._log(
+                        f"Skipping task {task.uri}: No fetcher for type '{task.source_type}'"
+                    )
+                    continue
+
+                for cb in self._callbacks:
+                    fetcher.add_callback(cb)
+
+                # 5. Fetch 수행
+                packet = fetcher.fetch(task)
+
+                # 6. 결과 처리
+                if packet.success:
+                    success_count += 1
+                    yield packet
+                else:
+                    self._log(f"Fetch failed: {packet.error}")
+
+                # 7. Feedback Loop
+                generator.feedback(packet)
+                count += 1
+
+            self._emit("on_finish", result_data={"count": count}, success=True)
+
+        except Exception as e:
+            self._emit("on_error", error=e)
+            raise e
+
+        self._log(f"Connector finished. Processed: {count}, Success: {success_count}")
+
+    def _resolve_generator(
+        self,
+        source: str,
+        strategy: str,
+    ) -> BaseGenerator:
+        """
+        Determines the appropriate generator strategy to use.
+
+        Prioritizes the strategy explicitly specified by the user. If the strategy
+        is set to 'auto' or None, it attempts to detect the most suitable generator
+        based on the source string using the `can_handle` method of registered generators.
+
+        Args:
+            source (str): The input source string (e.g., file path, URL, connection string).
+            strategy (str): The name of the strategy to use (e.g., 'file', 'sqlite').
+                            If 'auto' or None, automatic detection is performed.
+
+        Returns:
+            BaseGenerator: The initialized generator instance ready for execution.
+
+        Raises:
+            ValueError: If a specific strategy is requested but not found in the registry.
+        """
+        if strategy and strategy != "auto":
+            gen = self.generator_cls_map.get(strategy)
+            if not gen:
+                raise ValueError(f"Unknown strategy: {strategy}")
+            return gen
+
+        best_score = 0.0
+        best_cls = None
+
+        obj_type = getattr(source, "type", type(source).__name__)
+        content_len = 0
+        if hasattr(source, "content"):
+            c = source.content
+            if hasattr(c, "__len__"):
+                content_len = len(c)
+        elif isinstance(source, (str, bytes, list, dict)):
+            content_len = len(source)
+
+        log_lines = [f"Scoring for Item (Type: {obj_type}, Len: {content_len}):"]
+        if hasattr(source, "content") and isinstance(source.content, str):
+            log_lines.append(f"Content Preview: {source.content[:50]}...")
+        elif isinstance(source, str):
+            log_lines.append(f"Content Preview: {source[:50]}...")
+
+        for cls in set(self.generator_cls_map.values()):
+            try:
+                score = cls.can_handle(source)
+
+                mark = ""
+                if score > best_score:
+                    best_score = score
+                    best_cls = cls
+                    mark = "👑"
+
+                log_lines.append(f"   - {cls.__name__}: {score} {mark}")
+
+            except Exception as e:
+                log_lines.append(f"   - {cls.__name__}: Error ({e})")
+
+        self._log("\n".join(log_lines))
+
+        if best_cls and best_score > 0.0:
+            return best_cls
+
+        self._log(
+            "Auto-detection failed. Falling back to default strategy 'file'.",
+            level="warning",
+        )
+        return self.generator_cls_map["file"]

sayou/connector/plugins/gmail_fetcher.py

@@ -0,0 +1,127 @@
+import base64
+from typing import Dict, Any
+
+from sayou.core.registry import register_component
+from sayou.core.schemas import SayouTask
+from ..interfaces.base_fetcher import BaseFetcher
+
+try:
+    from google.oauth2.credentials import Credentials
+    from googleapiclient.discovery import build
+except ImportError:
+    build = None
+
+
+@register_component("fetcher")
+class GmailFetcher(BaseFetcher):
+    """
+    Fetches specific email content using Gmail API.
+    Reconstructs the email into a standardized HTML format suitable for Refinery.
+    """
+
+    component_name = "GmailFetcher"
+    SUPPORTED_TYPES = ["gmail"]
+
+    @classmethod
+    def can_handle(cls, uri: str) -> float:
+        return 1.0 if uri.startswith("gmail-msg://") else 0.0
+
+    def _do_fetch(self, task: SayouTask) -> Dict[str, Any]:
+        token_path = task.params.get("token_path")
+        msg_id = task.params.get("msg_id")
+
+        if not build:
+            raise ImportError("Please install google-api-python-client")
+
+        creds = Credentials.from_authorized_user_file(token_path)
+        service = build("gmail", "v1", credentials=creds)
+
+        # 1. Fetch email details (format='full')
+        message = (
+            service.users()
+            .messages()
+            .get(userId="me", id=msg_id, format="full")
+            .execute()
+        )
+
+        payload = message.get("payload", {})
+        headers = payload.get("headers", [])
+
+        # 2. Parse headers (Subject, From, Date)
+        subject = self._get_header(headers, "Subject", "(No Subject)")
+        sender = self._get_header(headers, "From", "Unknown")
+        date = self._get_header(headers, "Date", "")
+
+        # 3. Extract body (Recursive)
+        body_content = self._extract_body(payload)
+
+        # 4. Reconstruct HTML (User Request Format)
+        html_doc = f"""<!DOCTYPE html>
+<html>
+<head>
+    <title>{subject}</title>
+    <meta name="sender" content="{sender}">
+    <meta name="date" content="{date}">
+    <meta name="msg_id" content="{msg_id}">
+    <meta name="source" content="gmail">
+</head>
+<body>
+{body_content}
+</body>
+</html>"""
+
+        return html_doc.strip()
+
+    def _get_header(self, headers: list, name: str, default: str) -> str:
+        for h in headers:
+            if h["name"].lower() == name.lower():
+                return h["value"]
+        return default
+
+    def _extract_body(self, payload: dict) -> str:
+        body = ""
+
+        # Case A: Single Part
+        if "body" in payload and payload["body"].get("data"):
+            mime_type = payload.get("mimeType", "")
+            data = payload["body"]["data"]
+            decoded_text = self._decode_base64url(data)
+
+            if mime_type == "text/html":
+                return decoded_text
+            elif mime_type == "text/plain":
+                return f"<pre>{decoded_text}</pre>"
+
+        # Case B: Multi Part
+        if "parts" in payload:
+            html_part = None
+            text_part = None
+
+            for part in payload["parts"]:
+                mime_type = part.get("mimeType", "")
+
+                content = self._extract_body(part)
+
+                if mime_type == "text/html":
+                    html_part = content
+                elif mime_type == "text/plain":
+                    text_part = content
+                elif "multipart" in mime_type:
+                    if content:
+                        html_part = content
+
+            if html_part:
+                return html_part
+            if text_part:
+                return text_part
+
+        return body
+
+    def _decode_base64url(self, data: str) -> str:
+        try:
+            padding = len(data) % 4
+            if padding:
+                data += "=" * (4 - padding)
+            return base64.urlsafe_b64decode(data).decode("utf-8", errors="replace")
+        except Exception:
+            return ""

sayou/connector/plugins/gmail_generator.py

@@ -0,0 +1,79 @@
+from typing import Iterator
+from sayou.core.registry import register_component
+from sayou.core.schemas import SayouTask
+from ..interfaces.base_generator import BaseGenerator
+
+try:
+    from google.oauth2.credentials import Credentials
+    from googleapiclient.discovery import build
+except ImportError:
+    build = None
+
+
+@register_component("generator")
+class GmailGenerator(BaseGenerator):
+    """
+    Scans Gmail inbox using Gmail API (OAuth) and generates tasks.
+    """
+
+    component_name = "GmailGenerator"
+    SUPPORTED_TYPES = ["gmail"]
+
+    @classmethod
+    def can_handle(cls, source: str) -> float:
+        return 1.0 if source.startswith("gmail://") else 0.0
+
+    def _do_generate(self, source: str, **kwargs) -> Iterator[SayouTask]:
+        """
+        Connects to Gmail API -> Search (List) -> Yield Tasks.
+        source example: gmail://me (default) or gmail://me?q=is:unread
+        """
+        if not build:
+            raise ImportError(
+                "Please install google-api-python-client google-auth-oauthlib"
+            )
+
+        token_path = kwargs.get("token_path")
+        if not token_path:
+            raise ValueError("GmailGenerator requires 'token_path' in kwargs.")
+
+        # 1. Parsing Parameters
+        query = kwargs.get("query", "is:inbox")
+        max_results = int(kwargs.get("limit", 10))
+
+        # 2. Connect to Gmail API
+        creds = Credentials.from_authorized_user_file(token_path)
+        service = build("gmail", "v1", credentials=creds)
+
+        try:
+            # 3. Fetch email list
+            results = (
+                service.users()
+                .messages()
+                .list(userId="me", q=query, maxResults=max_results)
+                .execute()
+            )
+
+            messages = results.get("messages", [])
+
+            self._log(f"📧 Found {len(messages)} emails. Generating tasks...")
+
+            # 4. Generate tasks
+            for msg in messages:
+                msg_id = msg["id"]
+                thread_id = msg["threadId"]
+                task_uri = f"gmail-msg://{msg_id}"
+
+                yield SayouTask(
+                    uri=task_uri,
+                    source_type="gmail",
+                    params={
+                        "token_path": token_path,
+                        "msg_id": msg_id,
+                        "thread_id": thread_id,
+                    },
+                )
+
+        except Exception as e:
+            self._log(f"Gmail API List failed: {e}", level="error")
+            raise e

sayou/connector/plugins/google_calendar_fetcher.py

@@ -0,0 +1,89 @@
+import datetime
+import os
+from typing import Any, Dict
+
+from sayou.core.registry import register_component
+from sayou.core.schemas import SayouTask
+
+from ..interfaces.base_fetcher import BaseFetcher
+
+try:
+    from google.oauth2.credentials import Credentials
+    from googleapiclient.discovery import build
+except ImportError:
+    build = None
+
+
+@register_component("fetcher")
+class GoogleCalendarFetcher(BaseFetcher):
+    """
+    Fetches events using Google API with User OAuth Token.
+    Works for Workspace (Corporate) & Personal accounts.
+    """
+
+    component_name = "GoogleCalendarFetcher"
+    SUPPORTED_TYPES = ["google_calendar"]
+
+    @classmethod
+    def can_handle(cls, uri: str) -> float:
+        return 1.0 if uri.startswith("gcal://") else 0.0
+
+    def _do_fetch(self, task: SayouTask) -> Dict[str, Any]:
+        if not build:
+            raise ImportError("Google API libraries required.")
+
+        token_path = task.params.get("token_path")
+        if not token_path or not os.path.exists(token_path):
+            raise ValueError("Token path invalid.")
+
+        creds = Credentials.from_authorized_user_file(token_path)
+
+        service = build("calendar", "v3", credentials=creds)
+
+        now = datetime.datetime.utcnow()
+        time_min = (now - datetime.timedelta(days=30)).isoformat() + "Z"
+        time_max = (now + datetime.timedelta(days=30)).isoformat() + "Z"
+
+        events_result = (
+            service.events()
+            .list(
+                calendarId="primary",
+                timeMin=time_min,
+                timeMax=time_max,
+                singleEvents=True,
+                orderBy="startTime",
+            )
+            .execute()
+        )
+
+        events = events_result.get("items", [])
+
+        parsed_events = []
+        for event in events:
+            start = event["start"].get("dateTime", event["start"].get("date"))
+            end = event["end"].get("dateTime", event["end"].get("date"))
+
+            parsed_events.append(
+                {
+                    "id": event.get("id"),
+                    "summary": event.get("summary", "No Title"),
+                    "description": event.get("description", ""),
+                    "start": start,
+                    "end": end,
+                    "location": event.get("location", ""),
+                    "htmlLink": event.get("htmlLink", ""),
+                    "attendees": [
+                        {"email": a.get("email"), "status": a.get("responseStatus")}
+                        for a in event.get("attendees", [])
+                    ],
+                }
+            )
+
+        return {
+            "content": parsed_events,
+            "meta": {
+                "source": "google_calendar",
+                "account": "authenticated_user",
+                "count": len(parsed_events),
+            },
+        }