sayou-connector 0.3.12__py3-none-any.whl

sayou/connector/plugins/google_calendar_generator.py

@@ -0,0 +1,46 @@
+import os
+from typing import Iterator
+
+from sayou.core.registry import register_component
+from sayou.core.schemas import SayouTask
+
+from ..interfaces.base_generator import BaseGenerator
+
+
+@register_component("generator")
+class GoogleCalendarGenerator(BaseGenerator):
+    """
+    Generates tasks using OAuth 2.0 Token.
+    Requires 'sayou_google_token.json' (generated by auth script).
+    """
+
+    component_name = "GoogleCalendarGenerator"
+    SUPPORTED_TYPES = ["google_calendar"]
+
+    @classmethod
+    def can_handle(cls, uri: str) -> float:
+        return 1.0 if uri.startswith("gcal://") else 0.0
+
+    def _do_generate(self, source: str, **kwargs) -> Iterator[SayouTask]:
+        token_path = kwargs.get("google_token_path")
+
+        if not os.path.exists(token_path):
+            raise FileNotFoundError(
+                f"Google Token not found at {token_path}. Run authentication script first."
+            )
+
+        calendar_id = source.replace("gcal://", "") or "primary"
+
+        yield SayouTask(
+            uri=source,
+            source_type="google_calendar",
+            params={
+                "calendar_id": calendar_id,
+                "token_path": token_path,
+            },
+            meta={
+                "source": "google_calendar",
+                "calendar_id": calendar_id,
+                "filename": f"calendar_{calendar_id.replace('@', '_')}",
+            },
+        )

sayou/connector/plugins/google_drive_fetcher.py

@@ -0,0 +1,151 @@
+import io
+import os
+from typing import Any, Dict
+
+from sayou.core.registry import register_component
+from sayou.core.schemas import SayouTask
+
+from ..interfaces.base_fetcher import BaseFetcher
+
+try:
+    import chardet
+    from google.oauth2.credentials import Credentials
+    from googleapiclient.discovery import build
+    from googleapiclient.errors import HttpError
+    from googleapiclient.http import MediaIoBaseDownload
+except ImportError:
+    build = None
+    chardet = None
+
+
+@register_component("fetcher")
+class GoogleDriveFetcher(BaseFetcher):
+    """
+    Fetches content from Google Drive files.
+    - Google Native Formats -> Converted to MS Office formats (.docx, .xlsx, .pptx)
+    - Standard Files (PDF, JPG, ZIP...) -> Downloaded as original binary.
+    """
+
+    component_name = "GoogleDriveFetcher"
+    SUPPORTED_TYPES = ["drive"]
+
+    @classmethod
+    def can_handle(cls, uri: str) -> float:
+        return 1.0 if uri.startswith("gdrive://file/") else 0.0
+
+    def _do_fetch(self, task: SayouTask) -> Dict[str, Any]:
+        token_path = task.params.get("token_path")
+        file_id = task.params.get("file_id")
+        mime_type = task.params.get("mime_type")
+        original_name = task.meta.get("filename", "unknown_file")
+
+        creds = Credentials.from_authorized_user_file(token_path)
+        service = build("drive", "v3", credentials=creds)
+
+        request = None
+        extension = ""
+        is_google_doc = False
+
+        # 1. Google Native Formats
+        if mime_type == "application/vnd.google-apps.document":
+            request = service.files().export_media(
+                fileId=file_id,
+                mimeType="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+            )
+            extension = ".docx"
+            is_google_doc = True
+        elif mime_type == "application/vnd.google-apps.spreadsheet":
+            request = service.files().export_media(
+                fileId=file_id,
+                mimeType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+            )
+            extension = ".xlsx"
+            is_google_doc = True
+        elif mime_type == "application/vnd.google-apps.presentation":
+            request = service.files().export_media(
+                fileId=file_id,
+                mimeType="application/vnd.openxmlformats-officedocument.presentationml.presentation",
+            )
+            extension = ".pptx"
+            is_google_doc = True
+        else:
+            request = service.files().get_media(fileId=file_id)
+            _, ext = os.path.splitext(original_name)
+            extension = ext if ext else ""
+
+        # 2. Execute Download
+        try:
+            fh = io.BytesIO()
+            downloader = MediaIoBaseDownload(fh, request)
+            done = False
+            while done is False:
+                status, done = downloader.next_chunk()
+
+            raw_bytes = fh.getvalue()
+
+            final_content = raw_bytes
+            is_text_candidate = False
+
+            if mime_type.startswith("text/") or mime_type == "application/json":
+                is_text_candidate = True
+            elif extension.lower() in [
+                ".csv",
+                ".txt",
+                ".json",
+                ".md",
+                ".py",
+                ".html",
+                ".xml",
+            ]:
+                is_text_candidate = True
+
+            if not is_google_doc and is_text_candidate:
+                detected = chardet.detect(raw_bytes)
+                encoding = detected.get("encoding")
+                confidence = detected.get("confidence", 0)
+
+                # 2) EUC-KR -> UTF-8
+                if (
+                    encoding
+                    and encoding.lower() not in ["utf-8", "ascii"]
+                    and confidence > 0.6
+                ):
+                    try:
+                        # Decode (Bytes -> Str)
+                        text_content = raw_bytes.decode(encoding)
+                        # Encode back to Bytes (Str -> UTF-8 Bytes)
+                        final_content = text_content.encode("utf-8")
+                        self._log(
+                            f"Transcoded {original_name} from {encoding} to utf-8 bytes."
+                        )
+                    except Exception as e:
+                        self._log(
+                            f"Encoding conversion failed: {e}. Keeping raw bytes.",
+                            level="warning",
+                        )
+                        final_content = raw_bytes
+
+            # 3. Return
+            return {
+                "content": final_content,
+                "meta": {
+                    "source": "google_drive",
+                    "file_id": file_id,
+                    "mime_type": mime_type,
+                    "original_filename": original_name,
+                    "suggested_filename": (
+                        f"{original_name}{extension}"
+                        if not original_name.endswith(extension)
+                        else original_name
+                    ),
+                    "extension": extension,
+                    "is_binary": isinstance(final_content, bytes),
+                },
+            }
+
+        except HttpError as e:
+            self._log(f"Drive Download Failed ({file_id}): {e}", level="error")
+            return {
+                "content": b"",
+                "meta": {"source": "google_drive", "error": str(e), "file_id": file_id},
+            }

sayou/connector/plugins/google_drive_generator.py

@@ -0,0 +1,107 @@
+import os
+from typing import Iterator
+
+from sayou.core.registry import register_component
+from sayou.core.schemas import SayouTask
+
+from ..interfaces.base_generator import BaseGenerator
+
+try:
+    from google.oauth2.credentials import Credentials
+    from googleapiclient.discovery import build
+except ImportError:
+    build = None
+
+
+@register_component("generator")
+class GoogleDriveGenerator(BaseGenerator):
+    """
+    Generates tasks for files in Google Drive.
+    URI Schema:
+      - gdrive://root       (My Drive Root)
+      - gdrive://{folderID} (Specific Folder)
+    """
+
+    component_name = "GoogleDriveGenerator"
+    SUPPORTED_TYPES = ["drive"]
+
+    @classmethod
+    def can_handle(cls, uri: str) -> float:
+        return 1.0 if uri.startswith("gdrive://") else 0.0
+
+    def _do_generate(self, source: str, **kwargs) -> Iterator[SayouTask]:
+        # 1. Certification
+        token_path = kwargs.get("google_token_path")
+
+        if not os.path.exists(token_path):
+            raise FileNotFoundError(
+                f"Google Token not found at {token_path}. Run authentication script first."
+            )
+
+        creds = Credentials.from_authorized_user_file(token_path)
+        service = build("drive", "v3", credentials=creds)
+
+        # 2. Search Query
+        root_id = source.replace("gdrive://", "") or "root"
+
+        query_override = None
+        if "?" in root_id:
+            root_id, query_part = root_id.split("?", 1)
+
+        # 3. File Search (Recursive or Flat Search)
+        query = f"'{root_id}' in parents and trashed = false"
+        if root_id == "root":
+            pass
+
+        results = (
+            service.files()
+            .list(
+                q=query,
+                pageSize=100,
+                fields="nextPageToken, files(id, name, mimeType, webViewLink, createdTime, modifiedTime)",
+            )
+            .execute()
+        )
+
+        files = results.get("files", [])
+
+        for file in files:
+            mime_type = file.get("mimeType")
+            file_id = file["id"]
+            file_name = file["name"]
+
+            if mime_type == "application/vnd.google-apps.document":
+                target_uri = f"gdocs://document/{file_id}"
+                source = "docs"
+
+            elif mime_type == "application/vnd.google-apps.spreadsheet":
+                target_uri = f"gsheets://spreadsheet/{file_id}"
+                source = "sheets"
+
+            elif mime_type == "application/vnd.google-apps.presentation":
+                target_uri = f"gslides://presentation/{file_id}"
+                source = "slides"
+
+            elif mime_type == "application/vnd.google-apps.folder":
+                continue
+
+            else:
+                target_uri = f"gdrive://file/{file_id}"
+                source = "drive"
+
+            yield SayouTask(
+                uri=target_uri,
+                source_type=source,
+                params={
+                    "file_id": file_id,
+                    "mime_type": mime_type,
+                    "token_path": token_path,
+                },
+                meta={
+                    "source": source,
+                    "filename": file_name,
+                    "file_id": file_id,
+                    "mime_type": mime_type,
+                    "link": file.get("webViewLink"),
+                },
+            )

sayou/connector/plugins/imap_email_fetcher.py

@@ -0,0 +1,140 @@
+import email
+import imaplib
+from email.header import decode_header
+from typing import Any, Dict
+
+from sayou.core.registry import register_component
+from sayou.core.schemas import SayouTask
+
+from ..interfaces.base_fetcher import BaseFetcher
+
+try:
+    import html2text
+except ImportError:
+    html2text = None
+
+
+@register_component("fetcher")
+class ImapEmailFetcher(BaseFetcher):
+    """
+    Fetches a specific email body from ANY IMAP server and converts it to HTML.
+    """
+
+    component_name = "ImapEmailFetcher"
+    SUPPORTED_TYPES = ["imap", "email"]
+
+    @classmethod
+    def can_handle(cls, uri: str) -> float:
+        return 1.0 if uri.startswith("imap-msg://") else 0.0
+
+    def _do_fetch(self, task: SayouTask) -> Dict[str, Any]:
+        """
+        Reconnects -> Fetches UID -> Parses -> Returns HTML String.
+        """
+        params = task.params
+        uid = params["uid"]
+        folder = params.get("folder", "INBOX")
+        imap_server = params.get("imap_server")
+
+        if not imap_server:
+            imap_server = "imap.gmail.com"
+
+        mail = imaplib.IMAP4_SSL(imap_server)
+
+        try:
+            mail.login(params["username"], params["password"])
+            mail.select(folder)
+
+            status, msg_data = mail.fetch(uid, "(RFC822)")
+
+            if status != "OK" or not msg_data:
+                raise ValueError(
+                    f"Email UID {uid} not found or fetch failed on {imap_server}."
+                )
+
+            raw_email = msg_data[0][1]
+            msg = email.message_from_bytes(raw_email)
+
+            parsed_content = self._parse_email(msg)
+
+            html_doc = f"""
+                <!DOCTYPE html>
+                <html>
+                <head>
+                    <title>{parsed_content['subject']}</title>
+                    <meta name="sender" content="{parsed_content['sender']}">
+                    <meta name="date" content="{parsed_content['date']}">
+                    <meta name="uid" content="{uid}">
+                    <meta name="source" content="imap">
+                    <meta name="server" content="{imap_server}">
+                </head>
+                <body>
+                    {parsed_content['body']}
+                </body>
+                </html>
+            """
+
+            return html_doc.strip()
+
+        except Exception as e:
+            raise RuntimeError(f"Failed to fetch email from {imap_server}: {e}")
+
+        finally:
+            try:
+                mail.logout()
+            except:
+                pass
+
+    def _parse_email(self, msg) -> Dict[str, Any]:
+        subject = self._decode_header(msg["Subject"])
+        sender = self._decode_header(msg["From"])
+        date = msg["Date"]
+
+        body_content = ""
+        html_found = False
+
+        if msg.is_multipart():
+            for part in msg.walk():
+                ctype = part.get_content_type()
+                payload = part.get_payload(decode=True)
+
+                if not payload:
+                    continue
+
+                try:
+                    text = payload.decode(
+                        part.get_content_charset() or "utf-8", errors="ignore"
+                    )
+                except:
+                    text = payload.decode("utf-8", errors="ignore")
+
+                if ctype == "text/html":
+                    body_content = text
+                    html_found = True
+
+                elif ctype == "text/plain":
+                    if not html_found:
+                        body_content = text
+
+        else:
+            body_content = msg.get_payload(decode=True).decode(errors="ignore")
+
+        return {
+            "subject": subject,
+            "sender": sender,
+            "date": date,
+            "body": body_content,
+        }
+
+    def _decode_header(self, header_text):
+        """Decodes MIME headers (e.g., =?utf-8?b?...)"""
+        if not header_text:
+            return "(No Subject)"
+        decoded_list = decode_header(header_text)
+        text = ""
+        for bytes_str, encoding in decoded_list:
+            if isinstance(bytes_str, bytes):
+                text += bytes_str.decode(encoding or "utf-8", errors="ignore")
+            else:
+                text += str(bytes_str)
+        return text

sayou/connector/plugins/imap_email_generator.py

@@ -0,0 +1,93 @@
+import imaplib
+from typing import Iterator
+
+from sayou.core.registry import register_component
+from sayou.core.schemas import SayouTask
+
+from ..interfaces.base_generator import BaseGenerator
+
+
+@register_component("generator")
+class ImapEmailGenerator(BaseGenerator):
+    """
+    Scans Generic IMAP inbox and generates tasks for individual emails.
+    Supports Gmail, Naver, Daum, Outlook, etc.
+    """
+
+    component_name = "ImapEmailGenerator"
+    SUPPORTED_TYPES = ["imap", "email"]
+
+    @classmethod
+    def can_handle(cls, source: str) -> float:
+        return 1.0 if source.startswith("imap://") else 0.0
+
+    def _do_generate(self, source: str, **kwargs) -> Iterator[SayouTask]:
+        """
+        Connects to IMAP Server -> Search -> Yield Tasks.
+        """
+        # 1. Parse connection information
+        parsed_host = source.replace("imap://", "").strip()
+        imap_server = (
+            parsed_host if parsed_host else kwargs.get("imap_server", "imap.gmail.com")
+        )
+
+        username = kwargs.get("username")
+        password = kwargs.get("password")
+
+        if not username or not password:
+            raise ValueError(
+                "IMAP credentials (username, password) required in kwargs."
+            )
+
+        folder = kwargs.get("folder", "INBOX")
+        limit = int(kwargs.get("limit", 10))
+
+        # 2. IMAP connection and search
+        mail = imaplib.IMAP4_SSL("imap.gmail.com")
+        try:
+            mail = imaplib.IMAP4_SSL(imap_server)
+            mail.login(username, password)
+            mail.select(folder)
+
+            # Search criteria (e.g., '(UNSEEN)' or 'ALL')
+            criteria = kwargs.get("search_criteria", "ALL")
+            status, messages = mail.search(None, criteria)
+
+            if status != "OK":
+                return
+
+            mail_ids = messages[0].split()
+            target_ids = mail_ids[-limit:]
+
+            self._log(
+                f"📧 [{imap_server}] Found {len(mail_ids)} emails. Generating tasks for last {len(target_ids)}."
+            )
+
+            # 3. Task generation (one task per email)
+            for b_id in reversed(target_ids):
+                uid = b_id.decode()
+
+                # Fetcher will process this internal protocol
+                task_uri = f"imap-msg://{imap_server}/{folder}/{uid}"
+
+                yield SayouTask(
+                    uri=task_uri,
+                    source_type="imap",
+                    params={
+                        "imap_server": imap_server,
+                        "username": username,
+                        "password": password,
+                        "uid": uid,
+                        "folder": folder,
+                    },
+                    meta={"source": "imap", "server": imap_server, "email_id": uid},
+                )
+
+        except Exception as e:
+            raise RuntimeError(f"IMAP connection failed to {imap_server}: {e}")
+        finally:
+            try:
+                mail.close()
+                mail.logout()
+            except:
+                pass