PyPI - sayou-connector - Versions diffs - 0.3.7__tar.gz → 0.3.16__tar.gz - Mend

sayou-connector 0.3.7tar.gz → 0.3.16tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (39) hide show

{sayou_connector-0.3.7 → sayou_connector-0.3.16}/PKG-INFO RENAMED Viewed

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sayou-connector
-Version: 0.3.7
+Version: 0.3.16
 Summary: Connector components for the Sayou Data Platform
 Project-URL: Homepage, https://www.sayouzone.com/
 Project-URL: Documentation, https://sayouzone.github.io/sayou-fabric/

{sayou_connector-0.3.7 → sayou_connector-0.3.16}/pyproject.toml RENAMED Viewed

@@ -7,7 +7,7 @@ build-backend = "hatchling.build"
 # -----------------
 [project]
 name = "sayou-connector"
-version = "0.3.7"
+version = "0.3.16"
 authors = [
     { name = "Sayouzone", email = "contact@sayouzone.com" },
 ]

sayou_connector-0.3.16/src/sayou/connector/plugins/gmail_fetcher.py ADDED Viewed

@@ -0,0 +1,128 @@
+import base64
+from typing import Any, Dict
+from sayou.core.registry import register_component
+from sayou.core.schemas import SayouTask
+from ..interfaces.base_fetcher import BaseFetcher
+try:
+    from google.oauth2.credentials import Credentials
+    from googleapiclient.discovery import build
+except ImportError:
+    build = None
+@register_component("fetcher")
+class GmailFetcher(BaseFetcher):
+    """
+    Fetches specific email content using Gmail API.
+    Reconstructs the email into a standardized HTML format suitable for Refinery.
+    """
+    component_name = "GmailFetcher"
+    SUPPORTED_TYPES = ["gmail"]
+    @classmethod
+    def can_handle(cls, uri: str) -> float:
+        return 1.0 if uri.startswith("gmail-msg://") else 0.0
+    def _do_fetch(self, task: SayouTask) -> Dict[str, Any]:
+        token_path = task.params.get("token_path")
+        msg_id = task.params.get("msg_id")
+        if not build:
+            raise ImportError("Please install google-api-python-client")
+        creds = Credentials.from_authorized_user_file(token_path)
+        service = build("gmail", "v1", credentials=creds)
+        # 1. Fetch email details (format='full')
+        message = (
+            service.users()
+            .messages()
+            .get(userId="me", id=msg_id, format="full")
+            .execute()
+        )
+        payload = message.get("payload", {})
+        headers = payload.get("headers", [])
+        # 2. Parse headers (Subject, From, Date)
+        subject = self._get_header(headers, "Subject", "(No Subject)")
+        sender = self._get_header(headers, "From", "Unknown")
+        date = self._get_header(headers, "Date", "")
+        # 3. Extract body (Recursive)
+        body_content = self._extract_body(payload)
+        # 4. Reconstruct HTML (User Request Format)
+        html_doc = f"""<!DOCTYPE html>
+<html>
+<head>
+    <title>{subject}</title>
+    <meta name="sender" content="{sender}">
+    <meta name="date" content="{date}">
+    <meta name="msg_id" content="{msg_id}">
+    <meta name="source" content="gmail">
+</head>
+<body>
+{body_content}
+</body>
+</html>"""
+        return html_doc.strip()
+    def _get_header(self, headers: list, name: str, default: str) -> str:
+        for h in headers:
+            if h["name"].lower() == name.lower():
+                return h["value"]
+        return default
+    def _extract_body(self, payload: dict) -> str:
+        body = ""
+        # Case A: Single Part
+        if "body" in payload and payload["body"].get("data"):
+            mime_type = payload.get("mimeType", "")
+            data = payload["body"]["data"]
+            decoded_text = self._decode_base64url(data)
+            if mime_type == "text/html":
+                return decoded_text
+            elif mime_type == "text/plain":
+                return f"<pre>{decoded_text}</pre>"
+        # Case B: Multi Part
+        if "parts" in payload:
+            html_part = None
+            text_part = None
+            for part in payload["parts"]:
+                mime_type = part.get("mimeType", "")
+                content = self._extract_body(part)
+                if mime_type == "text/html":
+                    html_part = content
+                elif mime_type == "text/plain":
+                    text_part = content
+                elif "multipart" in mime_type:
+                    if content:
+                        html_part = content
+            if html_part:
+                return html_part
+            if text_part:
+                return text_part
+        return body
+    def _decode_base64url(self, data: str) -> str:
+        try:
+            padding = len(data) % 4
+            if padding:
+                data += "=" * (4 - padding)
+            return base64.urlsafe_b64decode(data).decode("utf-8", errors="replace")
+        except Exception:
+            return ""

sayou_connector-0.3.16/src/sayou/connector/plugins/gmail_generator.py ADDED Viewed

@@ -0,0 +1,81 @@
+from typing import Iterator
+from sayou.core.registry import register_component
+from sayou.core.schemas import SayouTask
+from ..interfaces.base_generator import BaseGenerator
+try:
+    from google.oauth2.credentials import Credentials
+    from googleapiclient.discovery import build
+except ImportError:
+    build = None
+@register_component("generator")
+class GmailGenerator(BaseGenerator):
+    """
+    Scans Gmail inbox using Gmail API (OAuth) and generates tasks.
+    """
+    component_name = "GmailGenerator"
+    SUPPORTED_TYPES = ["gmail"]
+    @classmethod
+    def can_handle(cls, source: str) -> float:
+        return 1.0 if source.startswith("gmail://") else 0.0
+    def _do_generate(self, source: str, **kwargs) -> Iterator[SayouTask]:
+        """
+        Connects to Gmail API -> Search (List) -> Yield Tasks.
+        source example: gmail://me (default) or gmail://me?q=is:unread
+        """
+        if not build:
+            raise ImportError(
+                "Please install google-api-python-client google-auth-oauthlib"
+            )
+        token_path = kwargs.get("token_path")
+        if not token_path:
+            raise ValueError("GmailGenerator requires 'token_path' in kwargs.")
+        # 1. Parsing Parameters
+        query = kwargs.get("query", "is:inbox")
+        max_results = int(kwargs.get("limit", 10))
+        # 2. Connect to Gmail API
+        creds = Credentials.from_authorized_user_file(token_path)
+        service = build("gmail", "v1", credentials=creds)
+        try:
+            # 3. Fetch email list
+            results = (
+                service.users()
+                .messages()
+                .list(userId="me", q=query, maxResults=max_results)
+                .execute()
+            )
+            messages = results.get("messages", [])
+            self._log(f"📧 Found {len(messages)} emails. Generating tasks...")
+            # 4. Generate tasks
+            for msg in messages:
+                msg_id = msg["id"]
+                thread_id = msg["threadId"]
+                task_uri = f"gmail-msg://{msg_id}"
+                yield SayouTask(
+                    uri=task_uri,
+                    source_type="gmail",
+                    params={
+                        "token_path": token_path,
+                        "msg_id": msg_id,
+                        "thread_id": thread_id,
+                    },
+                )
+        except Exception as e:
+            self._log(f"Gmail API List failed: {e}", level="error")
+            raise e

sayou_connector-0.3.16/src/sayou/connector/plugins/google_calendar_fetcher.py ADDED Viewed

@@ -0,0 +1,89 @@
+import datetime
+import os
+from typing import Any, Dict
+from sayou.core.registry import register_component
+from sayou.core.schemas import SayouTask
+from ..interfaces.base_fetcher import BaseFetcher
+try:
+    from google.oauth2.credentials import Credentials
+    from googleapiclient.discovery import build
+except ImportError:
+    build = None
+@register_component("fetcher")
+class GoogleCalendarFetcher(BaseFetcher):
+    """
+    Fetches events using Google API with User OAuth Token.
+    Works for Workspace (Corporate) & Personal accounts.
+    """
+    component_name = "GoogleCalendarFetcher"
+    SUPPORTED_TYPES = ["google_calendar"]
+    @classmethod
+    def can_handle(cls, uri: str) -> float:
+        return 1.0 if uri.startswith("gcal://") else 0.0
+    def _do_fetch(self, task: SayouTask) -> Dict[str, Any]:
+        if not build:
+            raise ImportError("Google API libraries required.")
+        token_path = task.params.get("token_path")
+        if not token_path or not os.path.exists(token_path):
+            raise ValueError("Token path invalid.")
+        creds = Credentials.from_authorized_user_file(token_path)
+        service = build("calendar", "v3", credentials=creds)
+        now = datetime.datetime.utcnow()
+        time_min = (now - datetime.timedelta(days=30)).isoformat() + "Z"
+        time_max = (now + datetime.timedelta(days=30)).isoformat() + "Z"
+        events_result = (
+            service.events()
+            .list(
+                calendarId="primary",
+                timeMin=time_min,
+                timeMax=time_max,
+                singleEvents=True,
+                orderBy="startTime",
+            )
+            .execute()
+        )
+        events = events_result.get("items", [])
+        parsed_events = []
+        for event in events:
+            start = event["start"].get("dateTime", event["start"].get("date"))
+            end = event["end"].get("dateTime", event["end"].get("date"))
+            parsed_events.append(
+                {
+                    "id": event.get("id"),
+                    "summary": event.get("summary", "No Title"),
+                    "description": event.get("description", ""),
+                    "start": start,
+                    "end": end,
+                    "location": event.get("location", ""),
+                    "htmlLink": event.get("htmlLink", ""),
+                    "attendees": [
+                        {"email": a.get("email"), "status": a.get("responseStatus")}
+                        for a in event.get("attendees", [])
+                    ],
+                }
+            )
+        return {
+            "content": parsed_events,
+            "meta": {
+                "source": "google_calendar",
+                "account": "authenticated_user",
+                "count": len(parsed_events),
+            },
+        }

sayou_connector-0.3.16/src/sayou/connector/plugins/google_calendar_generator.py ADDED Viewed

@@ -0,0 +1,46 @@
+import os
+from typing import Iterator
+from sayou.core.registry import register_component
+from sayou.core.schemas import SayouTask
+from ..interfaces.base_generator import BaseGenerator
+@register_component("generator")
+class GoogleCalendarGenerator(BaseGenerator):
+    """
+    Generates tasks using OAuth 2.0 Token.
+    Requires 'sayou_google_token.json' (generated by auth script).
+    """
+    component_name = "GoogleCalendarGenerator"
+    SUPPORTED_TYPES = ["google_calendar"]
+    @classmethod
+    def can_handle(cls, uri: str) -> float:
+        return 1.0 if uri.startswith("gcal://") else 0.0
+    def _do_generate(self, source: str, **kwargs) -> Iterator[SayouTask]:
+        token_path = kwargs.get("google_token_path")
+        if not os.path.exists(token_path):
+            raise FileNotFoundError(
+                f"Google Token not found at {token_path}. Run authentication script first."
+            )
+        calendar_id = source.replace("gcal://", "") or "primary"
+        yield SayouTask(
+            uri=source,
+            source_type="google_calendar",
+            params={
+                "calendar_id": calendar_id,
+                "token_path": token_path,
+            },
+            meta={
+                "source": "google_calendar",
+                "calendar_id": calendar_id,
+                "filename": f"calendar_{calendar_id.replace('@', '_')}",
+            },
+        )

sayou_connector-0.3.16/src/sayou/connector/plugins/google_docs_fetcher.py ADDED Viewed

@@ -0,0 +1,113 @@
+import html
+import io
+import re
+from typing import Any, Dict
+from sayou.core.registry import register_component
+from sayou.core.schemas import SayouTask
+from ..interfaces.base_fetcher import BaseFetcher
+try:
+    from google.oauth2.credentials import Credentials
+    from googleapiclient.discovery import build
+    from googleapiclient.http import MediaIoBaseDownload
+except ImportError:
+    build = None
+@register_component("fetcher")
+class GoogleDocsFetcher(BaseFetcher):
+    """
+    Fetches content from Google Docs using the Docs API (v1).
+    Extracts text and converts basic styling to Markdown.
+    Bypasses the 10MB export limit of Drive API.
+    """
+    component_name = "GoogleDocsFetcher"
+    SUPPORTED_TYPES = ["docs"]
+    @classmethod
+    def can_handle(cls, uri: str) -> float:
+        return 1.0 if uri.startswith("gdocs://document/") else 0.0
+    def _do_fetch(self, task: SayouTask) -> Dict[str, Any]:
+        token_path = task.params.get("token_path")
+        doc_id = task.uri.replace("gdocs://document/", "").split("/")[0]
+        original_title = task.meta.get("filename", "Untitled")
+        fetch_mode = task.params.get("fetch_mode", "html")
+        creds = Credentials.from_authorized_user_file(token_path)
+        if fetch_mode == "html":
+            return self._fetch_as_html(creds, doc_id, original_title)
+        else:
+            return self._fetch_as_json(creds, doc_id, original_title)
+    # =========================================================
+    # Mode 1: JSON (Docs API)
+    # =========================================================
+    def _fetch_as_json(self, creds, doc_id, title) -> Dict[str, Any]:
+        try:
+            service = build("docs", "v1", credentials=creds)
+            document = service.documents().get(documentId=doc_id).execute()
+            tab_count = len(document.get("tabs", [])) if "tabs" in document else 0
+            if tab_count > 0:
+                self._log(f"ℹ️ Fetched JSON with {tab_count} tabs.")
+            return document
+            # {
+            #     "content": document,
+            #     "meta": {
+            #         "source": "google_docs",
+            #         "doc_id": doc_id,
+            #         "title": title,
+            #         "extension": ".json",
+            #         "mode": "api_json",
+            #     },
+            # }
+        except Exception as e:
+            self._log(f"Docs API Failed: {e}", level="error")
+            raise e
+    # =========================================================
+    # Mode 2: HTML (Drive API Export)
+    # =========================================================
+    def _fetch_as_html(self, creds, doc_id, title) -> Dict[str, Any]:
+        try:
+            service = build("drive", "v3", credentials=creds)
+            request = service.files().export_media(fileId=doc_id, mimeType="text/html")
+            fh = io.BytesIO()
+            downloader = MediaIoBaseDownload(fh, request)
+            done = False
+            while done is False:
+                status, done = downloader.next_chunk()
+            raw_bytes = fh.getvalue()
+            html_str = raw_bytes.decode("utf-8", errors="replace")
+            decoded_html = html.unescape(html_str)
+            formatted_html = re.sub(
+                r"(</(p|div|h[1-6]|li|ul|ol|table|tr|blockquote)>)",
+                r"\1\n",
+                decoded_html,
+                flags=re.IGNORECASE,
+            )
+            return formatted_html
+            # {
+            #     "content": human_readable_html,
+            #     "meta": {
+            #         "source": "google_docs",
+            #         "doc_id": doc_id,
+            #         "title": title,
+            #         "extension": ".html",
+            #         "mode": "export_html",
+            #     },
+            # }
+        except Exception as e:
+            self._log(f"HTML Export Failed: {e}", level="error")
+            raise e

sayou_connector-0.3.16/src/sayou/connector/plugins/google_drive_fetcher.py ADDED Viewed

@@ -0,0 +1,151 @@
+import io
+import os
+from typing import Any, Dict
+from sayou.core.registry import register_component
+from sayou.core.schemas import SayouTask
+from ..interfaces.base_fetcher import BaseFetcher
+try:
+    import chardet
+    from google.oauth2.credentials import Credentials
+    from googleapiclient.discovery import build
+    from googleapiclient.errors import HttpError
+    from googleapiclient.http import MediaIoBaseDownload
+except ImportError:
+    build = None
+    chardet = None
+@register_component("fetcher")
+class GoogleDriveFetcher(BaseFetcher):
+    """
+    Fetches content from Google Drive files.
+    - Google Native Formats -> Converted to MS Office formats (.docx, .xlsx, .pptx)
+    - Standard Files (PDF, JPG, ZIP...) -> Downloaded as original binary.
+    """
+    component_name = "GoogleDriveFetcher"
+    SUPPORTED_TYPES = ["drive"]
+    @classmethod
+    def can_handle(cls, uri: str) -> float:
+        return 1.0 if uri.startswith("gdrive://file/") else 0.0
+    def _do_fetch(self, task: SayouTask) -> Dict[str, Any]:
+        token_path = task.params.get("token_path")
+        file_id = task.params.get("file_id")
+        mime_type = task.params.get("mime_type")
+        original_name = task.meta.get("filename", "unknown_file")
+        creds = Credentials.from_authorized_user_file(token_path)
+        service = build("drive", "v3", credentials=creds)
+        request = None
+        extension = ""
+        is_google_doc = False
+        # 1. Google Native Formats
+        if mime_type == "application/vnd.google-apps.document":
+            request = service.files().export_media(
+                fileId=file_id,
+                mimeType="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+            )
+            extension = ".docx"
+            is_google_doc = True
+        elif mime_type == "application/vnd.google-apps.spreadsheet":
+            request = service.files().export_media(
+                fileId=file_id,
+                mimeType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+            )
+            extension = ".xlsx"
+            is_google_doc = True
+        elif mime_type == "application/vnd.google-apps.presentation":
+            request = service.files().export_media(
+                fileId=file_id,
+                mimeType="application/vnd.openxmlformats-officedocument.presentationml.presentation",
+            )
+            extension = ".pptx"
+            is_google_doc = True
+        else:
+            request = service.files().get_media(fileId=file_id)
+            _, ext = os.path.splitext(original_name)
+            extension = ext if ext else ""
+        # 2. Execute Download
+        try:
+            fh = io.BytesIO()
+            downloader = MediaIoBaseDownload(fh, request)
+            done = False
+            while done is False:
+                status, done = downloader.next_chunk()
+            raw_bytes = fh.getvalue()
+            final_content = raw_bytes
+            is_text_candidate = False
+            if mime_type.startswith("text/") or mime_type == "application/json":
+                is_text_candidate = True
+            elif extension.lower() in [
+                ".csv",
+                ".txt",
+                ".json",
+                ".md",
+                ".py",
+                ".html",
+                ".xml",
+            ]:
+                is_text_candidate = True
+            if not is_google_doc and is_text_candidate:
+                detected = chardet.detect(raw_bytes)
+                encoding = detected.get("encoding")
+                confidence = detected.get("confidence", 0)
+                # 2) EUC-KR -> UTF-8
+                if (
+                    encoding
+                    and encoding.lower() not in ["utf-8", "ascii"]
+                    and confidence > 0.6
+                ):
+                    try:
+                        # Decode (Bytes -> Str)
+                        text_content = raw_bytes.decode(encoding)
+                        # Encode back to Bytes (Str -> UTF-8 Bytes)
+                        final_content = text_content.encode("utf-8")
+                        self._log(
+                            f"Transcoded {original_name} from {encoding} to utf-8 bytes."
+                        )
+                    except Exception as e:
+                        self._log(
+                            f"Encoding conversion failed: {e}. Keeping raw bytes.",
+                            level="warning",
+                        )
+                        final_content = raw_bytes
+            # 3. Return
+            return {
+                "content": final_content,
+                "meta": {
+                    "source": "google_drive",
+                    "file_id": file_id,
+                    "mime_type": mime_type,
+                    "original_filename": original_name,
+                    "suggested_filename": (
+                        f"{original_name}{extension}"
+                        if not original_name.endswith(extension)
+                        else original_name
+                    ),
+                    "extension": extension,
+                    "is_binary": isinstance(final_content, bytes),
+                },
+            }
+        except HttpError as e:
+            self._log(f"Drive Download Failed ({file_id}): {e}", level="error")
+            return {
+                "content": b"",
+                "meta": {"source": "google_drive", "error": str(e), "file_id": file_id},
+            }

sayou-connector 0.3.7__tar.gz → 0.3.16__tar.gz

sayou-connector 0.3.7tar.gz → 0.3.16tar.gz