sayou-connector 0.3.12__py3-none-any.whl

sayou/connector/__init__.py

@@ -0,0 +1,11 @@
+from .generator.file_generator import FileGenerator
+from .generator.requests_generator import RequestsGenerator
+from .generator.sqlite_generator import SqliteGenerator
+from .pipeline import ConnectorPipeline
+
+__all__ = [
+    "ConnectorPipeline",
+    "FileGenerator",
+    "RequestsGenerator",
+    "SqliteGenerator",
+]

sayou/connector/core/exceptions.py

@@ -0,0 +1,38 @@
+from sayou.core.exceptions import SayouCoreError
+
+
+class ConnectorError(SayouCoreError):
+    """
+    Base exception for all errors within the sayou-connector toolkit.
+
+    All specific exceptions in this module should inherit from this class
+    to allow catching connector-specific issues globally.
+    """
+
+    pass
+
+
+class FetcherError(ConnectorError):
+    """
+    Exception raised when a Fetcher fails to retrieve data.
+
+    Examples:
+        - File not found on disk.
+        - HTTP connection timeout or 404/500 errors.
+        - Database connection failure.
+    """
+
+    pass
+
+
+class GeneratorError(ConnectorError):
+    """
+    Exception raised when a Generator fails to produce tasks.
+
+    Examples:
+        - Invalid start path or configuration.
+        - Failure to parse a sitemap or initial seed page.
+        - Logic errors in the generation strategy.
+    """
+
+    pass

sayou/connector/fetcher/file_fetcher.py

@@ -0,0 +1,42 @@
+import os
+
+from sayou.core.registry import register_component
+from sayou.core.schemas import SayouTask
+
+from ..interfaces.base_fetcher import BaseFetcher
+
+
+@register_component("fetcher")
+class FileFetcher(BaseFetcher):
+    """
+    Concrete implementation of BaseFetcher for local file systems.
+
+    This fetcher reads binary data directly from the path specified in `task.uri`.
+    It handles basic file I/O operations and raises wrapped exceptions if the file
+    is inaccessible or missing.
+    """
+
+    component_name = "FileFetcher"
+    SUPPORTED_TYPES = ["file"]
+
+    def _do_fetch(self, task: SayouTask) -> bytes:
+        """
+        Read a file from the local file system.
+
+        Args:
+            task (SayouTask): The task containing the file path in `task.uri`.
+
+        Returns:
+            bytes: The raw binary content of the file.
+
+        Raises:
+            FileNotFoundError: If the file does not exist.
+            IOError: If the file cannot be read.
+        """
+        file_path = task.uri
+
+        if not os.path.exists(file_path):
+            raise FileNotFoundError(f"File not found: {file_path}")
+
+        with open(file_path, "rb") as f:
+            return f.read()

sayou/connector/fetcher/requests_fetcher.py

@@ -0,0 +1,77 @@
+from urllib.parse import urljoin
+
+import requests
+
+try:
+    from bs4 import BeautifulSoup
+except ImportError:
+    BeautifulSoup = None
+
+from sayou.core.registry import register_component
+from sayou.core.schemas import SayouTask
+
+from ..interfaces.base_fetcher import BaseFetcher
+
+
+@register_component("fetcher")
+class RequestsFetcher(BaseFetcher):
+    """
+    Concrete implementation of BaseFetcher for static web pages.
+
+    Retrieves HTML content via HTTP requests. It supports optional CSS selector
+    extraction (via `task.params['selectors']`) and automatically discovers
+    hyperlinks on the page to support the `WebCrawlGenerator` feedback loop.
+    """
+
+    component_name = "RequestsFetcher"
+    SUPPORTED_TYPES = ["requests"]
+
+    def _do_fetch(self, task: SayouTask) -> dict:
+        """
+        Fetch a web page and extract data/links.
+
+        Args:
+            task (SayouTask): The task containing the URL in `task.uri`.
+                            `task.params` may contain 'selectors'.
+
+        Returns:
+            dict: A dictionary containing extracted text, raw preview,
+                and found links under '__found_links__'.
+
+        Raises:
+            requests.RequestException: For network-related errors.
+            ImportError: If BeautifulSoup is not installed.
+        """
+        if not BeautifulSoup:
+            raise ImportError("BeautifulSoup4 not installed.")
+
+        headers = {"User-Agent": "Sayou-Connector/0.1.0"}
+        resp = requests.get(task.uri, headers=headers, timeout=10)
+        resp.raise_for_status()
+
+        soup = BeautifulSoup(resp.text, "html.parser")
+        extracted_data = {}
+
+        # 1. Selectors logic
+        selectors = task.params.get("selectors", {})
+        if selectors:
+            for key, sel in selectors.items():
+                el = soup.select(sel)
+                if el:
+                    extracted_data[key] = "\n".join(
+                        [e.get_text(strip=True) for e in el]
+                    )
+
+        if not extracted_data:
+            extracted_data["_raw_preview"] = resp.text[:200]
+
+        # 2. Link extraction logic
+        found_links = set()
+        for a in soup.find_all("a", href=True):
+            abs_link = urljoin(task.uri, a["href"])
+            if abs_link.startswith("http"):
+                found_links.add(abs_link)
+
+        extracted_data["__found_links__"] = list(found_links)
+
+        return extracted_data

sayou/connector/fetcher/sqlite_fetcher.py

@@ -0,0 +1,50 @@
+import sqlite3
+from typing import Any, Dict, List
+
+from sayou.core.registry import register_component
+from sayou.core.schemas import SayouTask
+
+from ..interfaces.base_fetcher import BaseFetcher
+
+
+@register_component("fetcher")
+class SqliteFetcher(BaseFetcher):
+    """
+    Concrete implementation of BaseFetcher for SQLite databases.
+
+    Connects to the SQLite database file specified in `task.uri` and executes
+    the SQL query provided in `task.params['query']`. It manages connection
+    lifecycles using context managers and returns results as a list of dictionaries.
+    """
+
+    component_name = "SqliteFetcher"
+    SUPPORTED_TYPES = ["sqlite"]
+
+    def _do_fetch(self, task: SayouTask) -> List[Dict[str, Any]]:
+        """
+        Execute a SQL query against a SQLite database.
+
+        Args:
+            task (SayouTask): The task containing the DB path in `task.uri`
+                                and the SQL query in `task.params['query']`.
+
+        Returns:
+            List[Dict[str, Any]]: A list of rows, where each row is a dictionary.
+
+        Raises:
+            sqlite3.Error: If the database connection or query execution fails.
+        """
+        db_path = task.uri
+        query = task.params.get("query")
+
+        if not query:
+            raise ValueError("Query param is missing in SayouTask")
+
+        with sqlite3.connect(db_path) as conn:
+            conn.row_factory = sqlite3.Row
+            cursor = conn.cursor()
+
+            self._log(f"Executing query on {db_path}: {query[:50]}...", level="debug")
+
+            cursor.execute(query)
+            return [dict(row) for row in cursor.fetchall()]

sayou/connector/generator/file_generator.py

@@ -0,0 +1,124 @@
+import fnmatch
+import os
+from typing import Iterator
+
+from sayou.core.registry import register_component
+from sayou.core.schemas import SayouTask
+
+from ..interfaces.base_generator import BaseGenerator
+
+
+@register_component("generator")
+class FileGenerator(BaseGenerator):
+    """
+    Concrete implementation of BaseGenerator for file system traversal.
+
+    Scans a directory tree starting from a source path. It yields `SayouTask`s
+    for files that match specific criteria, such as file extensions or name patterns.
+    Supports both recursive and flat directory scanning.
+    """
+
+    component_name = "FileGenerator"
+    SUPPORTED_TYPES = ["file"]
+
+    @classmethod
+    def can_handle(cls, source: str) -> float:
+        """
+        Evaluates whether this generator can handle the given source.
+
+        Analyzes the source string to determine if it matches the pattern or format
+        supported by this generator. Returns a confidence score between 0.0 and 1.0.
+
+        Args:
+            source (str): The input source string to evaluate.
+
+        Returns:
+            float: A confidence score where 1.0 means full confidence,
+                    0.0 means the source is incompatible, and intermediate values
+                    indicate partial matches or heuristics.
+        """
+        if os.path.exists(source):
+            return 1.0
+        if source.startswith("/") or source.startswith("./") or ":\\" in source:
+            return 0.8
+        return 0.0
+
+    def initialize(
+        self,
+        source: str,
+        recursive: bool = True,
+        extensions: list = None,
+        name_pattern: str = "*",
+        **kwargs
+    ):
+        """
+        Configure the file scanning strategy.
+
+        Args:
+            source (str): The root directory or file path to start scanning.
+            recursive (bool): If True, scan subdirectories recursively.
+            extensions (Optional[List[str]]): List of allowed extensions (e.g., ['.pdf', '.txt']).
+            name_pattern (str): Glob pattern for filename matching (e.g., '*report*').
+            **kwargs: Ignored additional arguments.
+        """
+        self.root_path = os.path.abspath(source)
+        self.recursive = recursive
+        self.extensions = [e.lower() for e in extensions] if extensions else None
+        self.name_pattern = name_pattern
+
+    def _do_generate(self, source: str, **kwargs) -> Iterator[SayouTask]:
+        """
+        Walk through the file system and yield tasks for valid files.
+
+        Yields:
+            Iterator[SayouTask]: Tasks with `source_type='file'`.
+        """
+        if os.path.isfile(self.root_path):
+            if self._is_valid(self.root_path):
+                yield self._create_task(self.root_path)
+            return
+
+        walker = (
+            os.walk(self.root_path)
+            if self.recursive
+            else [(self.root_path, [], os.listdir(self.root_path))]
+        )
+
+        for root, _, files in walker:
+            for file in files:
+                full_path = os.path.join(root, file)
+                if self._is_valid(file):
+                    yield self._create_task(full_path)
+
+    def _is_valid(self, filename: str) -> bool:
+        """
+        Check if a file matches the extension and name pattern criteria.
+
+        Args:
+            filename (str): The name of the file to check.
+
+        Returns:
+            bool: True if the file should be processed, False otherwise.
+        """
+        if not fnmatch.fnmatch(filename, self.name_pattern):
+            return False
+        if (
+            self.extensions
+            and os.path.splitext(filename)[1].lower() not in self.extensions
+        ):
+            return False
+        return True
+
+    def _create_task(self, path: str) -> SayouTask:
+        """
+        Create a SayouTask for a valid file path.
+
+        Args:
+            path (str): The absolute path to the file.
+
+        Returns:
+            SayouTask: The configured task object.
+        """
+        return SayouTask(
+            source_type="file", uri=path, meta={"filename": os.path.basename(path)}
+        )

sayou/connector/generator/requests_generator.py

@@ -0,0 +1,113 @@
+import re
+from collections import deque
+from typing import Iterator
+
+from sayou.core.registry import register_component
+from sayou.core.schemas import SayouPacket, SayouTask
+
+from ..interfaces.base_generator import BaseGenerator
+
+
+@register_component("generator")
+class RequestsGenerator(BaseGenerator):
+    """
+    Concrete implementation of BaseGenerator for web crawling.
+
+    Manages a frontier queue of URLs to visit. It starts from a seed URL and
+    dynamically adds new targets based on links discovered by the Fetcher (Feedback),
+    respecting maximum depth and URL pattern constraints.
+    """
+
+    component_name = "RequestsGenerator"
+    SUPPORTED_TYPES = ["requests"]
+
+    @classmethod
+    def can_handle(cls, source: str) -> float:
+        """
+        Evaluates whether this generator can handle the given source.
+
+        Analyzes the source string to determine if it matches the pattern or format
+        supported by this generator. Returns a confidence score between 0.0 and 1.0.
+
+        Args:
+            source (str): The input source string to evaluate.
+
+        Returns:
+            float: A confidence score where 1.0 means full confidence,
+                    0.0 means the source is incompatible, and intermediate values
+                    indicate partial matches or heuristics.
+        """
+        s = source.strip().lower()
+
+        if s.startswith("http://") or s.startswith("https://"):
+            return 1.0
+
+        if s.startswith("www."):
+            return 0.8
+
+        return 0.0
+
+    def initialize(
+        self,
+        source: str,
+        link_pattern: str = ".*",
+        selectors: dict = None,
+        max_depth: int = 1,
+        **kwargs,
+    ):
+        """
+        Configure the web crawling strategy.
+
+        Args:
+            source (str): The seed URL to start crawling.
+            link_pattern (str): Regex pattern to filter links to follow.
+            selectors (Optional[dict]): CSS selectors to extract specific data from pages.
+            max_depth (int): Maximum depth to traverse from the seed URL.
+            **kwargs: Ignored additional arguments.
+        """
+        self.queue = deque([(source, 0)])
+        self.visited = {source}
+        self.link_regex = re.compile(link_pattern)
+        self.selectors = selectors or {}
+        self.max_depth = max_depth
+
+    def _do_generate(self, source: str, **kwargs) -> Iterator[SayouTask]:
+        """
+        Yield tasks from the crawling queue.
+
+        Yields:
+            Iterator[SayouTask]: Tasks for URLs in the queue.
+        """
+        while self.queue:
+            url, depth = self.queue.popleft()
+            yield SayouTask(
+                source_type="requests",
+                uri=url,
+                params={"selectors": self.selectors, "depth": depth},
+            )
+
+    def _do_feedback(self, result: SayouPacket):
+        """
+        Extract links from the fetched page and add them to the queue.
+
+        Args:
+            result (SayouPacket): The result containing extracted links ('__found_links__').
+        """
+        if not result.success or not result.data:
+            return
+
+        current_depth = result.task.params.get("depth", 0)
+        if current_depth >= self.max_depth:
+            return
+
+        links = result.data.get("__found_links__", [])
+        new_links = 0
+
+        for link in links:
+            if link not in self.visited and self.link_regex.search(link):
+                self.visited.add(link)
+                self.queue.append((link, current_depth + 1))
+                new_links += 1
+
+        if new_links > 0:
+            self._log(f"Added {new_links} new links (Depth {current_depth+1})")

sayou/connector/generator/sqlite_generator.py

@@ -0,0 +1,140 @@
+import os
+from typing import Iterator
+
+from sayou.core.registry import register_component
+from sayou.core.schemas import SayouPacket, SayouTask
+
+from ..interfaces.base_generator import BaseGenerator
+
+
+@register_component("generator")
+class SqliteGenerator(BaseGenerator):
+    """
+    Concrete implementation of BaseGenerator for SQL pagination.
+
+    Generates a sequence of database query tasks using LIMIT and OFFSET strategies.
+    It continues to yield tasks by incrementing the offset until the Fetcher returns
+    an empty result or a partial batch, indicating the end of the dataset.
+    """
+
+    component_name = "SqliteGenerator"
+    SUPPORTED_TYPES = ["sqlite"]
+
+    @classmethod
+    def can_handle(cls, source: str) -> float:
+        """
+        Evaluates whether this generator can handle the given source.
+
+        Analyzes the source string to determine if it matches the pattern or format
+        supported by this generator. Returns a confidence score between 0.0 and 1.0.
+
+        Args:
+            source (str): The input source string to evaluate.
+
+        Returns:
+            float: A confidence score where 1.0 means full confidence,
+                    0.0 means the source is incompatible, and intermediate values
+                    indicate partial matches or heuristics.
+        """
+        s = source.strip()
+
+        if s.lower().startswith("sqlite:///"):
+            return 1.0
+
+        if any(s.lower().endswith(ext) for ext in [".db", ".sqlite", ".sqlite3"]):
+            if os.path.isfile(s):
+                return 1.0
+            return 0.9
+
+        return 0.0
+
+    def initialize(
+        self,
+        source: str,
+        query: str = None,
+        batch_size: int = 1000,
+        **kwargs,
+    ):
+        """
+        Configure the SQL scanning strategy with pagination.
+
+        Args:
+            source (str): The database connection string or file path.
+            query (str): The base SQL query (without LIMIT/OFFSET).
+            batch_size (int): Number of rows to fetch per task.
+            **kwargs: Ignored additional arguments.
+        """
+        self.conn_str = self._clean_source(source)
+        self.base_query = (
+            query.strip().rstrip(";")
+            if query
+            else "SELECT name FROM sqlite_master WHERE type='table'"
+        )
+        self.batch_size = batch_size
+        self.current_offset = 0
+        self.stop_flag = False
+
+    def _clean_source(self, source: str) -> str:
+        """
+        Extracts the actual file path from a source URI.
+
+        Removes prefixes like 'sqlite:///' or 'sqlite://' from the source string
+        to return a clean file path compatible with the standard `sqlite3` connect method.
+
+        Args:
+            source (str): The input source string (e.g., 'sqlite:///data/test.db').
+
+        Returns:
+            str: The cleaned file path (e.g., '/data/test.db').
+        """
+        s = source.strip()
+        if s.lower().startswith("sqlite:///"):
+            return s[10:]
+        elif s.lower().startswith("sqlite://"):
+            return s[9:]
+        return s
+
+    def _do_generate(self, source: str, **kwargs) -> Iterator[SayouTask]:
+        """
+        Yield pagination tasks until the stop flag is set.
+
+        Yields:
+            Iterator[SayouTask]: Tasks with `source_type='sqlite'` and pagination params.
+        """
+        while not self.stop_flag:
+            paginated_query = f"{self.base_query} LIMIT {self.batch_size} OFFSET {self.current_offset}"
+
+            task = SayouTask(
+                source_type="sqlite",
+                uri=self.conn_str,
+                params={"query": paginated_query},
+                meta={"offset": self.current_offset, "batch": self.batch_size},
+            )
+
+            yield task
+
+            self.current_offset += self.batch_size
+
+    def _do_feedback(self, result: SayouPacket):
+        """
+        Determine if pagination should stop based on the fetch result.
+
+        If the number of fetched rows is less than `batch_size` or if the fetch failed,
+        the generator stops producing tasks.
+
+        Args:
+            result (SayouPacket): The result from the Fetcher.
+        """
+        # 1. 실패했거나 데이터가 없으면 종료
+        if not result.success or not result.data:
+            self._log("No data returned or fetch failed. Stopping.", level="warning")
+            self.stop_flag = True
+            return
+
+        # 2. 가져온 데이터가 배치 사이즈보다 작으면 '마지막 페이지'임
+        rows = result.data
+        if isinstance(rows, list) and len(rows) < self.batch_size:
+            self._log(
+                f"Reached end of records (Fetched {len(rows)} < Batch {self.batch_size})."
+            )
+            self.stop_flag = True

sayou/connector/interfaces/base_fetcher.py

@@ -0,0 +1,81 @@
+from abc import abstractmethod
+from typing import Any
+
+from sayou.core.base_component import BaseComponent
+from sayou.core.decorators import measure_time, retry
+from sayou.core.schemas import SayouPacket, SayouTask
+
+from ..core.exceptions import FetcherError
+
+
+class BaseFetcher(BaseComponent):
+    """
+    (Tier 1) Abstract base class for all data fetchers.
+
+    This class implements the Template Method pattern. It handles common logic
+    like logging, error wrapping, and retries in `fetch()`, while delegating
+    the actual retrieval logic to `_do_fetch()`.
+    """
+
+    component_name = "BaseFetcher"
+    SUPPORTED_TYPES = []
+
+    @classmethod
+    def can_handle(cls, uri: str) -> float:
+        """
+        Evaluates whether this fetcher can handle the specific Task URI.
+        """
+        return 0.0
+
+    @measure_time
+    @retry(max_retries=3, delay=1.0)
+    def fetch(self, task: SayouTask) -> SayouPacket:
+        """
+        Execute the fetching process for a given task.
+
+        This method wraps the actual fetching logic with error handling and logging.
+        It guarantees to return a SayouPacket, even if the operation fails.
+
+        Args:
+            task (SayouTask): The task definition containing the URI and parameters.
+
+        Returns:
+            SayouPacket: A packet containing the fetched data (on success)
+                        or error details (on failure).
+        """
+        self._emit("on_start", input_data=task)
+        self._log(f"Fetching: {task.uri} ({task.source_type})", level="debug")
+
+        try:
+            data = self._do_fetch(task)
+            packet = SayouPacket(task=task, data=data, success=True)
+            self._emit("on_finish", result_data=packet, success=True)
+            return packet
+
+        except Exception as e:
+            self._emit("on_error", error=e)
+            wrapped_error = FetcherError(
+                f"[{self.component_name}] Failed to fetch: {str(e)}"
+            )
+            self.logger.error(wrapped_error, exc_info=True)
+
+            return SayouPacket(
+                task=task, data=None, success=False, error=str(wrapped_error)
+            )
+
+    @abstractmethod
+    def _do_fetch(self, task: SayouTask) -> Any:
+        """
+        [Abstract Hook] Implement the actual data retrieval logic here.
+
+        Args:
+            task (SayouTask): The task containing source URI and params.
+
+        Returns:
+            Any: The raw data retrieved (e.g., bytes, str, list).
+
+        Raises:
+            Exception: Raise any exception if retrieval fails.
+                        The parent `fetch` method will catch and wrap it.
+        """
+        raise NotImplementedError