sayou-refinery 0.1.6__py3-none-any.whl

sayou/refinery/core/exceptions.py

@@ -0,0 +1,27 @@
+from sayou.core.exceptions import SayouCoreError
+
+
+class RefineryError(SayouCoreError):
+    """
+    Base exception for all errors within the sayou-refinery toolkit.
+    """
+
+    pass
+
+
+class NormalizationError(RefineryError):
+    """
+    Raised when raw data cannot be converted to ContentBlocks.
+    (e.g., Malformed JSON, Unsupported format)
+    """
+
+    pass
+
+
+class ProcessingError(RefineryError):
+    """
+    Raised when a processor fails to clean or transform blocks.
+    (e.g., PII masking failure, Imputation rule error)
+    """
+
+    pass

sayou/refinery/core/schemas.py

@@ -0,0 +1,27 @@
+from typing import Any, Dict, List, Union
+
+from pydantic import BaseModel, Field
+
+
+class ContentBlock(BaseModel):
+    """
+    Standard unit of content refined from raw data.
+
+    Refinery normalizes raw inputs into a list of these blocks.
+    Processors iterate over these blocks to clean or modify them.
+    """
+
+    type: str = Field(
+        ..., description="Block type (e.g., 'text', 'md', 'record', 'table')"
+    )
+
+    content: Union[str, Dict[str, Any], List[Any]] = Field(
+        ..., description="The actual data payload"
+    )
+
+    metadata: Dict[str, Any] = Field(
+        default_factory=dict, description="Context info (page_num, source_id, etc.)"
+    )
+
+    class Config:
+        arbitrary_types_allowed = True

sayou/refinery/interfaces/base_normalizer.py

@@ -0,0 +1,62 @@
+from abc import abstractmethod
+from typing import Any, List
+
+from sayou.core.base_component import BaseComponent
+from sayou.core.decorators import measure_time
+
+from ..core.exceptions import NormalizationError
+from ..core.schemas import ContentBlock
+
+
+class BaseNormalizer(BaseComponent):
+    """
+    (Tier 1) Abstract base class for converting raw input into ContentBlocks.
+
+    Normalizers are responsible for structural transformation:
+    Raw Data (JSON, HTML, DB Row) -> List[ContentBlock]
+    """
+
+    component_name = "BaseNormalizer"
+    SUPPORTED_TYPES = []
+
+    @measure_time
+    def normalize(self, raw_data: Any) -> List[ContentBlock]:
+        """
+        Execute the normalization process.
+
+        Args:
+            raw_data: The raw input data from Connector or Document.
+
+        Returns:
+            List[ContentBlock]: A list of normalized content blocks.
+
+        Raises:
+            NormalizationError: If transformation fails.
+        """
+        self._log(f"Normalizing data (Type: {type(raw_data).__name__})")
+        try:
+            blocks = self._do_normalize(raw_data)
+            if not isinstance(blocks, list):
+                raise NormalizationError(f"Output must be a list, got {type(blocks)}")
+
+            return blocks
+
+        except Exception as e:
+            wrapped_error = NormalizationError(
+                f"[{self.component_name}] Failed: {str(e)}"
+            )
+            self.logger.error(wrapped_error, exc_info=True)
+            raise wrapped_error
+
+    @abstractmethod
+    def _do_normalize(self, raw_data: Any) -> List[ContentBlock]:
+        """
+        [Abstract Hook] Implement logic to convert specific raw format to ContentBlocks.
+
+        Args:
+            raw_data: The raw input.
+
+        Returns:
+            List[ContentBlock]: The standardized blocks.
+        """
+        raise NotImplementedError

sayou/refinery/interfaces/base_processor.py

@@ -0,0 +1,57 @@
+from abc import abstractmethod
+from typing import List
+
+from sayou.core.base_component import BaseComponent
+from sayou.core.decorators import measure_time
+
+from ..core.exceptions import ProcessingError
+from ..core.schemas import ContentBlock
+
+
+class BaseProcessor(BaseComponent):
+    """
+    (Tier 1) Abstract base class for processing/cleaning ContentBlocks.
+
+    Processors operate on data that is already normalized. They can modify content
+    (e.g., PII masking, Imputation) or filter out blocks (e.g., Deduplication).
+    """
+
+    component_name = "BaseProcessor"
+
+    @measure_time
+    def process(self, blocks: List[ContentBlock]) -> List[ContentBlock]:
+        """
+        Execute the processing logic on a list of blocks.
+
+        Args:
+            blocks: Input list of ContentBlocks.
+
+        Returns:
+            List[ContentBlock]: Processed list of ContentBlocks.
+
+        Raises:
+            ProcessingError: If processing logic fails.
+        """
+        try:
+            if not blocks:
+                return []
+
+            return self._do_process(blocks)
+
+        except Exception as e:
+            wrapped_error = ProcessingError(f"[{self.component_name}] Failed: {str(e)}")
+            self.logger.error(wrapped_error, exc_info=True)
+            raise wrapped_error
+
+    @abstractmethod
+    def _do_process(self, blocks: List[ContentBlock]) -> List[ContentBlock]:
+        """
+        [Abstract Hook] Implement cleaning/filtering logic.
+
+        Args:
+            blocks: List of input ContentBlocks.
+
+        Returns:
+            List[ContentBlock]: Modified list of ContentBlocks.
+        """
+        raise NotImplementedError

sayou/refinery/normalizer/doc_markdown_normalizer.py

@@ -0,0 +1,307 @@
+from typing import Any, Dict, List
+
+from ..core.exceptions import NormalizationError
+from ..core.schemas import ContentBlock
+from ..interfaces.base_normalizer import BaseNormalizer
+
+
+class DocMarkdownNormalizer(BaseNormalizer):
+    """
+    (Tier 2) Normalizes a Sayou Document Dictionary into Markdown ContentBlocks.
+
+    This engine parses the structured dictionary output from 'sayou-document' and
+    converts individual elements (Text, Table, Image, Chart) into semantically
+    rich Markdown blocks. It also handles metadata conversion to Frontmatter.
+    """
+
+    component_name = "DocMarkdownNormalizer"
+    SUPPORTED_TYPES = ["standard_doc", "sayou_doc_json"]
+
+    def initialize(
+        self,
+        include_headers: bool = True,
+        include_footers: bool = False,
+        **kwargs,
+    ):
+        """
+        Configure the normalizer's behavior regarding document structure.
+
+        Args:
+            include_headers (bool): If True, processes elements found in page headers.
+            include_footers (bool): If True, processes elements found in page footers.
+            **kwargs: Additional configuration parameters passed to parent.
+        """
+        super().initialize(**kwargs)
+        self.include_headers = include_headers
+        self.include_footers = include_footers
+
+    def _do_normalize(self, raw_data: Any) -> List[ContentBlock]:
+        """
+        Execute the normalization logic on the document dictionary.
+
+        Args:
+            raw_data (Any): The input dictionary adhering to Sayou Document Schema.
+
+        Returns:
+            List[ContentBlock]: A list of normalized content blocks (mostly 'md' type).
+
+        Raises:
+            NormalizationError: If `raw_data` is not a valid dictionary.
+        """
+        if not isinstance(raw_data, dict):
+            raise NormalizationError(
+                f"Input must be a Dictionary, got {type(raw_data).__name__}"
+            )
+
+        doc_data = raw_data
+        blocks: List[ContentBlock] = []
+
+        if "metadata" in doc_data and doc_data["metadata"]:
+            blocks.extend(self._handle_doc_metadata(doc_data))
+
+        for page in doc_data.get("pages", []):
+            if self.include_headers and "header_elements" in page:
+                for element in page.get("header_elements", []):
+                    blocks.extend(
+                        self._handle_element(element, is_header=True, is_footer=False)
+                    )
+
+            for element in page.get("elements", []):
+                blocks.extend(
+                    self._handle_element(element, is_header=False, is_footer=False)
+                )
+
+            if self.include_footers and "footer_elements" in page:
+                for element in page.get("footer_elements", []):
+                    # T2의 기본 규칙: include_footers가 True여도 _handle_element에서
+                    # is_footer=True 플래그를 보고 무시할 수 있음 (T3가 오버라이드 가능)
+                    blocks.extend(
+                        self._handle_element(element, is_header=False, is_footer=True)
+                    )
+
+        return blocks
+
+    def _handle_element(
+        self, element: Dict[str, Any], is_header: bool, is_footer: bool
+    ) -> List[ContentBlock]:
+        """
+        Dispatch the element to specific handlers based on its 'type' field.
+
+        Args:
+            element (Dict[str, Any]): The element dictionary.
+            is_header (bool): True if the element is part of the page header.
+            is_footer (bool): True if the element is part of the page footer.
+
+        Returns:
+            List[ContentBlock]: The resulting block(s) from the element.
+        """
+        if is_footer and not self.include_footers:
+            return []
+
+        elem_type = element.get("type")
+
+        if elem_type == "text":
+            return self._handle_text(element, is_header, is_footer)
+
+        if elem_type == "table":
+            return self._handle_table(element, is_header, is_footer)
+
+        if elem_type == "image":
+            return self._handle_image(element, is_header, is_footer)
+
+        if elem_type == "chart":
+            return self._handle_chart(element, is_header, is_footer)
+
+        return []
+
+    def _handle_doc_metadata(self, doc_data: Dict[str, Any]) -> List[ContentBlock]:
+        """
+        Convert document-level metadata into a Markdown Frontmatter block.
+
+        Args:
+            doc_data (Dict[str, Any]): The root document dictionary containing 'metadata'.
+
+        Returns:
+            List[ContentBlock]: A single block containing YAML-like frontmatter.
+        """
+        md_frontmatter = "---\n"
+        metadata = doc_data.get("metadata", {})
+
+        title = metadata.get("title")
+        author = metadata.get("author")
+
+        if title:
+            md_frontmatter += f"title: {title}\n"
+        if author:
+            md_frontmatter += f"author: {author}\n"
+        md_frontmatter += "---\n\n"
+
+        return [
+            ContentBlock(
+                type="md",
+                content=md_frontmatter,
+                metadata={"page_num": 0, "id": "metadata", "is_footer": False},
+            )
+        ]
+
+    def _handle_text(
+        self, element: Dict[str, Any], is_header: bool, is_footer: bool
+    ) -> List[ContentBlock]:
+        """
+        Convert a text element to a Markdown block, handling headings and lists.
+
+        Uses 'semantic_type' (heading/list) and 'level' attributes to generate
+        appropriate Markdown syntax (e.g., '# Title', '- Item').
+        """
+        text = element.get("text", "").strip()
+        if not text:
+            return []
+
+        raw_attrs = element.get("raw_attributes", {})
+        semantic_type = raw_attrs.get("semantic_type")
+
+        content = None
+
+        # 1. '●' (List) 처리
+        if semantic_type == "list":
+            level = raw_attrs.get("list_level", 0)
+            indent = "  " * level
+            content = f"{indent}- {text}"
+
+        # 2. 'Heading 1-9' (제목) 처리
+        elif semantic_type == "heading":
+            level = raw_attrs.get("heading_level", 1)
+            hashes = "#" * level
+            content = f"{hashes} {text}"
+
+        # 3. PPT 플레이스홀더 (레거시 호환)
+        elif raw_attrs.get("placeholder_type") == "TITLE":
+            content = f"# {text}"
+
+        # 4. 그 외 (기본 텍스트)
+        else:
+            content = text
+
+        return [
+            ContentBlock(
+                type="md",
+                content=content,
+                metadata={
+                    "page_num": element.get("meta", {}).get("page_num"),
+                    "id": element.get("id"),
+                    "style": raw_attrs.get("style"),
+                    "is_footer": is_footer,
+                },
+            )
+        ]
+
+    def _handle_table(
+        self, element: Dict[str, Any], is_header: bool, is_footer: bool
+    ) -> List[ContentBlock]:
+        """
+        Convert a table element into a Markdown table representation.
+
+        Args:
+            element (Dict[str, Any]): Must contain 'data' (2D list).
+        """
+        md_table = ""
+        table_data = element.get("data", [])
+
+        if not table_data:
+            return []
+
+        max_cols = 0
+        for row in table_data:
+            if row:
+                max_cols = max(max_cols, len(row))
+
+        if max_cols == 0:
+            return []
+
+        # 1. 헤더 행 (첫 번째 행)
+        header = table_data[0]
+        header_cells = list(map(str, header)) + [""] * (max_cols - len(header))
+        md_table += "| " + " | ".join(header_cells) + " |\n"
+
+        # 2. 구분자 행 (최대 열 개수 기준)
+        md_table += "| " + " | ".join(["---"] * max_cols) + " |\n"
+
+        # 3. 본문 행 (두 번째 행부터)
+        for row in table_data[1:]:
+            body_cells = list(map(str, row)) + [""] * (max_cols - len(row))
+            md_table += "| " + " | ".join(body_cells) + " |\n"
+
+        return [
+            ContentBlock(
+                type="md",
+                content=md_table.strip(),
+                metadata={
+                    "page_num": element.get("meta", {}).get("page_num"),
+                    "id": element.get("id"),
+                    "is_footer": is_footer,
+                },
+            )
+        ]
+
+    def _handle_image(
+        self, element: Dict[str, Any], is_header: bool, is_footer: bool
+    ) -> List[ContentBlock]:
+        """
+        Process an image element.
+
+        Depending on implementation, this might return an 'image_base64' block
+        or a Markdown image link if an external URL is provided.
+        """
+        image_base64 = element.get("image_base64")
+        image_base64 = element.get("image_base64")
+        if not image_base64:
+            return []
+
+        ocr_text = (element.get("ocr_text") or "").strip()
+        if not ocr_text:
+            alt_text = "image"
+        else:
+            alt_text = ocr_text
+
+        img_format = element.get("image_format", "png")
+
+        return [
+            ContentBlock(
+                type="image_base64",
+                content=image_base64,
+                metadata={
+                    "page_num": element.get("meta", {}).get("page_num"),
+                    "id": element.get("id"),
+                    "is_footer": is_footer,
+                    "alt_text": alt_text,
+                    "format": img_format,
+                },
+            )
+        ]
+
+    def _handle_chart(
+        self, element: Dict[str, Any], is_header: bool, is_footer: bool
+    ) -> List[ContentBlock]:
+        """
+        Convert a chart element into its text representation.
+
+        Uses the 'text_representation' field from the element to create
+        a descriptive text block for LLM consumption.
+        """
+        text_rep = element.get("text_representation")
+        if not text_rep:
+            return []
+
+        content = f"--- Chart Data ---\n{text_rep}\n--------------------\n"
+
+        return [
+            ContentBlock(
+                type="md",
+                content=content,
+                metadata={
+                    "page_num": element.get("meta", {}).get("page_num"),
+                    "id": element.get("id"),
+                    "is_footer": is_footer,
+                },
+            )
+        ]

sayou/refinery/normalizer/html_text_normalizer.py

@@ -0,0 +1,59 @@
+try:
+    from bs4 import BeautifulSoup
+except ImportError:
+    BeautifulSoup = None
+
+from typing import Any, List
+
+from ..core.exceptions import NormalizationError
+from ..core.schemas import ContentBlock
+from ..interfaces.base_normalizer import BaseNormalizer
+
+
+class HtmlTextNormalizer(BaseNormalizer):
+    """
+    (Tier 2) Converts HTML string into a clean Text ContentBlock.
+
+    Uses BeautifulSoup to strip tags, scripts, and styles, returning only
+    the visible text content while preserving paragraph structure.
+    """
+
+    component_name = "HtmlTextNormalizer"
+    SUPPORTED_TYPES = ["html"]
+
+    def _do_normalize(self, raw_data: Any) -> List[ContentBlock]:
+        """
+        Parse HTML and extract text.
+
+        Args:
+            raw_data (Any): The input HTML string.
+
+        Returns:
+            List[ContentBlock]: A single block of type 'text'.
+
+        Raises:
+            ImportError: If BeautifulSoup4 is not installed.
+            NormalizationError: If input is not a string.
+        """
+        if not BeautifulSoup:
+            raise ImportError("BeautifulSoup4 is required for HtmlTextNormalizer.")
+
+        if not isinstance(raw_data, str):
+            raise NormalizationError(
+                f"Input must be HTML string, got {type(raw_data)}."
+            )
+
+        soup = BeautifulSoup(raw_data, "html.parser")
+
+        for tag in soup(["script", "style", "noscript", "iframe"]):
+            tag.extract()
+
+        text = soup.get_text(separator="\n")
+
+        import re
+
+        text = re.sub(r"\n{3,}", "\n\n", text).strip()
+
+        return [
+            ContentBlock(type="text", content=text, metadata={"source_type": "html"})
+        ]

sayou/refinery/normalizer/record_normalizer.py

@@ -0,0 +1,66 @@
+from typing import Any, Dict, List
+
+from ..core.exceptions import NormalizationError
+from ..core.schemas import ContentBlock
+from ..interfaces.base_normalizer import BaseNormalizer
+
+
+class RecordNormalizer(BaseNormalizer):
+    """
+    (Tier 2) Converts structured data (Dict/List) into 'record' ContentBlocks.
+
+    Suitable for processing database rows, CSV records, or JSON API responses.
+    Each dictionary becomes a separate ContentBlock of type 'record'.
+    """
+
+    component_name = "RecordNormalizer"
+    SUPPORTED_TYPES = ["json", "dict", "db_row", "record"]
+
+    def _do_normalize(self, raw_data: Any) -> List[ContentBlock]:
+        """
+        Convert dict or list of dicts into record blocks.
+
+        Args:
+            raw_data (Any): A Dictionary or a List of Dictionaries.
+
+        Returns:
+            List[ContentBlock]: Blocks of type 'record'.
+        """
+        blocks = []
+
+        # Case 1: Single Dictionary
+        if isinstance(raw_data, dict):
+            blocks.append(self._create_block(raw_data))
+
+        # Case 2: List of Dictionaries
+        elif isinstance(raw_data, list):
+            for item in raw_data:
+                if isinstance(item, dict):
+                    blocks.append(self._create_block(item))
+                else:
+                    self._log(
+                        f"Skipping non-dict item in list: {type(item)}", level="warning"
+                    )
+
+        else:
+            raise NormalizationError(
+                f"Input must be Dict or List[Dict], got {type(raw_data)}"
+            )
+
+        return blocks
+
+    def _create_block(self, data: Dict[str, Any]) -> ContentBlock:
+        """
+        Helper to wrap a single dictionary into a ContentBlock.
+
+        Args:
+            data (Dict[str, Any]): The data record.
+
+        Returns:
+            ContentBlock: A block with type='record' and content=data.
+        """
+        return ContentBlock(
+            type="record",
+            content=data,
+            metadata={"fields": list(data.keys())},
+        )

sayou/refinery/pipeline.py

@@ -0,0 +1,111 @@
+from typing import Any, Dict, List, Optional
+
+from sayou.core.base_component import BaseComponent
+from sayou.core.decorators import safe_run
+
+from .core.exceptions import RefineryError
+from .core.schemas import ContentBlock
+from .interfaces.base_normalizer import BaseNormalizer
+from .interfaces.base_processor import BaseProcessor
+from .normalizer.doc_markdown_normalizer import DocMarkdownNormalizer
+from .normalizer.html_text_normalizer import HtmlTextNormalizer
+from .normalizer.record_normalizer import RecordNormalizer
+from .processor.deduplicator import Deduplicator
+from .processor.imputer import Imputer
+from .processor.outlier_handler import OutlierHandler
+from .processor.pii_masker import PiiMasker
+from .processor.text_cleaner import TextCleaner
+
+
+class RefineryPipeline(BaseComponent):
+    """
+    Orchestrates the data refinement process.
+    1. Selects a Normalizer to convert raw data into standard ContentBlocks.
+    2. Runs a chain of Processors to clean and transform the blocks.
+    """
+
+    component_name = "RefineryPipeline"
+
+    def __init__(
+        self,
+        extra_normalizers: Optional[List[BaseNormalizer]] = None,
+        processors: Optional[List[BaseProcessor]] = None,
+    ):
+        super().__init__()
+        self.normalizers: Dict[str, BaseNormalizer] = {}
+
+        # 1. Register Default Normalizers
+        defaults = [DocMarkdownNormalizer(), HtmlTextNormalizer(), RecordNormalizer()]
+        self._register(defaults)
+
+        # 2. Register User Extras
+        if extra_normalizers:
+            self._register(extra_normalizers)
+
+        # 3. Setup Processors Chain
+        self.processors = (
+            processors
+            if processors is not None
+            else [
+                TextCleaner(),
+                PiiMasker(),
+                Deduplicator(),
+                Imputer(),
+                OutlierHandler(),
+            ]
+        )
+
+    def _register(self, comps: List[BaseNormalizer]):
+        for c in comps:
+            for t in getattr(c, "SUPPORTED_TYPES", []):
+                self.normalizers[t] = c
+
+    @safe_run(default_return=None)
+    def initialize(self, **kwargs):
+        """
+        Initialize all sub-components (Normalizers and Processors).
+        Passes global configuration (like PII masking rules) down to components.
+        """
+        for norm in set(self.normalizers.values()):
+            norm.initialize(**kwargs)
+
+        for proc in self.processors:
+            proc.initialize(**kwargs)
+
+        self._log(
+            f"Refinery initialized with {len(self.processors)} processors in chain."
+        )
+
+    def run(
+        self, raw_data: Any, source_type: str = "standard_doc"
+    ) -> List[ContentBlock]:
+        """
+        Execute the refinement pipeline.
+
+        Args:
+            raw_data: The raw input data (dict, html string, db row list, etc.)
+            source_type: The type of input data (e.g., 'standard_doc', 'html', 'json')
+
+        Returns:
+            List[ContentBlock]: A list of clean, normalized blocks.
+        """
+        # Step 1: Normalize (Structure Transformation)
+        normalizer = self.normalizers.get(source_type)
+        if not normalizer:
+            supported = list(self.normalizers.keys())
+            raise RefineryError(
+                f"Unknown source_type '{source_type}'. Supported: {supported}"
+            )
+
+        try:
+            blocks = normalizer.normalize(raw_data)
+        except Exception as e:
+            self.logger.error(f"Normalization step failed: {e}")
+            return []
+
+        # Step 2: Process (Content Cleaning)
+        # Processors modify blocks in-place or return new lists
+        for processor in self.processors:
+            blocks = processor.process(blocks)
+
+        return blocks

sayou/refinery/processor/deduplicator.py

@@ -0,0 +1,48 @@
+import json
+from typing import List, Set
+
+from ..core.schemas import ContentBlock
+from ..interfaces.base_processor import BaseProcessor
+
+
+class Deduplicator(BaseProcessor):
+    """
+    (Tier 2) Removes duplicate blocks based on content hashing.
+
+    It computes a hash of the content for each block and filters out
+    subsequent blocks that match an already seen hash.
+    """
+
+    component_name = "Deduplicator"
+
+    def _do_process(self, blocks: List[ContentBlock]) -> List[ContentBlock]:
+        """
+        Iterate through blocks and remove duplicates.
+
+        Args:
+            blocks (List[ContentBlock]): The input list of blocks.
+
+        Returns:
+            List[ContentBlock]: A new list with duplicates removed.
+        """
+        seen_hashes: Set[int] = set()
+        unique_blocks: List[ContentBlock] = []
+
+        for block in blocks:
+            # Generate stable hash key
+            if isinstance(block.content, dict):
+                content_str = json.dumps(block.content, sort_keys=True)
+            else:
+                content_str = str(block.content)
+
+            if len(content_str) < 5:
+                unique_blocks.append(block)
+                continue
+
+            content_hash = hash(content_str)
+
+            if content_hash not in seen_hashes:
+                seen_hashes.add(content_hash)
+                unique_blocks.append(block)
+
+        return unique_blocks

sayou/refinery/processor/imputer.py

@@ -0,0 +1,51 @@
+from typing import Any, Dict, List
+
+from ..core.schemas import ContentBlock
+from ..interfaces.base_processor import BaseProcessor
+
+
+class Imputer(BaseProcessor):
+    """
+    (Tier 2) Fills missing values in 'record' type blocks using defined rules.
+
+    Only operates on blocks with type='record' where the content is a dictionary.
+    """
+
+    component_name = "Imputer"
+
+    def initialize(self, imputation_rules: Dict[str, Any] = None, **kwargs):
+        """
+        Set imputation rules.
+
+        Args:
+            rules (Dict[str, Any]): Mapping of field names to default values.
+                                    Example: {"category": "Unknown", "price": 0.0}
+            **kwargs: Additional arguments.
+        """
+        self.rules = imputation_rules or {}
+        if not self.rules:
+            self._log("Imputer initialized with no rules.", level="warning")
+
+    def _do_process(self, blocks: List[ContentBlock]) -> List[ContentBlock]:
+        """
+        Apply imputation rules to record blocks.
+
+        Args:
+            blocks (List[ContentBlock]): Input blocks.
+
+        Returns:
+            List[ContentBlock]: Blocks with missing values filled.
+        """
+        for block in blocks:
+            if block.type != "record" or not isinstance(block.content, dict):
+                continue
+
+            record = block.content
+
+            for field, default_value in self.rules.items():
+                if record.get(field) is None:
+                    record[field] = default_value
+
+            block.content = record
+
+        return blocks

sayou/refinery/processor/outlier_handler.py

@@ -0,0 +1,84 @@
+from typing import Any, Dict, List
+
+from ..core.schemas import ContentBlock
+from ..interfaces.base_processor import BaseProcessor
+
+
+class OutlierHandler(BaseProcessor):
+    """
+    (Tier 2) Handles numerical outliers in 'record' blocks.
+
+    Can either 'drop' the entire block or 'clamp' the value to a boundary
+    if a field violates the defined min/max rules.
+    """
+
+    component_name = "OutlierHandler"
+
+    def initialize(self, outlier_rules: Dict[str, Dict[str, Any]] = None, **kwargs):
+        """
+        Set outlier handling rules.
+
+        Args:
+            rules (Dict[str, Dict[str, Any]]): Mapping of field names to constraints.
+                Example:
+                {
+                    "age": {"min": 0, "max": 120, "action": "drop"},
+                    "score": {"min": 0, "max": 100, "action": "clamp"}
+                }
+            **kwargs: Additional arguments.
+        """
+        self.rules = outlier_rules or {}
+
+    def _do_process(self, blocks: List[ContentBlock]) -> List[ContentBlock]:
+        """
+        Check numerical fields against rules and filter/modify blocks.
+
+        Args:
+            blocks (List[ContentBlock]): Input blocks.
+
+        Returns:
+            List[ContentBlock]: Filtered or modified list of blocks.
+        """
+        valid_blocks = []
+
+        for block in blocks:
+            if block.type != "record" or not isinstance(block.content, dict):
+                valid_blocks.append(block)
+                continue
+
+            record = block.content
+            should_drop = False
+
+            for field, rule in self.rules.items():
+                val = record.get(field)
+                if val is None:
+                    continue
+
+                try:
+                    val_f = float(val)
+                    min_v = rule.get("min")
+                    max_v = rule.get("max")
+                    action = rule.get("action", "drop")
+
+                    if min_v is not None and val_f < min_v:
+                        if action == "drop":
+                            should_drop = True
+                            break
+                        elif action == "clamp":
+                            record[field] = min_v
+
+                    if max_v is not None and val_f > max_v:
+                        if action == "drop":
+                            should_drop = True
+                            break
+                        elif action == "clamp":
+                            record[field] = max_v
+
+                except (ValueError, TypeError):
+                    continue
+
+            if not should_drop:
+                block.content = record
+                valid_blocks.append(block)
+
+        return valid_blocks

sayou/refinery/processor/pii_masker.py

@@ -0,0 +1,52 @@
+import re
+from typing import List
+
+from ..core.schemas import ContentBlock
+from ..interfaces.base_processor import BaseProcessor
+
+
+class PiiMasker(BaseProcessor):
+    """
+    (Tier 2) Masks Personally Identifiable Information (PII) in text blocks.
+
+    Uses Regex patterns to identify and redact sensitive data like emails
+    and phone numbers in 'text' and 'md' blocks.
+    """
+
+    component_name = "PiiMasker"
+
+    def initialize(self, mask_email: bool = True, mask_phone: bool = True, **kwargs):
+        """
+        Configure masking targets.
+
+        Args:
+            mask_email (bool): Whether to mask email addresses (default: True).
+            mask_phone (bool): Whether to mask phone numbers (default: True).
+            **kwargs: Additional arguments.
+        """
+        self.mask_email = mask_email
+        self.mask_phone = mask_phone
+        self._email_re = re.compile(r"[\w\.-]+@[\w\.-]+")
+        # Simple phone regex (customizable)
+        self._phone_re = re.compile(r"\d{3}[-\.\s]??\d{3,4}[-\.\s]??\d{4}")
+
+    def _do_process(self, blocks: List[ContentBlock]) -> List[ContentBlock]:
+        """
+        Apply masking regex to text content.
+
+        Args:
+            blocks (List[ContentBlock]): Input blocks.
+
+        Returns:
+            List[ContentBlock]: Blocks with sensitive info replaced by tokens.
+        """
+        for block in blocks:
+            if block.type not in ["text", "md"] or not isinstance(block.content, str):
+                continue
+
+            if self.mask_email:
+                block.content = self._email_re.sub("[EMAIL]", block.content)
+            if self.mask_phone:
+                block.content = self._phone_re.sub("[PHONE]", block.content)
+
+        return blocks

sayou/refinery/processor/text_cleaner.py

@@ -0,0 +1,61 @@
+import re
+from typing import List
+
+from ..core.schemas import ContentBlock
+from ..interfaces.base_processor import BaseProcessor
+
+
+class TextCleaner(BaseProcessor):
+    """
+    (Tier 2) Cleans text content using regex and whitespace normalization.
+
+    Operates on 'text' and 'md' blocks to remove noise characters or custom patterns.
+    """
+
+    component_name = "TextCleaner"
+
+    def initialize(
+        self, patterns: List[str] = None, normalize_space: bool = True, **kwargs
+    ):
+        """
+        Configure cleaning patterns.
+
+        Args:
+            patterns (List[str]): List of regex patterns to remove from text.
+            normalize_space (bool): If True, collapses multiple spaces/tabs/newlines.
+            **kwargs: Additional arguments.
+        """
+        self.normalize_space = normalize_space
+        self.patterns = [re.compile(p) for p in (patterns or [])]
+        self._space_re = re.compile(r"[ \t]+")
+
+    def _do_process(self, blocks: List[ContentBlock]) -> List[ContentBlock]:
+        """
+        Apply cleaning logic to text blocks.
+
+        Args:
+            blocks (List[ContentBlock]): Input blocks.
+
+        Returns:
+            List[ContentBlock]: Cleaned blocks.
+        """
+        for block in blocks:
+            if block.type not in ["text", "md"]:
+                continue
+
+            if not isinstance(block.content, str):
+                continue
+
+            text = block.content
+
+            # 1. Custom Patterns Removal
+            for pat in self.patterns:
+                text = pat.sub("", text)
+
+            # 2. Whitespace Normalization
+            if self.normalize_space:
+                text = self._space_re.sub(" ", text)
+
+            block.content = text.strip()
+
+        return blocks

sayou_refinery-0.1.6.dist-info/METADATA

@@ -0,0 +1,310 @@
+Metadata-Version: 2.4
+Name: sayou-refinery
+Version: 0.1.6
+Summary: Refinery components for the Sayou Data Platform
+Project-URL: Homepage, https://www.sayouzone.com/
+Project-URL: Documentation, https://sayouzone.github.io/sayou-fabric/
+Project-URL: Repository, https://github.com/sayouzone/sayou-fabric
+Author-email: Sayouzone <contact@sayouzone.com>
+License:                                  Apache License
+                                   Version 2.0, January 2004
+                                http://www.apache.org/licenses/
+        
+           TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+        
+           1. Definitions.
+        
+              "License" shall mean the terms and conditions for use, reproduction,
+              and distribution as defined by Sections 1 through 9 of this document.
+        
+              "Licensor" shall mean the copyright owner or entity authorized by
+              the copyright owner that is granting the License.
+        
+              "Legal Entity" shall mean the union of the acting entity and all
+              other entities that control, are controlled by, or are under common
+              control with that entity. For the purposes of this definition,
+              "control" means (i) the power, direct or indirect, to cause the
+              direction or management of such entity, whether by contract or
+              otherwise, or (ii) ownership of fifty percent (50%) or more of the
+              outstanding shares, or (iii) beneficial ownership of such entity.
+        
+              "You" (or "Your") shall mean an individual or Legal Entity
+              exercising permissions granted by this License.
+        
+              "Source" form shall mean the preferred form for making modifications,
+              including but not limited to software source code, documentation
+              source, and configuration files.
+        
+              "Object" form shall mean any form resulting from mechanical
+              transformation or translation of a Source form, including but
+              not limited to compiled object code, generated documentation,
+              and conversions to other media types.
+        
+              "Work" shall mean the work of authorship, whether in Source or
+              Object form, made available under the License, as indicated by a
+              copyright notice that is included in or attached to the work
+              (an example is provided in the Appendix below).
+        
+              "Derivative Works" shall mean any work, whether in Source or Object
+              form, that is based on (or derived from) the Work and for which the
+              editorial revisions, annotations, elaborations, or other modifications
+              represent, as a whole, an original work of authorship. For the purposes
+              of this License, Derivative Works shall not include works that remain
+              separable from, or merely link (or bind by name) to the interfaces of,
+              the Work and Derivative Works thereof.
+        
+              "Contribution" shall mean any work of authorship, including
+              the original version of the Work and any modifications or additions
+              to that Work or Derivative Works thereof, that is intentionally
+              submitted to Licensor for inclusion in the Work by the copyright owner
+              or by an individual or Legal Entity authorized to submit on behalf of
+              the copyright owner. For the purposes of this definition, "submitted"
+              means any form of electronic, verbal, or written communication sent
+              to the Licensor or its representatives, including but not limited to
+              communication on electronic mailing lists, source code control systems,
+              and issue tracking systems that are managed by, or on behalf of, the
+              Licensor for the purpose of discussing and improving the Work, but
+              excluding communication that is conspicuously marked or otherwise
+              designated in writing by the copyright owner as "Not a Contribution."
+        
+              "Contributor" shall mean Licensor and any individual or Legal Entity
+              on behalf of whom a Contribution has been received by Licensor and
+              subsequently incorporated within the Work.
+        
+           2. Grant of Copyright License. Subject to the terms and conditions of
+              this License, each Contributor hereby grants to You a perpetual,
+              worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+              copyright license to reproduce, prepare Derivative Works of,
+              publicly display, publicly perform, sublicense, and distribute the
+              Work and such Derivative Works in Source or Object form.
+        
+           3. Grant of Patent License. Subject to the terms and conditions of
+              this License, each Contributor hereby grants to You a perpetual,
+              worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+              (except as stated in this section) patent license to make, have made,
+              use, offer to sell, sell, import, and otherwise transfer the Work,
+              where such license applies only to those patent claims licensable
+              by such Contributor that are necessarily infringed by their
+              Contribution(s) alone or by combination of their Contribution(s)
+              with the Work to which such Contribution(s) was submitted. If You
+              institute patent litigation against any entity (including a
+              cross-claim or counterclaim in a lawsuit) alleging that the Work
+              or a Contribution incorporated within the Work constitutes direct
+              or contributory patent infringement, then any patent licenses
+              granted to You under this License for that Work shall terminate
+              as of the date such litigation is filed.
+        
+           4. Redistribution. You may reproduce and distribute copies of the
+              Work or Derivative Works thereof in any medium, with or without
+              modifications, and in Source or Object form, provided that You
+              meet the following conditions:
+        
+              (a) You must give any other recipients of the Work or
+                  Derivative Works a copy of this License; and
+        
+              (b) You must cause any modified files to carry prominent notices
+                  stating that You changed the files; and
+        
+              (c) You must retain, in the Source form of any Derivative Works
+                  that You distribute, all copyright, patent, trademark, and
+                  attribution notices from the Source form of the Work,
+                  excluding those notices that do not pertain to any part of
+                  the Derivative Works; and
+        
+              (d) If the Work includes a "NOTICE" text file as part of its
+                  distribution, then any Derivative Works that You distribute must
+                  include a readable copy of the attribution notices contained
+                  within such NOTICE file, excluding those notices that do not
+                  pertain to any part of the Derivative Works, in at least one
+                  of the following places: within a NOTICE text file distributed
+                  as part of the Derivative Works; within the Source form or
+                  documentation, if provided along with the Derivative Works; or,
+                  within a display generated by the Derivative Works, if and
+                  wherever such third-party notices normally appear. The contents
+                  of the NOTICE file are for informational purposes only and
+                  do not modify the License. You may add Your own attribution
+                  notices within Derivative Works that You distribute, alongside
+                  or as an addendum to the NOTICE text from the Work, provided
+                  that such additional attribution notices cannot be construed
+                  as modifying the License.
+        
+              You may add Your own copyright statement to Your modifications and
+              may provide additional or different license terms and conditions
+              for use, reproduction, or distribution of Your modifications, or
+              for any such Derivative Works as a whole, provided Your use,
+              reproduction, and distribution of the Work otherwise complies with
+              the conditions stated in this License.
+        
+           5. Submission of Contributions. Unless You explicitly state otherwise,
+              any Contribution intentionally submitted for inclusion in the Work
+              by You to the Licensor shall be under the terms and conditions of
+              this License, without any additional terms or conditions.
+              Notwithstanding the above, nothing herein shall supersede or modify
+              the terms of any separate license agreement you may have executed
+              with Licensor regarding such Contributions.
+        
+           6. Trademarks. This License does not grant permission to use the trade
+              names, trademarks, service marks, or product names of the Licensor,
+              except as required for reasonable and customary use in describing the
+              origin of the Work and reproducing the content of the NOTICE file.
+        
+           7. Disclaimer of Warranty. Unless required by applicable law or
+              agreed to in writing, Licensor provides the Work (and each
+              Contributor provides its Contributions) on an "AS IS" BASIS,
+              WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+              implied, including, without limitation, any warranties or conditions
+              of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+              PARTICULAR PURPOSE. You are solely responsible for determining the
+              appropriateness of using or redistributing the Work and assume any
+              risks associated with Your exercise of permissions under this License.
+        
+           8. Limitation of Liability. In no event and under no legal theory,
+              whether in tort (including negligence), contract, or otherwise,
+              unless required by applicable law (such as deliberate and grossly
+              negligent acts) or agreed to in writing, shall any Contributor be
+              liable to You for damages, including any direct, indirect, special,
+              incidental, or consequential damages of any character arising as a
+              result of this License or out of the use or inability to use the
+              Work (including but not limited to damages for loss of goodwill,
+              work stoppage, computer failure or malfunction, or any and all
+              other commercial damages or losses), even if such Contributor
+              has been advised of the possibility of such damages.
+        
+           9. Accepting Warranty or Additional Liability. While redistributing
+              the Work or Derivative Works thereof, You may choose to offer,
+              and charge a fee for, acceptance of support, warranty, indemnity,
+              or other liability obligations and/or rights consistent with this
+              License. However, in accepting such obligations, You may act only
+              on Your own behalf and on Your sole responsibility, not on behalf
+              of any other Contributor, and only if You agree to indemnify,
+              defend, and hold each Contributor harmless for any liability
+              incurred by, or claims asserted against, such Contributor by reason
+              of your accepting any such warranty or additional liability.
+        
+           END OF TERMS AND CONDITIONS
+        
+           APPENDIX: How to apply the Apache License to your work.
+        
+              To apply the Apache License to your work, attach the following
+              boilerplate notice, with the fields enclosed by brackets "[]"
+              replaced with your own identifying information. (Don't include
+              the brackets!)  The text should be enclosed in the appropriate
+              comment syntax for the file format. We also recommend that a
+              file or class name and description of purpose be included on the
+              same "printed page" as the copyright notice for easier
+              identification within third-party archives.
+        
+           Copyright [yyyy] [name of copyright owner]
+        
+           Licensed under the Apache License, Version 2.0 (the "License");
+           you may not use this file except in compliance with the License.
+           You may obtain a copy of the License at
+        
+               http://www.apache.org/licenses/LICENSE-2.0
+        
+           Unless required by applicable law or agreed to in writing, software
+           distributed under the License is distributed on an "AS IS" BASIS,
+           WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+           See the License for the specific language governing permissions and
+           limitations under the License.
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
+Requires-Python: >=3.9
+Requires-Dist: sayou-core~=0.1.2
+Description-Content-Type: text/markdown
+
+# sayou-refinery
+
+[![PyPI version](https://img.shields.io/pypi/v/sayou-refinery.svg?color=blue)](https://pypi.org/project/sayou-refinery/)
+[![License](https://img.shields.io/badge/License-Apache%202.0-red.svg)](https://www.apache.org/licenses/LICENSE-2.0)
+[![Docs](https://img.shields.io/badge/docs-mkdocs-success.svg?logo=materialformkdocs)](https://sayouzone.github.io/sayou-fabric/library-guides/refinery/)
+
+**The Universal Data Cleaning & Normalization Engine for Sayou Fabric.**
+
+`sayou-refinery` acts as the "Cleaning Plant" in your data pipeline.
+
+It transforms heterogeneous raw data (JSON Documents, HTML, DB Records) into a standardized stream of **ContentBlocks**, ensuring that downstream components (like Chunkers or LLMs) receive clean, uniform data regardless of the original source format.
+
+## 💡 Core Philosophy
+
+**"Flatten Structure, Polish Content."**
+
+Refinery operates in two distinct stages to guarantee data quality:
+
+1.  **Normalization (Shape Shifting):** Converts complex structures (nested JSON, HTML trees, DB Rows) into a linear list of `ContentBlocks`.
+2.  **Processing (Cleaning):** Applies a chain of cleaning agents (Regex, Masking, Deduplication) to improve data hygiene.
+
+## 📦 Installation
+
+```bash
+pip install sayou-refinery
+```
+
+## ⚡ Quick Start
+
+The `RefineryPipeline` orchestrates the normalization and processing chain.
+
+```python
+from sayou.refinery.pipeline import RefineryPipeline
+
+def run_demo():
+    # 1. Initialize with specific cleaning rules
+    pipeline = RefineryPipeline()
+    pipeline.initialize(
+        mask_email=True,
+        outlier_rules={"price": {"min": 0, "max": 1000, "action": "clamp"}}
+    )
+
+    # 2. Raw Data (e.g., from sayou-document)
+    raw_doc = {
+        "metadata": {"title": "Test Doc"},
+        "pages": [{
+            "elements": [
+                {"type": "text", "text": "Contact: admin@sayou.ai"},
+                {"type": "text", "text": "   Dirty   Whitespace   "}
+            ]
+        }]
+    }
+
+    # 3. Run Pipeline
+    # source_type: 'standard_doc', 'html', 'json', etc.
+    blocks = pipeline.run(raw_doc, source_type="standard_doc")
+
+    # 4. Result
+    for block in blocks:
+        print(f"[{block.type}] {block.content}")
+        
+        # Output:
+        # [md_meta] --- title: Test Doc ...
+        # [md] Contact: [EMAIL]
+        # [md] Dirty Whitespace
+
+if __name__ == "__main__":
+    run_demo()
+```
+
+## 🔑 Key Components
+
+### Normalizers
+* **`DocMarkdownNormalizer`**: Converts Sayou Document Dicts into Markdown blocks.
+* **`HtmlTextNormalizer`**: Strips HTML tags and scripts, extracting clean text.
+* **`RecordNormalizer`**: Converts DB rows or JSON objects into 'record' blocks.
+
+### Processors
+* **`TextCleaner`**: Normalizes whitespace and removes noise via regex.
+* **`PiiMasker`**: Masks sensitive info like emails and phone numbers.
+* **`Deduplicator`**: Removes duplicate content blocks.
+* **`Imputer`**: Fills missing values in record blocks.
+* **`OutlierHandler`**: Filters or clamps numerical outliers in records.
+
+## 🤝 Contributing
+
+We welcome contributions for new Normalizers (e.g., `CsvNormalizer`, `LogNormalizer`) or Processors (e.g., `LangChainFilter`).
+
+## 📜 License
+
+Apache 2.0 License © 2025 Sayouzone

sayou_refinery-0.1.6.dist-info/RECORD

@@ -0,0 +1,16 @@
+sayou/refinery/pipeline.py,sha256=oJbygy300ounS3xL3UdCpwnmdmUTRib-W-ADsPJ1Vjs,3756
+sayou/refinery/core/exceptions.py,sha256=LhY8tDk9UzqjGjy-7UPpzBSRpH4vUl3ZemmW-BssdJY,547
+sayou/refinery/core/schemas.py,sha256=LhKV5X8WIiUV273OpX9y7TduQUX03qZh-rgRSMn_eCs,727
+sayou/refinery/interfaces/base_normalizer.py,sha256=nYQ40IM83WXnIiSIDqhWHfiHAC0O4F9iymb7-u7cSIE,1862
+sayou/refinery/interfaces/base_processor.py,sha256=A9YvD1ZwHhlK290Y77xkSeJTtBCJ53wAYI-KeuVgShM,1650
+sayou/refinery/normalizer/doc_markdown_normalizer.py,sha256=ZVkTEjCesrbjWRRetD1Lp6_YjHsdDL9GNXO4yvUYIUw,10277
+sayou/refinery/normalizer/html_text_normalizer.py,sha256=hX0UTbJwND0Rv-_HuGL-p4Popdrg6_m_mDkKZDYW5AE,1675
+sayou/refinery/normalizer/record_normalizer.py,sha256=bzErNEVw9g-QtQiq_wdm_AxiZi_uuvtx23AL8DRjgxQ,2029
+sayou/refinery/processor/deduplicator.py,sha256=yKZkaPyY4P_a-IwIK8f3XCYqgnoF0us0NbMsBDY03ic,1402
+sayou/refinery/processor/imputer.py,sha256=vvaGvxQNajKSjbYr-gNHmd4HYsD4FtchJhGfnKv-cpo,1562
+sayou/refinery/processor/outlier_handler.py,sha256=bWcECxwVVvfcjiy5lm3YYABUA_7lIW_Do5Q70rK-mtM,2693
+sayou/refinery/processor/pii_masker.py,sha256=fbXPE2HLESQFUvNITnLp-9q2L4MEPcfIkLUUGRIva8k,1726
+sayou/refinery/processor/text_cleaner.py,sha256=8_Hu6H_W__tNfwebm9cS43DRc8EExkhfpkmxslShDjU,1748
+sayou_refinery-0.1.6.dist-info/METADATA,sha256=zmor5IfNcoOzqqmX2OVwudo42b-43etk20LFf5r5wkg,16989
+sayou_refinery-0.1.6.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+sayou_refinery-0.1.6.dist-info/RECORD,,

sayou_refinery-0.1.6.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.28.0
+Root-Is-Purelib: true
+Tag: py3-none-any