notionary 0.2.27__py3-none-any.whl → 0.3.0__py3-none-any.whl

notionary/page/content/parser/post_processing/handlers/rich_text_length.py

@@ -0,0 +1,95 @@
+"""
+Handles request limits for rich texts (see https://developers.notion.com/reference/request-limits)
+"""
+
+from typing import Any, override
+
+from notionary.blocks.rich_text.models import RichText, RichTextType
+from notionary.blocks.schemas import BlockCreatePayload
+from notionary.page.content.parser.post_processing.port import PostProcessor
+from notionary.utils.mixins.logging import LoggingMixin
+
+type _NestedBlockList = BlockCreatePayload | list["_NestedBlockList"]
+
+
+class RichTextLengthTruncationPostProcessor(PostProcessor, LoggingMixin):
+    NOTION_MAX_LENGTH = 2000
+
+    def __init__(self, max_text_length: int = NOTION_MAX_LENGTH) -> None:
+        self.max_text_length = max_text_length
+
+    @override
+    def process(self, blocks: list[BlockCreatePayload]) -> list[BlockCreatePayload]:
+        if not blocks:
+            return blocks
+
+        flattened_blocks = self._flatten_blocks(blocks)
+        return [self._process_block(block) for block in flattened_blocks]
+
+    def _flatten_blocks(self, blocks: list[_NestedBlockList]) -> list[BlockCreatePayload]:
+        flattened: list[BlockCreatePayload] = []
+
+        for item in blocks:
+            if isinstance(item, list):
+                flattened.extend(self._flatten_blocks(item))
+            else:
+                flattened.append(item)
+
+        return flattened
+
+    def _process_block(self, block: BlockCreatePayload) -> BlockCreatePayload:
+        block_copy = block.model_copy(deep=True)
+        content = self._get_block_content(block_copy)
+
+        if content is not None:
+            self._truncate_content(content)
+
+        return block_copy
+
+    def _get_block_content(self, block: BlockCreatePayload) -> Any | None:
+        content = getattr(block, block.type.value, None)
+
+        if content is None:
+            return None
+
+        if hasattr(content, "rich_text") or hasattr(content, "children"):
+            return content
+
+        return None
+
+    def _truncate_content(self, content: object) -> None:
+        if hasattr(content, "rich_text"):
+            self._truncate_rich_text_list(content.rich_text)
+
+        if hasattr(content, "caption"):
+            self._truncate_rich_text_list(content.caption)
+
+        if hasattr(content, "children"):
+            children = getattr(content, "children", None)
+            if children:
+                for child in children:
+                    child_content = self._get_block_content(child)
+                    if child_content:
+                        self._truncate_content(child_content)
+
+    def _truncate_rich_text_list(self, rich_text_list: list[RichText]) -> None:
+        for rich_text in rich_text_list:
+            if not self._is_text_type(rich_text):
+                continue
+
+            content = rich_text.text.content
+            if len(content) > self.max_text_length:
+                self.logger.warning(
+                    "Truncating text content from %d to %d characters",
+                    len(content),
+                    self.max_text_length,
+                )
+                truncated_content = self._create_truncated_text_with_ellipsis(content)
+                rich_text.text.content = truncated_content
+
+    def _create_truncated_text_with_ellipsis(self, content: str) -> str:
+        cutoff = self.max_text_length - 3
+        return content[:cutoff] + "..."
+
+    def _is_text_type(self, rich_text: RichText) -> bool:
+        return rich_text.type == RichTextType.TEXT and rich_text.text and rich_text.text.content

notionary/page/content/parser/post_processing/handlers/rich_text_length_truncation.py

@@ -0,0 +1,114 @@
+"""
+Handles request limits for rich texts (see https://developers.notion.com/reference/request-limits)
+"""
+
+from typing import Any, override
+
+from notionary.blocks.rich_text.models import RichText, RichTextType
+from notionary.blocks.schemas import BlockCreatePayload
+from notionary.page.content.parser.post_processing.port import PostProcessor
+from notionary.utils.mixins.logging import LoggingMixin
+
+type _NestedBlockList = BlockCreatePayload | list["_NestedBlockList"]
+
+
+class RichTextLengthTruncationPostProcessor(PostProcessor, LoggingMixin):
+    NOTION_MAX_LENGTH = 2000
+    ELLIPSIS = "..."
+
+    def __init__(self, max_text_length: int = NOTION_MAX_LENGTH) -> None:
+        self.max_text_length = max_text_length
+
+    @override
+    def process(self, blocks: list[BlockCreatePayload]) -> list[BlockCreatePayload]:
+        if not blocks:
+            return blocks
+
+        flattened_blocks = self._flatten_blocks(blocks)
+        return [self._process_block(block) for block in flattened_blocks]
+
+    def _flatten_blocks(self, blocks: list[_NestedBlockList]) -> list[BlockCreatePayload]:
+        flattened: list[BlockCreatePayload] = []
+
+        for item in blocks:
+            if isinstance(item, list):
+                flattened.extend(self._flatten_blocks(item))
+            else:
+                flattened.append(item)
+
+        return flattened
+
+    def _process_block(self, block: BlockCreatePayload) -> BlockCreatePayload:
+        block_copy = block.model_copy(deep=True)
+        content = self._get_block_content(block_copy)
+
+        if content is not None:
+            self._truncate_content(content)
+
+        return block_copy
+
+    def _get_block_content(self, block: BlockCreatePayload) -> Any | None:
+        content = getattr(block, block.type.value, None)
+
+        if content is None:
+            return None
+
+        if hasattr(content, "rich_text") or hasattr(content, "children"):
+            return content
+
+        return None
+
+    def _truncate_content(self, content: object) -> None:
+        self._truncate_rich_text_fields(content)
+        self._truncate_children_recursively(content)
+
+    def _truncate_rich_text_fields(self, content: object) -> None:
+        if hasattr(content, "rich_text"):
+            self._truncate_rich_text_list(content.rich_text)
+
+        if hasattr(content, "caption"):
+            self._truncate_rich_text_list(content.caption)
+
+    def _truncate_children_recursively(self, content: object) -> None:
+        if not hasattr(content, "children"):
+            return
+
+        children = getattr(content, "children", None)
+        if not children:
+            return
+
+        for child in children:
+            self._truncate_child_content(child)
+
+    def _truncate_child_content(self, child: Any) -> None:
+        child_content = self._get_block_content(child)
+        if child_content:
+            self._truncate_content(child_content)
+
+    def _truncate_rich_text_list(self, rich_text_list: list[RichText]) -> None:
+        for rich_text in rich_text_list:
+            if self._should_truncate(rich_text):
+                self._truncate_single_rich_text(rich_text)
+
+    def _should_truncate(self, rich_text: RichText) -> bool:
+        if not self._is_text_type(rich_text):
+            return False
+
+        return len(rich_text.text.content) > self.max_text_length
+
+    def _truncate_single_rich_text(self, rich_text: RichText) -> None:
+        original_length = len(rich_text.text.content)
+        rich_text.text.content = self._create_truncated_text(rich_text.text.content)
+
+        self.logger.warning(
+            "Truncating text content from %d to %d characters",
+            original_length,
+            self.max_text_length,
+        )
+
+    def _create_truncated_text(self, content: str) -> str:
+        cutoff = self.max_text_length - len(self.ELLIPSIS)
+        return content[:cutoff] + self.ELLIPSIS
+
+    def _is_text_type(self, rich_text: RichText) -> bool:
+        return rich_text.type == RichTextType.TEXT and rich_text.text is not None and rich_text.text.content

notionary/page/content/parser/post_processing/port.py

@@ -0,0 +1,9 @@
+from abc import ABC, abstractmethod
+
+from notionary.blocks.schemas import BlockCreatePayload
+
+
+class PostProcessor(ABC):
+    @abstractmethod
+    def process(self, blocks: list[BlockCreatePayload]) -> list[BlockCreatePayload]:
+        pass

notionary/page/content/parser/post_processing/service.py

@@ -0,0 +1,16 @@
+from notionary.blocks.schemas import BlockCreatePayload
+from notionary.page.content.parser.post_processing.port import PostProcessor
+
+
+class BlockPostProcessor:
+    def __init__(self) -> None:
+        self._processors: list[PostProcessor] = []
+
+    def register(self, processor: PostProcessor) -> None:
+        self._processors.append(processor)
+
+    def process(self, created_blocks: list[BlockCreatePayload]) -> list[BlockCreatePayload]:
+        result = created_blocks
+        for processor in self._processors:
+            result = processor.process(created_blocks)
+        return result

notionary/page/content/parser/pre_processsing/handlers/__init__.py

@@ -0,0 +1,11 @@
+from .column_syntax import ColumnSyntaxPreProcessor
+from .indentation import IndentationNormalizer
+from .port import PreProcessor
+from .whitespace import WhitespacePreProcessor
+
+__all__ = [
+    "ColumnSyntaxPreProcessor",
+    "IndentationNormalizer",
+    "PreProcessor",
+    "WhitespacePreProcessor",
+]

notionary/page/content/parser/pre_processsing/handlers/column_syntax.py

@@ -0,0 +1,130 @@
+import re
+from typing import override
+
+from notionary.exceptions.block_parsing import InsufficientColumnsError, InvalidColumnRatioSumError
+from notionary.page.content.parser.pre_processsing.handlers.port import PreProcessor
+from notionary.page.content.syntax import MarkdownGrammar, SyntaxRegistry
+from notionary.utils.mixins.logging import LoggingMixin
+
+RATIO_TOLERANCE = 0.0001
+MINIMUM_COLUMNS = 2
+
+
+class ColumnSyntaxPreProcessor(PreProcessor, LoggingMixin):
+    def __init__(
+        self, syntax_registry: SyntaxRegistry | None = None, markdown_grammar: MarkdownGrammar | None = None
+    ) -> None:
+        super().__init__()
+        self._syntax_registry = syntax_registry or SyntaxRegistry()
+        self._markdown_grammar = markdown_grammar or MarkdownGrammar()
+
+        self._spaces_per_nesting_level = self._markdown_grammar.spaces_per_nesting_level
+        self._column_list_delimiter = self._syntax_registry.get_column_list_syntax().start_delimiter
+        self._column_delimiter = self._syntax_registry.get_column_syntax().start_delimiter
+        self._column_pattern = self._syntax_registry.get_column_syntax().regex_pattern
+
+    @override
+    def process(self, markdown_text: str) -> str:
+        if not self._contains_column_lists(markdown_text):
+            return markdown_text
+
+        self._validate_all_column_lists(markdown_text)
+        return markdown_text
+
+    def _contains_column_lists(self, markdown_text: str) -> bool:
+        return self._column_list_delimiter in markdown_text
+
+    def _validate_all_column_lists(self, markdown_text: str) -> None:
+        column_list_blocks = self._extract_column_list_blocks(markdown_text)
+
+        for block in column_list_blocks:
+            self._validate_column_list_block(block)
+
+    def _extract_column_list_blocks(self, markdown_text: str) -> list[str]:
+        lines = markdown_text.split("\n")
+        blocks = []
+
+        for index, line in enumerate(lines):
+            if self._is_column_list_start(line):
+                block_content = self._extract_indented_block(lines, index + 1)
+                blocks.append(block_content)
+
+        return blocks
+
+    def _is_column_list_start(self, line: str) -> bool:
+        return line.strip() == self._column_list_delimiter
+
+    def _extract_indented_block(self, lines: list[str], start_index: int) -> str:
+        if start_index >= len(lines):
+            return ""
+
+        base_indentation = self._get_indentation_level(lines[start_index])
+        base_spaces = base_indentation * self._spaces_per_nesting_level
+        block_lines = []
+
+        for line in lines[start_index:]:
+            if self._is_empty_line(line):
+                block_lines.append(line)
+                continue
+
+            current_indentation = self._get_indentation_level(line)
+
+            if current_indentation < base_indentation:
+                break
+
+            block_lines.append(line[base_spaces:] if len(line) >= base_spaces else line)
+
+        return "\n".join(block_lines)
+
+    def _is_empty_line(self, line: str) -> bool:
+        return not line.strip()
+
+    def _get_indentation_level(self, line: str) -> int:
+        leading_spaces = len(line) - len(line.lstrip())
+        return leading_spaces // self._spaces_per_nesting_level
+
+    def _validate_column_list_block(self, block_content: str) -> None:
+        column_matches = self._find_all_columns(block_content)
+        column_count = len(column_matches)
+
+        self._validate_minimum_column_count(column_count)
+
+        ratios = self._extract_column_ratios(column_matches)
+        self._validate_ratio_sum(ratios, column_count)
+
+    def _find_all_columns(self, content: str) -> list[re.Match]:
+        return list(self._column_pattern.finditer(content))
+
+    def _validate_minimum_column_count(self, column_count: int) -> None:
+        if column_count < MINIMUM_COLUMNS:
+            self.logger.error(f"Column list must contain at least {MINIMUM_COLUMNS} columns, found {column_count}")
+            raise InsufficientColumnsError(column_count)
+
+    def _extract_column_ratios(self, column_matches: list[re.Match]) -> list[float]:
+        ratios = []
+
+        for match in column_matches:
+            ratio_text = match.group(1)
+            if self._has_explicit_ratio(ratio_text):
+                ratios.append(float(ratio_text))
+
+        return ratios
+
+    def _has_explicit_ratio(self, ratio_text: str | None) -> bool:
+        return ratio_text is not None and ratio_text != "1"
+
+    def _validate_ratio_sum(self, ratios: list[float], column_count: int) -> None:
+        if not self._should_validate_ratios(ratios, column_count):
+            return
+
+        total_ratio = sum(ratios)
+
+        if not self._is_ratio_sum_valid(total_ratio):
+            self.logger.error(f"Column ratios must sum to 1.0 (±{RATIO_TOLERANCE}), but sum to {total_ratio:.4f}")
+            raise InvalidColumnRatioSumError(total_ratio, RATIO_TOLERANCE)
+
+    def _should_validate_ratios(self, ratios: list[float], column_count: int) -> bool:
+        return len(ratios) > 0 and len(ratios) == column_count
+
+    def _is_ratio_sum_valid(self, total: float) -> bool:
+        return abs(total - 1.0) <= RATIO_TOLERANCE

notionary/page/content/parser/pre_processsing/handlers/indentation.py

@@ -0,0 +1,84 @@
+import math
+from typing import override
+
+from notionary.page.content.parser.pre_processsing.handlers.port import PreProcessor
+from notionary.page.content.syntax import MarkdownGrammar, SyntaxRegistry
+from notionary.utils.mixins.logging import LoggingMixin
+
+
+class IndentationNormalizer(PreProcessor, LoggingMixin):
+    def __init__(
+        self, syntax_registry: SyntaxRegistry | None = None, markdown_grammar: MarkdownGrammar | None = None
+    ) -> None:
+        super().__init__()
+        self._syntax_registry = syntax_registry or SyntaxRegistry()
+        self._markdown_grammar = markdown_grammar or MarkdownGrammar()
+
+        self._spaces_per_nesting_level = self._markdown_grammar.spaces_per_nesting_level
+        self._code_block_start_delimiter = self._syntax_registry.get_code_syntax().start_delimiter
+
+    @override
+    def process(self, markdown_text: str) -> str:
+        if self._is_empty(markdown_text):
+            return ""
+
+        normalized = self._normalize_to_markdown_indentation(markdown_text)
+
+        if normalized != markdown_text:
+            self.logger.warning(
+                "Corrected non-standard indentation. Check the result for formatting errors and use consistent indentation in the source."
+            )
+
+        return normalized
+
+    def _is_empty(self, text: str) -> bool:
+        return not text
+
+    def _normalize_to_markdown_indentation(self, markdown_text: str) -> str:
+        lines = markdown_text.split("\n")
+        processed_lines = []
+        inside_code_block = False
+
+        for line in lines:
+            if self._is_code_fence(line):
+                inside_code_block = not inside_code_block
+                processed_lines.append(line)
+            elif inside_code_block:
+                processed_lines.append(line)
+            else:
+                processed_lines.append(self._normalize_to_standard_indentation(line))
+
+        return "\n".join(processed_lines)
+
+    def _is_code_fence(self, line: str) -> bool:
+        return line.lstrip().startswith(self._code_block_start_delimiter)
+
+    def _normalize_to_standard_indentation(self, line: str) -> str:
+        if self._is_blank_line(line):
+            return ""
+
+        indentation_level = self._round_to_nearest_indentation_level(line)
+        content = self._extract_content(line)
+
+        return self._build_indented_line(indentation_level, content)
+
+    def _is_blank_line(self, line: str) -> bool:
+        return not line.strip()
+
+    def _round_to_nearest_indentation_level(self, line: str) -> int:
+        leading_spaces = self._count_leading_spaces(line)
+        return math.ceil(leading_spaces / self._spaces_per_nesting_level)
+
+    def _count_leading_spaces(self, line: str) -> int:
+        return len(line) - len(line.lstrip())
+
+    def _extract_content(self, line: str) -> str:
+        return line.lstrip()
+
+    def _build_indented_line(self, level: int, content: str) -> str:
+        standard_indent = self._create_standard_indent(level)
+        return standard_indent + content
+
+    def _create_standard_indent(self, level: int) -> str:
+        spaces = level * self._spaces_per_nesting_level
+        return " " * spaces

notionary/page/content/parser/pre_processsing/handlers/port.py

@@ -0,0 +1,7 @@
+from abc import ABC, abstractmethod
+
+
+class PreProcessor(ABC):
+    @abstractmethod
+    def process(self, markdown_text: str) -> str:
+        pass

notionary/page/content/parser/pre_processsing/handlers/whitespace.py

@@ -0,0 +1,73 @@
+from typing import override
+
+from notionary.page.content.parser.pre_processsing.handlers.port import PreProcessor
+
+
+class WhitespacePreProcessor(PreProcessor):
+    @override
+    def process(self, markdown_text: str) -> str:
+        if not markdown_text:
+            return ""
+
+        lines = markdown_text.split("\n")
+        processed_lines = []
+        code_block_lines = []
+        non_code_lines = []
+        in_code_block = False
+
+        for line in lines:
+            if self._is_code_fence(line):
+                if in_code_block:
+                    # Format and add code block
+                    processed_lines.extend(self._format_code_block(code_block_lines))
+                    processed_lines.append("```")
+                    code_block_lines = []
+                    in_code_block = False
+                else:
+                    # Format accumulated non-code lines before starting code block
+                    if non_code_lines:
+                        processed_lines.extend(self._format_code_block(non_code_lines))
+                        non_code_lines = []
+
+                    language = self._extract_language(line)
+                    processed_lines.append(f"```{language}")
+                    in_code_block = True
+            elif in_code_block:
+                code_block_lines.append(line)
+            else:
+                non_code_lines.append(line)
+
+        # Format remaining non-code lines at the end
+        if non_code_lines:
+            processed_lines.extend(self._format_code_block(non_code_lines))
+
+        return "\n".join(processed_lines)
+
+    def _is_code_fence(self, line: str) -> bool:
+        return line.lstrip().startswith("```")
+
+    def _extract_language(self, fence_line: str) -> str:
+        return fence_line.lstrip().removeprefix("```").strip()
+
+    def _format_code_block(self, lines: list[str]) -> list[str]:
+        if not lines:
+            return []
+
+        non_empty_lines = [line for line in lines if line.strip()]
+        if not non_empty_lines:
+            return ["" for _ in lines]
+
+        min_indent = min(self._count_leading_spaces(line) for line in non_empty_lines)
+
+        if min_indent == 0:
+            return lines
+
+        return [self._remove_indent(line, min_indent) for line in lines]
+
+    def _count_leading_spaces(self, line: str) -> int:
+        return len(line) - len(line.lstrip())
+
+    def _remove_indent(self, line: str, indent_size: int) -> str:
+        if not line.strip():
+            return ""
+        return line[indent_size:]

notionary/page/content/parser/pre_processsing/service.py

@@ -0,0 +1,15 @@
+from notionary.page.content.parser.pre_processsing.handlers.port import PreProcessor
+
+
+class MarkdownPreProcessor:
+    def __init__(self) -> None:
+        self._processors: list[PreProcessor] = []
+
+    def register(self, processor: PreProcessor) -> None:
+        self._processors.append(processor)
+
+    def process(self, markdown_text: str) -> str:
+        result = markdown_text
+        for processor in self._processors:
+            result = processor.process(result)
+        return result

notionary/page/content/parser/service.py

@@ -0,0 +1,78 @@
+from notionary.blocks.schemas import BlockCreatePayload
+from notionary.page.content.parser.parsers import (
+    BlockParsingContext,
+    ParentBlockContext,
+)
+from notionary.page.content.parser.parsers.base import LineParser
+from notionary.page.content.parser.post_processing.service import BlockPostProcessor
+from notionary.page.content.parser.pre_processsing.service import MarkdownPreProcessor
+from notionary.utils.mixins.logging import LoggingMixin
+
+
+class MarkdownToNotionConverter(LoggingMixin):
+    def __init__(
+        self, line_parser: LineParser, pre_processor: MarkdownPreProcessor, post_processor: BlockPostProcessor
+    ) -> None:
+        self._line_parser = line_parser
+        self._pre_processor = pre_processor
+        self._post_processor = post_processor
+
+    async def convert(self, markdown_text: str) -> list[BlockCreatePayload]:
+        if not markdown_text:
+            return []
+
+        markdown_text = self._pre_processor.process(markdown_text)
+        all_blocks = await self._process_lines(markdown_text)
+        all_blocks = self._post_processor.process(all_blocks)
+
+        return all_blocks
+
+    async def _process_lines(self, text: str) -> list[BlockCreatePayload]:
+        lines = text.split("\n")
+        result_blocks: list[BlockCreatePayload] = []
+        parent_stack: list[ParentBlockContext] = []
+
+        current_line_index = 0
+        previous_line_was_empty = False
+
+        while current_line_index < len(lines):
+            line = lines[current_line_index]
+
+            context = self._create_line_processing_context(
+                line=line,
+                lines=lines,
+                line_index=current_line_index,
+                result_blocks=result_blocks,
+                parent_stack=parent_stack,
+                is_previous_line_empty=previous_line_was_empty,
+            )
+
+            await self._line_parser.handle(context)
+            previous_line_was_empty = self._is_processed_line_empty(line)
+
+            current_line_index += 1 + context.lines_consumed
+
+        return result_blocks
+
+    def _create_line_processing_context(
+        self,
+        line: str,
+        lines: list[str],
+        line_index: int,
+        result_blocks: list[BlockCreatePayload],
+        parent_stack: list[ParentBlockContext],
+        is_previous_line_empty: bool = False,
+    ) -> BlockParsingContext:
+        return BlockParsingContext(
+            line=line,
+            result_blocks=result_blocks,
+            parent_stack=parent_stack,
+            parse_children_callback=self._process_lines,
+            all_lines=lines,
+            current_line_index=line_index,
+            lines_consumed=0,
+            is_previous_line_empty=is_previous_line_empty,
+        )
+
+    def _is_processed_line_empty(self, line: str) -> bool:
+        return line.strip() == ""