ai-pipeline-core 0.1.12__py3-none-any.whl → 0.4.1__py3-none-any.whl

ai_pipeline_core/documents/document.py

@@ -1,23 +1,18 @@
 """Document abstraction layer for AI pipeline flows.
 
-@public
-
-This module provides the core document abstraction for working with various types of data
-in AI pipelines. Documents are immutable Pydantic models that wrap binary content with metadata.
+Immutable Pydantic models wrapping binary content with metadata, MIME detection,
+SHA256 hashing, and serialization. All documents must be concrete subclasses of Document.
 """
 
 import base64
-import hashlib
+import functools
 import json
-from abc import ABC, abstractmethod
-from base64 import b32encode
 from enum import StrEnum
 from functools import cached_property
 from io import BytesIO
 from typing import (
     Any,
     ClassVar,
-    Literal,
     Self,
     TypeVar,
     cast,
@@ -27,18 +22,23 @@ from typing import (
     overload,
 )
 
+import tiktoken
 from pydantic import (
     BaseModel,
     ConfigDict,
     ValidationInfo,
     field_serializer,
     field_validator,
+    model_validator,
 )
 from ruamel.yaml import YAML
 
-from ai_pipeline_core.documents.utils import canonical_name_key
+from ai_pipeline_core.documents._context_vars import get_task_context, is_registration_suppressed
+from ai_pipeline_core.documents._hashing import compute_content_sha256, compute_document_sha256
+from ai_pipeline_core.documents.utils import canonical_name_key, is_document_sha256
 from ai_pipeline_core.exceptions import DocumentNameError, DocumentSizeError
 
+from .attachment import Attachment
 from .mime_type import (
     detect_mime_type,
     is_image_mime_type,
@@ -48,136 +48,77 @@ from .mime_type import (
 )
 
 TModel = TypeVar("TModel", bound=BaseModel)
+TDocument = TypeVar("TDocument", bound="Document")
 
+# Registry of canonical_name -> Document subclass for collision detection.
+# Only non-test classes are registered. Test modules (tests.*, conftest, etc.) are skipped.
+_canonical_name_registry: dict[str, type["Document"]] = {}  # nosemgrep: no-mutable-module-globals
 
-class Document(BaseModel, ABC):
-    r"""Abstract base class for all documents in the AI Pipeline Core system.
-
-    @public
-
-    Document is the fundamental data abstraction for all content flowing through
-    pipelines. It provides automatic encoding, MIME type detection, serialization,
-    and validation. All documents must be subclassed from FlowDocument or TaskDocument
-    based on their persistence requirements. TemporaryDocument is a special concrete
-    class that can be instantiated directly (not abstract).
-
-    VALIDATION IS AUTOMATIC - Do not add manual validation!
-        Size validation, name validation, and MIME type detection are built-in.
-        The framework handles all standard validations internally.
-
-        # WRONG - These checks already happen automatically:
-        if document.size > document.MAX_CONTENT_SIZE:
-            raise DocumentSizeError(...)  # NO! Already handled
-        document.validate_file_name(document.name)  # NO! Automatic
-
-    Best Practices:
-        - Use create() classmethod for automatic type conversion (90% of cases)
-        - Omit description parameter unless truly needed for metadata
-        - When using LLM functions, pass AIMessages or str. Wrap any Document values
-          in AIMessages([...]). Do not call .text yourself
-
-    Standard Usage:
-        >>> # CORRECT - minimal parameters
-        >>> doc = MyDocument.create(name="data.json", content={"key": "value"})
-
-        >>> # AVOID - unnecessary description
-        >>> doc = MyDocument.create(
-        ...     name="data.json",
-        ...     content={"key": "value"},
-        ...     description="This is data"  # Usually not needed!
-        ... )
-
-    Key features:
-    - Immutable by default (frozen Pydantic model)
-    - Automatic MIME type detection
-    - Content size validation
-    - SHA256 hashing for deduplication
-    - Support for text, JSON, YAML, PDF, and image formats
-    - Conversion utilities between different formats
-
-    Class Variables:
-        MAX_CONTENT_SIZE: Maximum allowed content size in bytes (default 25MB)
-
-    Attributes:
-        name: Document filename (validated for security)
-        description: Optional human-readable description
-        content: Raw document content as bytes
-
-    Creating Documents:
-        **Use the `create` classmethod** for most use cases. It accepts various
-        content types (str, dict, list, BaseModel) and converts them automatically.
-        Only use __init__ directly when you already have bytes content.
-
-        >>> # RECOMMENDED: Use create for automatic conversion
-        >>> doc = MyDocument.create(name="data.json", content={"key": "value"})
-        >>>
-        >>> # Direct constructor: Only for bytes
-        >>> doc = MyDocument(name="data.bin", content=b"\x00\x01\x02")
-
-    Warning:
-        - Document subclasses should NOT start with 'Test' prefix (pytest conflict)
-        - Cannot instantiate Document directly - must subclass FlowDocument or TaskDocument
-        - Cannot add custom fields - only name, description, content are allowed
-        - Document is an abstract class and cannot be instantiated directly
-
-    Metadata Attachment Patterns:
-        Since custom fields are not allowed, use these patterns for metadata:
-        1. Use the 'description' field for human-readable metadata
-        2. Embed metadata in content (e.g., JSON with data + metadata fields)
-        3. Create a separate MetadataDocument type to accompany data documents
-        4. Use document naming conventions (e.g., "data_v2_2024.json")
-        5. Store metadata in flow_options or pass through TraceInfo
-
-    Example:
-        >>> from enum import StrEnum
-        >>>
-        >>> # Simple document:
-        >>> class MyDocument(FlowDocument):
-        ...     pass
-        >>>
-        >>> # Document with file restrictions:
-        >>> class ConfigDocument(FlowDocument):
-        ...     class FILES(StrEnum):
-        ...         CONFIG = "config.yaml"
-        ...         SETTINGS = "settings.json"
-        >>>
-        >>> # RECOMMENDED: Use create for automatic conversion
-        >>> doc = MyDocument.create(name="data.json", content={"key": "value"})
-        >>> print(doc.is_text)  # True
-        >>> data = doc.as_json()  # {'key': 'value'}
-    """
 
-    MAX_CONTENT_SIZE: ClassVar[int] = 25 * 1024 * 1024
-    """Maximum allowed content size in bytes (default 25MB).
+def _is_test_module(cls: type) -> bool:
+    """Check if a class is defined in a test module (skip collision detection)."""
+    module = getattr(cls, "__module__", "") or ""
+    parts = module.split(".")
+    return any(p == "tests" or p.startswith("test_") or p == "conftest" for p in parts)
 
-    @public
-    """
 
-    DESCRIPTION_EXTENSION: ClassVar[str] = ".description.md"
-    """File extension for description files."""
+@functools.cache
+def get_tiktoken_encoding() -> tiktoken.Encoding:
+    """Lazy-cached tiktoken encoding. Deferred to first use, cached forever."""
+    return tiktoken.encoding_for_model("gpt-4")
 
-    MARKDOWN_LIST_SEPARATOR: ClassVar[str] = "\n\n-----------------\n\n"
-    """Separator for markdown list items."""
 
-    def __init_subclass__(cls, **kwargs: Any) -> None:
-        """Validate subclass configuration at definition time.
+def _serialize_to_json(data: Any) -> bytes:
+    """JSON serialize with 2-space indent."""
+    return json.dumps(data, indent=2).encode("utf-8")
 
-        Performs several validation checks when a Document subclass is defined:
-        1. Prevents class names starting with 'Test' (pytest conflict)
-        2. Validates FILES enum if present (must be StrEnum)
-        3. Prevents adding custom fields beyond name, description, content
 
-        Args:
-            **kwargs: Additional keyword arguments passed to parent __init_subclass__.
+def _serialize_to_yaml(data: Any) -> bytes:
+    """YAML serialize via ruamel."""
+    yaml = YAML()
+    stream = BytesIO()
+    yaml.dump(data, stream)  # pyright: ignore[reportUnknownMemberType]
+    return stream.getvalue()
 
-        Raises:
-            TypeError: If subclass violates naming rules, FILES enum requirements,
-                      or attempts to add extra fields.
 
-        Note:
-            This validation happens at class definition time, not instantiation,
-            providing early error detection during development.
-        """
+def _serialize_structured(name: str, data: Any) -> bytes:
+    """Serialize dict/list to JSON or YAML based on file extension."""
+    name_lower = name.lower()
+    if name_lower.endswith((".yaml", ".yml")):
+        return _serialize_to_yaml(data)
+    if name_lower.endswith(".json"):
+        return _serialize_to_json(data)
+    raise ValueError(f"Structured content ({type(data).__name__}) requires .json or .yaml extension, got: {name}")
+
+
+def _convert_content(name: str, content: str | bytes | dict[str, Any] | list[Any] | BaseModel) -> bytes:
+    """Convert any supported content type to bytes. Dispatch by isinstance."""
+    if isinstance(content, bytes):
+        return content
+    if isinstance(content, str):
+        return content.encode("utf-8")
+    if isinstance(content, dict):
+        return _serialize_structured(name, content)
+    if isinstance(content, BaseModel):
+        return _serialize_structured(name, content.model_dump(mode="json"))
+    if isinstance(content, list):  # pyright: ignore[reportUnnecessaryIsInstance]
+        data = [item.model_dump(mode="json") if isinstance(item, BaseModel) else item for item in content]
+        return _serialize_structured(name, data)
+    raise ValueError(f"Unsupported content type: {type(content)}")  # pyright: ignore[reportUnreachable]
+
+
+class Document(BaseModel):
+    """Immutable base class for all pipeline documents. Cannot be instantiated directly — must be subclassed.
+
+    Content is stored as bytes. Use `create()` for automatic conversion from str/dict/list/BaseModel.
+    Use `parse()` to reverse the conversion. Serialization is extension-driven (.json → JSON, .yaml → YAML).
+    """
+
+    MAX_CONTENT_SIZE: ClassVar[int] = 25 * 1024 * 1024
+    """Maximum allowed total size in bytes (default 25MB)."""
+
+    def __init_subclass__(cls, **kwargs: Any) -> None:
+        """Validate subclass at definition time. Cannot start with 'Test', cannot add custom fields."""
         super().__init_subclass__(**kwargs)
         if cls.__name__.startswith("Test"):
             raise TypeError(
@@ -186,14 +127,12 @@ class Document(BaseModel, ABC):
                 "Please use a different name (e.g., 'SampleDocument', 'ExampleDocument')."
             )
         if hasattr(cls, "FILES"):
-            files = getattr(cls, "FILES")
+            files: type[StrEnum] = cls.FILES  # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType]
             if not issubclass(files, StrEnum):
-                raise TypeError(
-                    f"Document subclass '{cls.__name__}'.FILES must be an Enum of string values"
-                )
+                raise TypeError(f"Document subclass '{cls.__name__}'.FILES must be an Enum of string values")
         # Check that the Document's model_fields only contain the allowed fields
         # It prevents AI models from adding additional fields to documents
-        allowed = {"name", "description", "content"}
+        allowed = {"name", "description", "content", "sources", "attachments", "origins"}
         current = set(getattr(cls, "model_fields", {}).keys())
         extras = current - allowed
         if extras:
@@ -202,27 +141,18 @@ class Document(BaseModel, ABC):
                 f"{', '.join(sorted(extras))}. Only {', '.join(sorted(allowed))} are allowed."
             )
 
-    @overload
-    @classmethod
-    def create(cls, *, name: str, content: bytes, description: str | None = None) -> Self: ...
-
-    @overload
-    @classmethod
-    def create(cls, *, name: str, content: str, description: str | None = None) -> Self: ...
-
-    @overload
-    @classmethod
-    def create(
-        cls, *, name: str, content: dict[str, Any], description: str | None = None
-    ) -> Self: ...
-
-    @overload
-    @classmethod
-    def create(cls, *, name: str, content: list[Any], description: str | None = None) -> Self: ...
-
-    @overload
-    @classmethod
-    def create(cls, *, name: str, content: BaseModel, description: str | None = None) -> Self: ...
+        # Canonical name collision detection (production classes only)
+        if not _is_test_module(cls):
+            canonical = canonical_name_key(cls)
+            existing = _canonical_name_registry.get(canonical)
+            if existing is not None and existing is not cls:
+                raise TypeError(
+                    f"Document subclass '{cls.__name__}' (in {cls.__module__}) produces "
+                    f"canonical_name '{canonical}' which collides with existing class "
+                    f"'{existing.__name__}' (in {existing.__module__}). "
+                    f"Rename one of the classes to avoid ambiguity."
+                )
+            _canonical_name_registry[canonical] = cls
 
     @classmethod
     def create(
@@ -231,86 +161,23 @@ class Document(BaseModel, ABC):
         name: str,
         content: str | bytes | dict[str, Any] | list[Any] | BaseModel,
         description: str | None = None,
+        sources: tuple[str, ...] | None = None,
+        origins: tuple[str, ...] | None = None,
+        attachments: tuple[Attachment, ...] | None = None,
     ) -> Self:
-        r"""Create a Document with automatic content type conversion (recommended).
-
-        @public
-
-        This is the **recommended way to create documents**. It accepts various
-        content types and automatically converts them to bytes based on the file
-        extension. Use the `parse` method to reverse this conversion.
-
-        Best Practice (90% of cases):
-            Only provide name and content. The description parameter is RARELY needed.
-
-        Args:
-            name: Document filename (required, keyword-only).
-                  Extension determines serialization:
-                  - .json → JSON serialization
-                  - .yaml/.yml → YAML serialization
-                  - .md → Markdown list joining (for list[str])
-                  - Others → UTF-8 encoding (for str)
-            content: Document content in various formats (required, keyword-only):
-                - bytes: Used directly without conversion
-                - str: Encoded to UTF-8 bytes
-                - dict[str, Any]: Serialized to JSON (.json) or YAML (.yaml/.yml)
-                - list[str]: Joined with separator for .md (validates no items
-                            contain separator), else JSON/YAML
-                - list[BaseModel]: Serialized to JSON or YAML based on extension
-                - BaseModel: Serialized to JSON or YAML based on extension
-            description: Optional description - USUALLY OMIT THIS (defaults to None).
-                        Only use when meaningful metadata helps downstream processing
-
-        Returns:
-            New Document instance with content converted to bytes
-
-        Raises:
-            ValueError: If content type is not supported for the file extension,
-                       or if markdown list items contain the separator
-            DocumentNameError: If filename violates validation rules
-            DocumentSizeError: If content exceeds MAX_CONTENT_SIZE
-
-        Note:
-            All conversions are reversible using the `parse` method.
-            For example: MyDocument.create(name="data.json", content={"key": "value"}).parse(dict)
-            returns the original dictionary {"key": "value"}.
-
-        Example:
-            >>> # CORRECT - no description needed (90% of cases)
-            >>> doc = MyDocument.create(name="test.txt", content="Hello World")
-            >>> doc.content  # b'Hello World'
-            >>> doc.parse(str)  # "Hello World"
-
-            >>> # CORRECT - Dictionary to JSON, no description
-            >>> doc = MyDocument.create(name="config.json", content={"key": "value"})
-            >>> doc.content  # b'{"key": "value", ...}'
-            >>> doc.parse(dict)  # {"key": "value"}
-
-            >>> # AVOID unless description adds real value
-            >>> doc = MyDocument.create(
-            ...     name="config.json",
-            ...     content={"key": "value"},
-            ...     description="Config file"  # Usually redundant!
-            ... )
-
-            >>> # Pydantic model to YAML
-            >>> from pydantic import BaseModel
-            >>> class Config(BaseModel):
-            ...     host: str
-            ...     port: int
-            >>> config = Config(host="localhost", port=8080)
-            >>> doc = MyDocument.create(name="config.yaml", content=config)
-            >>> doc.parse(Config)  # Returns Config instance
-
-            >>> # List to Markdown
-            >>> items = ["Section 1", "Section 2"]
-            >>> doc = MyDocument.create(name="sections.md", content=items)
-            >>> doc.parse(list)  # ["Section 1", "Section 2"]
+        """Create a document with automatic content-to-bytes conversion.
+
+        Serialization is extension-driven: .json → JSON, .yaml → YAML, others → UTF-8.
+        Reversible via parse(). Cannot be called on Document directly — must use a subclass.
         """
-        # Use model_validate to leverage the existing validator logic
-        temp = cls.model_validate({"name": name, "content": content, "description": description})
-        # Now construct with type-checker-friendly call (bytes only)
-        return cls(name=temp.name, content=temp.content, description=temp.description)
+        return cls(
+            name=name,
+            content=_convert_content(name, content),
+            description=description,
+            sources=sources,
+            origins=origins,
+            attachments=attachments,
+        )
 
     def __init__(
         self,
@@ -318,50 +185,53 @@ class Document(BaseModel, ABC):
         name: str,
         content: bytes,
         description: str | None = None,
+        sources: tuple[str, ...] | None = None,
+        origins: tuple[str, ...] | None = None,
+        attachments: tuple[Attachment, ...] | None = None,
     ) -> None:
-        """Initialize a Document instance with raw bytes content.
-
-        @public
-
-        Important:
-            **Most users should use the `create` classmethod instead of __init__.**
-            The create method provides automatic content conversion for various types
-            (str, dict, list, Pydantic models) while __init__ only accepts bytes.
-
-        This constructor accepts only bytes content for type safety. It prevents
-        direct instantiation of the abstract Document class.
+        """Initialize with raw bytes content. Most users should use `create()` instead."""
+        if type(self) is Document:
+            raise TypeError("Cannot instantiate Document directly — use a concrete subclass")
 
-        Args:
-            name: Document filename (required, keyword-only)
-            content: Document content as raw bytes (required, keyword-only)
-            description: Optional human-readable description (keyword-only)
+        super().__init__(
+            name=name,
+            content=content,
+            description=description,
+            sources=sources or (),
+            origins=origins or (),
+            attachments=attachments or (),
+        )
 
-        Raises:
-            TypeError: If attempting to instantiate Document directly.
+        # Register with task context for document lifecycle tracking
+        if not is_registration_suppressed():
+            task_ctx = get_task_context()
+            if task_ctx is not None:
+                task_ctx.register_created(self)  # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
 
-        Example:
-            >>> # Direct constructor - only for bytes content:
-            >>> doc = MyDocument(name="test.txt", content=b"Hello World")
-            >>> doc.content  # b'Hello World'
+    name: str
+    description: str | None = None
+    content: bytes
+    sources: tuple[str, ...] = ()
+    """Content provenance: documents and references this document's content was directly
+    derived from. Can be document SHA256 hashes (for pipeline documents) or external
+    references (URLs, file paths). Answers: 'where did this content come from?'
 
-            >>> # RECOMMENDED: Use create for automatic conversion:
-            >>> doc = MyDocument.create(name="text.txt", content="Hello World")
-            >>> doc = MyDocument.create(name="data.json", content={"key": "value"})
-            >>> doc = MyDocument.create(name="config.yaml", content=my_model)
-            >>> doc = MyDocument.create(name="items.md", content=["item1", "item2"])
+    Example: an analysis document derived from an input document has
+    sources=(input_doc.sha256,). A webpage capture has sources=("https://example.com",)."""
 
-        See Also:
-            create: Recommended factory method with automatic type conversion
-            parse: Method to reverse the conversion done by create
-        """
-        if type(self) is Document:
-            raise TypeError("Cannot instantiate abstract Document class directly")
+    origins: tuple[str, ...] = ()
+    """Causal provenance: documents that caused this document to be created without directly
+    contributing to its content. Always document SHA256 hashes, never arbitrary strings.
+    Answers: 'why does this document exist?'
 
-        super().__init__(name=name, content=content, description=description)
+    Example: a research plan causes 10 webpages to be captured. Each webpage's source is its
+    URL (content provenance), its origin is the research plan (causal provenance — the plan
+    caused the capture but didn't contribute to the webpage's content).
 
-    name: str
-    description: str | None = None
-    content: bytes  # Note: constructor accepts str | bytes, but field stores bytes only
+    A SHA256 hash must not appear in both sources and origins for the same document.
+    Within a pipeline task or flow, all source/origin SHA256 references must point to
+    documents that existed before the task/flow started executing."""
+    attachments: tuple[Attachment, ...] = ()
 
     # Pydantic configuration
     model_config = ConfigDict(
@@ -370,148 +240,27 @@ class Document(BaseModel, ABC):
         extra="forbid",
     )
 
-    @abstractmethod
-    def get_base_type(self) -> Literal["flow", "task", "temporary"]:
-        """Get the base type of the document.
-
-        Abstract method that must be implemented by all Document subclasses
-        to indicate their persistence behavior.
-
-        Returns:
-            One of "flow" (persisted across flow runs), "task" (temporary
-            within task execution), or "temporary" (never persisted).
-
-        Note:
-            This method determines document persistence and lifecycle.
-            FlowDocument returns "flow", TaskDocument returns "task",
-            TemporaryDocument returns "temporary".
-        """
-        raise NotImplementedError("Subclasses must implement this method")
-
-    @final
-    @property
-    def base_type(self) -> Literal["flow", "task", "temporary"]:
-        """Get the document's base type.
-
-        Property alias for get_base_type() providing a cleaner API.
-        This property cannot be overridden by subclasses.
-
-        Returns:
-            The document's base type: "flow", "task", or "temporary".
-        """
-        return self.get_base_type()
-
-    @final
-    @property
-    def is_flow(self) -> bool:
-        """Check if this is a flow document.
-
-        Flow documents persist across Prefect flow runs and are saved
-        to the file system between pipeline steps.
-
-        Returns:
-            True if this is a FlowDocument subclass, False otherwise.
-        """
-        return self.get_base_type() == "flow"
-
-    @final
-    @property
-    def is_task(self) -> bool:
-        """Check if this is a task document.
-
-        Task documents are temporary within Prefect task execution
-        and are not persisted between pipeline steps.
-
-        Returns:
-            True if this is a TaskDocument subclass, False otherwise.
-        """
-        return self.get_base_type() == "task"
-
-    @final
-    @property
-    def is_temporary(self) -> bool:
-        """Check if this is a temporary document.
-
-        Temporary documents are never persisted and exist only
-        during execution.
-
-        Returns:
-            True if this is a TemporaryDocument, False otherwise.
-        """
-        return self.get_base_type() == "temporary"
-
     @final
     @classmethod
     def get_expected_files(cls) -> list[str] | None:
-        """Get the list of allowed file names for this document class.
-
-        If the document class defines a FILES enum, returns the list of
-        valid file names. Used to restrict documents to specific files.
-
-        Returns:
-            List of allowed file names if FILES enum is defined,
-            None if unrestricted.
-
-        Raises:
-            DocumentNameError: If FILES is defined but not a valid StrEnum.
-
-        Example:
-            >>> class ConfigDocument(FlowDocument):
-            ...     class FILES(StrEnum):
-            ...         CONFIG = "config.yaml"
-            ...         SETTINGS = "settings.json"
-            >>> ConfigDocument.get_expected_files()
-            ['config.yaml', 'settings.json']
-        """
+        """Return allowed filenames from FILES enum, or None if unrestricted."""
         if not hasattr(cls, "FILES"):
             return None
-        files = getattr(cls, "FILES")
+        files: type[StrEnum] = cls.FILES  # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownVariableType]
         if not files:
             return None
         assert issubclass(files, StrEnum)
         try:
             values = [member.value for member in files]
         except TypeError:
-            raise DocumentNameError(f"{cls.__name__}.FILES must be an Enum of string values")
+            raise DocumentNameError(f"{cls.__name__}.FILES must be an Enum of string values") from None
         if len(values) == 0:
             return None
         return values
 
     @classmethod
     def validate_file_name(cls, name: str) -> None:
-        """Validate that a file name matches allowed patterns.
-
-        @public
-
-        DO NOT OVERRIDE this method if you define a FILES enum!
-        The validation is automatic when FILES enum is present.
-
-        # CORRECT - FILES enum provides automatic validation:
-        class MyDocument(FlowDocument):
-            class FILES(StrEnum):
-                CONFIG = "config.yaml"  # Validation happens automatically!
-
-        # WRONG - Unnecessary override:
-        class MyDocument(FlowDocument):
-            class FILES(StrEnum):
-                CONFIG = "config.yaml"
-
-            def validate_file_name(cls, name):  # DON'T DO THIS!
-                pass  # Validation already happens via FILES enum
-
-        Only override for custom validation logic BEYOND FILES enum constraints.
-
-        Args:
-            name: The file name to validate.
-
-        Raises:
-            DocumentNameError: If the name doesn't match allowed patterns.
-
-        Note:
-            - If FILES enum is defined, name must exactly match one of the values
-            - If FILES is not defined, any name is allowed
-            - Override in subclasses ONLY for custom regex patterns or logic
-        """
+        """Validate filename against FILES enum. Override only for custom validation beyond FILES."""
         allowed = cls.get_expected_files()
         if not allowed:
             return
@@ -521,42 +270,18 @@ class Document(BaseModel, ABC):
             raise DocumentNameError(f"Invalid filename '{name}'. Allowed names: {allowed_str}")
 
     @field_validator("name")
+    @classmethod
     def validate_name(cls, v: str) -> str:
-        r"""Pydantic validator for the document name field.
-
-        Ensures the document name is secure and follows conventions:
-        - No path traversal characters (.., \\, /)
-        - Cannot end with .description.md
-        - No leading/trailing whitespace
-        - Must match FILES enum if defined
-
-        Performance:
-            Validation is O(n) where n is the length of the name.
-            FILES enum check is O(m) where m is the number of allowed files
-
-        Args:
-            v: The name value to validate.
-
-        Returns:
-            The validated name.
-
-        Raises:
-            DocumentNameError: If the name violates any validation rules.
-
-        Note:
-            This is called automatically by Pydantic during model construction.
-        """
-        if v.endswith(cls.DESCRIPTION_EXTENSION):
-            raise DocumentNameError(
-                f"Document names cannot end with {cls.DESCRIPTION_EXTENSION}: {v}"
-            )
-
+        """Reject path traversal, whitespace issues, reserved suffixes. Must match FILES enum if defined."""
         if ".." in v or "\\" in v or "/" in v:
             raise DocumentNameError(f"Invalid filename - contains path traversal characters: {v}")
 
         if not v or v.startswith(" ") or v.endswith(" "):
             raise DocumentNameError(f"Invalid filename format: {v}")
 
+        if v.endswith(".meta.json"):
+            raise DocumentNameError(f"Document names cannot end with .meta.json (reserved): {v}")
+
         cls.validate_file_name(v)
 
         return v
@@ -564,174 +289,58 @@ class Document(BaseModel, ABC):
     @field_validator("content", mode="before")
     @classmethod
     def validate_content(cls, v: Any, info: ValidationInfo) -> bytes:
-        """Pydantic validator that converts various content types to bytes.
-
-        This validator is called automatically during model construction and
-        handles the intelligent type conversion that powers the `create` method.
-        It determines the appropriate serialization based on file extension.
-
-        Conversion Strategy:
-            1. bytes → Passthrough (no conversion)
-            2. str → UTF-8 encoding
-            3. dict/BaseModel + .json → JSON serialization (indented)
-            4. dict/BaseModel + .yaml/.yml → YAML serialization
-            5. list[str] + .md → Join with markdown separator (validates no items contain separator)
-            6. list[Any] + .json/.yaml → JSON/YAML array
-            7. int/float/bool + .json → JSON primitive
-
-        Args:
-            v: Content to validate (any supported type)
-            info: Validation context containing other field values
-
-        Returns:
-            Content converted to bytes
-
-        Raises:
-            DocumentSizeError: If content exceeds MAX_CONTENT_SIZE
-            ValueError: If content type unsupported for file extension
-
-        Note:
-            This validator enables create() to accept multiple types while
-            ensuring __init__ only receives bytes for type safety.
-        """
-        # Get the name from validation context if available
-        name = ""
-        if hasattr(info, "data") and "name" in info.data:
-            name = info.data["name"]
-        name_lower = name.lower()
-
-        # Convert based on content type
-        if isinstance(v, bytes):
-            pass  # Already bytes
-        elif isinstance(v, str):
-            v = v.encode("utf-8")
-        elif isinstance(v, dict):
-            # Serialize dict based on extension
-            if name_lower.endswith((".yaml", ".yml")):
-                # Use YAML format for YAML files
-                yaml = YAML()
-                stream = BytesIO()
-                yaml.dump(v, stream)
-                v = stream.getvalue()
-            elif name_lower.endswith(".json"):
-                # Use JSON for JSON files
-                v = json.dumps(v, indent=2).encode("utf-8")
-            else:
-                # Dict not supported for other file types
-                raise ValueError(f"Unsupported content type: {type(v)} for file {name}")
-        elif isinstance(v, list):
-            # Handle lists based on file extension
-            if name_lower.endswith(".md"):
-                # For markdown files, join with separator
-                if all(isinstance(item, str) for item in v):
-                    # Check that no string contains the separator
-                    for item in v:
-                        if cls.MARKDOWN_LIST_SEPARATOR in item:
-                            raise ValueError(
-                                f"Markdown list item cannot contain the separator "
-                                f"'{cls.MARKDOWN_LIST_SEPARATOR}' as it will mess up formatting"
-                            )
-                    v = cls.MARKDOWN_LIST_SEPARATOR.join(v).encode("utf-8")
-                else:
-                    raise ValueError(
-                        f"Unsupported content type: mixed-type list for markdown file {name}"
-                    )
-            elif name_lower.endswith((".yaml", ".yml")):
-                # Check if it's a list of Pydantic models
-                if v and isinstance(v[0], BaseModel):
-                    # Convert models to dicts first
-                    v = [item.model_dump(mode="json") for item in v]
-                # Use YAML format for YAML files
-                yaml = YAML()
-                stream = BytesIO()
-                yaml.dump(v, stream)
-                v = stream.getvalue()
-            elif name_lower.endswith(".json"):
-                # Check if it's a list of Pydantic models
-                if v and isinstance(v[0], BaseModel):
-                    # Convert models to dicts first
-                    v = [item.model_dump(mode="json") for item in v]
-                # For JSON files, serialize as JSON
-                v = json.dumps(v, indent=2).encode("utf-8")
-            else:
-                # Check if it's a list of BaseModel
-                if v and isinstance(v[0], BaseModel):
-                    raise ValueError("list[BaseModel] requires .json or .yaml extension")
-                # List content not supported for other file types
-                raise ValueError(f"Unsupported content type: {type(v)} for file {name}")
-        elif isinstance(v, BaseModel):
-            # Serialize Pydantic models
-            if name_lower.endswith((".yaml", ".yml")):
-                yaml = YAML()
-                stream = BytesIO()
-                yaml.dump(v.model_dump(mode="json"), stream)
-                v = stream.getvalue()
-            else:
-                v = json.dumps(v.model_dump(mode="json"), indent=2).encode("utf-8")
-        elif isinstance(v, (int, float, bool)):
-            # Numbers and booleans: JSON-serialize for .json, string for others
-            if name_lower.endswith(".json"):
-                v = json.dumps(v).encode("utf-8")
-            elif name_lower.endswith((".yaml", ".yml")):
-                v = str(v).encode("utf-8")
-            elif name_lower.endswith(".txt"):
-                v = str(v).encode("utf-8")
-            else:
-                # For other extensions, convert to string
-                v = str(v).encode("utf-8")
-        elif v is None:
-            # Handle None - only supported for JSON/YAML
-            if name_lower.endswith((".json", ".yaml", ".yml")):
-                if name_lower.endswith((".yaml", ".yml")):
-                    v = b"null\n"
-                else:
-                    v = b"null"
-            else:
-                raise ValueError(f"Unsupported content type: {type(None)} for file {name}")
-        else:
-            # Try to see if it has model_dump (duck typing for Pydantic-like)
-            if hasattr(v, "model_dump"):
-                if name_lower.endswith((".yaml", ".yml")):
-                    yaml = YAML()
-                    stream = BytesIO()
-                    yaml.dump(v.model_dump(mode="json"), stream)  # type: ignore[attr-defined]
-                    v = stream.getvalue()
-                else:
-                    v = json.dumps(v.model_dump(mode="json"), indent=2).encode("utf-8")  # type: ignore[attr-defined]
-            else:
-                # List non-.json files should raise error
-                if name_lower.endswith(".txt") and isinstance(v, list):
-                    raise ValueError("List content not supported for text files")
-                raise ValueError(f"Unsupported content type: {type(v)}")
-
-        # Check content size limit
-        max_size = getattr(cls, "MAX_CONTENT_SIZE", 100 * 1024 * 1024)
-        if len(v) > max_size:
-            raise DocumentSizeError(
-                f"Document size ({len(v)} bytes) exceeds maximum allowed size ({max_size} bytes)"
-            )
-
+        """Convert content to bytes via `_convert_content` if not already bytes. Enforces MAX_CONTENT_SIZE."""
+        if not isinstance(v, bytes):
+            name = info.data.get("name", "") if hasattr(info, "data") else ""
+            v = _convert_content(name, v)
+        if len(v) > cls.MAX_CONTENT_SIZE:
+            raise DocumentSizeError(f"Document size ({len(v)} bytes) exceeds maximum allowed size ({cls.MAX_CONTENT_SIZE} bytes)")
         return v
 
-    @field_serializer("content")
-    def serialize_content(self, v: bytes) -> str:
-        """Pydantic serializer for content field.
+    @field_validator("sources")
+    @classmethod
+    def validate_sources(cls, v: tuple[str, ...]) -> tuple[str, ...]:
+        """Sources must be document SHA256 hashes or URLs."""
+        for src in v:
+            if not is_document_sha256(src) and "://" not in src:
+                raise ValueError(f"Source must be a document SHA256 hash or a URL (containing '://'), got: {src!r}")
+        return v
 
-        Converts bytes content to string for JSON serialization.
-        Attempts UTF-8 decoding first, falls back to base64 encoding
-        for binary content.
+    @field_validator("origins")
+    @classmethod
+    def validate_origins(cls, v: tuple[str, ...]) -> tuple[str, ...]:
+        """Origins must be valid document SHA256 hashes."""
+        for origin in v:
+            if not is_document_sha256(origin):
+                raise ValueError(f"Origin must be a document SHA256 hash, got: {origin}")
+        return v
 
-        Args:
-            v: The content bytes to serialize.
+    @model_validator(mode="after")
+    def validate_no_source_origin_overlap(self) -> Self:
+        """Reject documents where the same SHA256 appears in both sources and origins."""
+        source_sha256s = {src for src in self.sources if is_document_sha256(src)}
+        if source_sha256s:
+            overlap = source_sha256s & set(self.origins)
+            if overlap:
+                sample = next(iter(overlap))
+                raise ValueError(
+                    f"SHA256 hash {sample[:12]}... appears in both sources and origins. "
+                    f"A document reference must be either a source (content provenance) "
+                    f"or an origin (causal provenance), not both."
+                )
+        return self
 
-        Returns:
-            UTF-8 decoded string for text content,
-            base64-encoded string for binary content.
+    @model_validator(mode="after")
+    def validate_total_size(self) -> Self:
+        """Validate that total document size (content + attachments) is within limits."""
+        total = self.size
+        if total > self.MAX_CONTENT_SIZE:
+            raise DocumentSizeError(f"Total document size ({total} bytes) including attachments exceeds maximum allowed size ({self.MAX_CONTENT_SIZE} bytes)")
+        return self
 
-        Note:
-            This is called automatically by Pydantic during
-            model serialization to JSON.
-        """
+    @field_serializer("content")
+    def serialize_content(self, v: bytes) -> str:  # noqa: PLR6301
+        """UTF-8 decode for text, base64 for binary. Called by Pydantic during serialization."""
         try:
             return v.decode("utf-8")
         except UnicodeDecodeError:
@@ -741,263 +350,89 @@ class Document(BaseModel, ABC):
     @final
     @property
     def id(self) -> str:
-        """Get a short unique identifier for the document.
-
-        @public
-
-        This ID is crucial for LLM interactions. When documents are provided to
-        LLMs via generate() or generate_structured(), their IDs are included,
-        allowing the LLM to reference documents in prompts by either name or ID.
-        The ID is content-based (derived from SHA256 hash of content only),
-        so the same content always produces the same ID. Changing the name or
-        description does NOT change the ID.
-
-        Returns:
-            6-character base32-encoded string (uppercase, e.g., "A7B2C9").
-            This is the first 6 chars of the full base32 SHA256, NOT hex.
-
-        Collision Rate:
-            With base32 encoding (5 bits per char), 6 chars = 30 bits.
-            Expect collisions after ~32K documents (birthday paradox).
-            For higher uniqueness requirements, use the full sha256 property.
-
-        Note:
-            While shorter than full SHA256, this provides
-            reasonable uniqueness for most use cases.
-        """
+        """First 6 chars of sha256. Used as short document identifier in LLM context."""
         return self.sha256[:6]
 
     @final
     @cached_property
     def sha256(self) -> str:
-        """Get the full SHA256 hash of the document content.
-
-        @public
-
-        Computes and caches the SHA256 hash of the content,
-        encoded in base32 (uppercase). Used for content
-        deduplication and integrity verification.
-
-        Returns:
-            Full SHA256 hash as base32-encoded uppercase string.
-
-        Why Base32 Instead of Hex:
-            - Base32 is case-insensitive, avoiding issues with different file systems
-              and AI interactions where casing might be inconsistent
-            - More compact than hex (52 chars vs 64 chars for SHA-256)
-            - Contains more information per character than hex (5 bits vs 4 bits)
-            - Safe for URLs without encoding
-            - Compatible with case-insensitive file systems
-            - Avoids confusion in AI interactions where models might change casing
-            - Not base64 because we want consistent uppercase for all uses
-
-        Note:
-            This is computed once and cached for performance.
-            The hash is deterministic based on content only.
-        """
-        return b32encode(hashlib.sha256(self.content).digest()).decode("ascii").upper()
+        """Full SHA256 identity hash (name + content + attachments). BASE32 encoded, cached."""
+        return compute_document_sha256(self)
+
+    @final
+    @cached_property
+    def content_sha256(self) -> str:
+        """SHA256 hash of raw content bytes only. Used for content deduplication."""
+        return compute_content_sha256(self.content)
 
     @final
     @property
     def size(self) -> int:
-        """Get the size of the document content.
-
-        @public
-
-        Returns:
-            Size of content in bytes.
-
-        Note:
-            Useful for monitoring document sizes and
-            ensuring they stay within limits.
-        """
-        return len(self.content)
+        """Total size of content + attachments in bytes."""
+        return len(self.content) + sum(att.size for att in self.attachments)
 
     @cached_property
-    def detected_mime_type(self) -> str:
-        """Detect the MIME type from document content.
-
-        Detection strategy (in order):
-        1. Returns 'application/x-empty' for empty content
-        2. Extension-based detection for known text formats (preferred)
-        3. python-magic content analysis for unknown extensions
-        4. Fallback to extension or 'application/octet-stream'
-
-        Returns:
-            MIME type string (e.g., "text/plain", "application/json").
-
-        Note:
-            This is cached after first access. Extension-based detection
-            is preferred for text formats to avoid misidentification.
-        """
-        return detect_mime_type(self.content, self.name)
-
-    @property
     def mime_type(self) -> str:
-        """Get the document's MIME type.
-
-        @public
-
-        Primary property for accessing MIME type information.
-        Automatically detects MIME type based on file extension and content.
-
-        Returns:
-            MIME type string (e.g., "text/plain", "application/json").
-
-        Note:
-            MIME type detection uses extension-based detection for known
-            text formats and content analysis for binary formats.
-        """
-        return self.detected_mime_type
+        """Detected MIME type. Extension-based for known formats, content analysis for others. Cached."""
+        return detect_mime_type(self.content, self.name)
 
     @property
     def is_text(self) -> bool:
-        """Check if document contains text content.
-
-        @public
-
-        Returns:
-            True if MIME type indicates text content
-            (text/*, application/json, application/x-yaml, text/yaml, etc.),
-            False otherwise.
-
-        Note:
-            Used to determine if text property can be safely accessed.
-        """
+        """True if MIME type indicates text content."""
         return is_text_mime_type(self.mime_type)
 
     @property
     def is_pdf(self) -> bool:
-        """Check if document is a PDF file.
-
-        @public
-
-        Returns:
-            True if MIME type is application/pdf, False otherwise.
-
-        Note:
-            PDF documents require special handling and are
-            supported by certain LLM models.
-        """
+        """True if MIME type is application/pdf."""
         return is_pdf_mime_type(self.mime_type)
 
     @property
     def is_image(self) -> bool:
-        """Check if document is an image file.
-
-        @public
-
-        Returns:
-            True if MIME type starts with "image/", False otherwise.
-
-        Note:
-            Image documents are automatically encoded for
-            vision-capable LLM models.
-        """
+        """True if MIME type starts with image/."""
         return is_image_mime_type(self.mime_type)
 
     @classmethod
     def canonical_name(cls) -> str:
-        """Get the canonical name for this document class.
-
-        Returns a standardized snake_case name derived from the
-        class name, used for directory naming and identification.
-
-        Returns:
-            Snake_case canonical name.
-
-        Example:
-            >>> class UserDataDocument(FlowDocument): ...
-            >>> UserDataDocument.canonical_name()
-            'user_data'
-        """
+        """Snake_case name derived from class name, used for directory naming."""
         return canonical_name_key(cls)
 
     @property
     def text(self) -> str:
-        """Get document content as UTF-8 text string.
-
-        @public
-
-        Decodes the bytes content as UTF-8 text. Only available for
-        text-based documents (check is_text property first).
-
-        Returns:
-            UTF-8 decoded string.
-
-        Raises:
-            ValueError: If document is not text (is_text == False).
-
-        Example:
-            >>> doc = MyDocument.create(name="data.txt", content="Hello \u2728")
-            >>> if doc.is_text:
-            ...     print(doc.text)  # "Hello \u2728"
-
-            >>> # Binary document raises error:
-            >>> binary_doc = MyDocument(name="image.png", content=png_bytes)
-            >>> binary_doc.text  # Raises ValueError
-        """
+        """Content decoded as UTF-8. Raises ValueError if not text."""
         if not self.is_text:
             raise ValueError(f"Document is not text: {self.name}")
         return self.content.decode("utf-8")
 
-    def as_yaml(self) -> Any:
-        r"""Parse document content as YAML.
-
-        @public
-
-        Parses the document's text content as YAML and returns Python objects.
-        Uses ruamel.yaml which is safe by default (no code execution).
-
-        Returns:
-            Parsed YAML data: dict, list, str, int, float, bool, or None.
-
-        Raises:
-            ValueError: If document is not text-based.
-            YAMLError: If content is not valid YAML.
+    @cached_property
+    def approximate_tokens_count(self) -> int:
+        """Approximate token count (tiktoken gpt-4 encoding). Images=1080, PDFs/other=1024."""
+        enc = get_tiktoken_encoding()
+        if self.is_text:
+            total = len(enc.encode(self.text))
+        elif self.is_image:
+            total = 1080
+        else:
+            total = 1024
+
+        for att in self.attachments:
+            if att.is_image:
+                total += 1080
+            elif att.is_pdf:
+                total += 1024
+            elif att.is_text:
+                total += len(enc.encode(att.text))
+            else:
+                total += 1024
 
-        Example:
-            >>> # From dict content
-            >>> doc = MyDocument.create(name="config.yaml", content={
-            ...     "server": {"host": "localhost", "port": 8080}
-            ... })
-            >>> doc.as_yaml()  # {'server': {'host': 'localhost', 'port': 8080}}
+        return total
 
-            >>> # From YAML string
-            >>> doc2 = MyDocument(name="simple.yml", content=b"key: value\nitems:\n  - a\n  - b")
-            >>> doc2.as_yaml()  # {'key': 'value', 'items': ['a', 'b']}
-        """
+    def as_yaml(self) -> Any:
+        """Parse content as YAML via ruamel.yaml."""
         yaml = YAML()
         return yaml.load(self.text)  # type: ignore[no-untyped-call, no-any-return]
 
     def as_json(self) -> Any:
-        """Parse document content as JSON.
-
-        @public
-
-        Parses the document's text content as JSON and returns Python objects.
-        Document must contain valid JSON text.
-
-        Returns:
-            Parsed JSON data: dict, list, str, int, float, bool, or None.
-
-        Raises:
-            ValueError: If document is not text-based.
-            JSONDecodeError: If content is not valid JSON.
-
-        Example:
-            >>> # From dict content
-            >>> doc = MyDocument.create(name="data.json", content={"key": "value"})
-            >>> doc.as_json()  # {'key': 'value'}
-
-            >>> # From JSON string
-            >>> doc2 = MyDocument(name="array.json", content=b'[1, 2, 3]')
-            >>> doc2.as_json()  # [1, 2, 3]
-
-            >>> # Invalid JSON
-            >>> bad_doc = MyDocument(name="bad.json", content=b"not json")
-            >>> bad_doc.as_json()  # Raises JSONDecodeError
-        """
+        """Parse content as JSON."""
         return json.loads(self.text)
 
     @overload
@@ -1006,50 +441,8 @@ class Document(BaseModel, ABC):
     @overload
     def as_pydantic_model(self, model_type: type[list[TModel]]) -> list[TModel]: ...
 
-    def as_pydantic_model(
-        self, model_type: type[TModel] | type[list[TModel]]
-    ) -> TModel | list[TModel]:
-        """Parse document content as Pydantic model with validation.
-
-        @public
-
-        Parses JSON or YAML content and validates it against a Pydantic model.
-        Automatically detects format based on MIME type. Supports both single
-        models and lists of models.
-
-        Args:
-            model_type: Pydantic model class to validate against.
-                Can be either:
-                - type[Model] for single model
-                - type[list[Model]] for list of models
-
-        Returns:
-            Validated Pydantic model instance or list of instances.
-
-        Raises:
-            ValueError: If document is not text or type mismatch.
-            ValidationError: If data doesn't match model schema.
-            JSONDecodeError/YAMLError: If content parsing fails.
-
-        Example:
-            >>> from pydantic import BaseModel
-            >>>
-            >>> class User(BaseModel):
-            ...     name: str
-            ...     age: int
-            >>>
-            >>> # Single model
-            >>> doc = MyDocument.create(name="user.json",
-            ...     content={"name": "Alice", "age": 30})
-            >>> user = doc.as_pydantic_model(User)
-            >>> print(user.name)  # "Alice"
-            >>>
-            >>> # List of models
-            >>> doc2 = MyDocument.create(name="users.json",
-            ...     content=[{"name": "Bob", "age": 25}, {"name": "Eve", "age": 28}])
-            >>> users = doc2.as_pydantic_model(list[User])
-            >>> print(len(users))  # 2
-        """
+    def as_pydantic_model(self, model_type: type[TModel] | type[list[TModel]]) -> TModel | list[TModel]:
+        """Parse JSON/YAML content and validate against a Pydantic model. Supports single and list types."""
         data = self.as_yaml() if is_yaml_mime_type(self.mime_type) else self.as_json()
 
         if get_origin(model_type) is list:
@@ -1064,271 +457,165 @@ class Document(BaseModel, ABC):
         single_model = cast(type[TModel], model_type)
         return single_model.model_validate(data)
 
-    def as_markdown_list(self) -> list[str]:
-        r"""Parse document as markdown-separated list of sections.
-
-        @public
-
-        Splits text content using markdown separator ("\n\n-----------------\n\n").
-        Designed for markdown documents with multiple sections.
-
-        Returns:
-            List of string sections (preserves whitespace within sections).
-
-        Raises:
-            ValueError: If document is not text-based.
-
-        Example:
-            >>> # Using create with list
-            >>> sections = ["# Chapter 1\nIntroduction", "# Chapter 2\nDetails"]
-            >>> doc = MyDocument.create(name="book.md", content=sections)
-            >>> doc.as_markdown_list()  # Returns original sections
-
-            >>> # Manual creation with separator
-            >>> content = "Part 1\n\n-----------------\n\nPart 2\n\n-----------------\n\nPart 3"
-            >>> doc2 = MyDocument(name="parts.md", content=content.encode())
-            >>> doc2.as_markdown_list()  # ['Part 1', 'Part 2', 'Part 3']
-        """
-        return self.text.split(self.MARKDOWN_LIST_SEPARATOR)
+    def _parse_structured(self) -> Any:
+        """Parse content as JSON or YAML based on extension. Strict — no guessing."""
+        name_lower = self.name.lower()
+        if name_lower.endswith(".json"):
+            return self.as_json()
+        if name_lower.endswith((".yaml", ".yml")):
+            return self.as_yaml()
+        raise ValueError(f"Cannot parse '{self.name}' as structured data — use .json or .yaml extension")
 
     def parse(self, type_: type[Any]) -> Any:
-        r"""Parse document content to original type (reverses create conversion).
-
-        @public
-
-        This method reverses the automatic conversion performed by the `create`
-        classmethod. It intelligently parses the bytes content based on the
-        document's file extension and converts to the requested type.
-
-        Designed for roundtrip conversion:
-            >>> original = {"key": "value"}
-            >>> doc = MyDocument.create(name="data.json", content=original)
-            >>> restored = doc.parse(dict)
-            >>> assert restored == original  # True
-
-        Args:
-            type_: Target type to parse content into. Supported types:
-                - bytes: Returns raw content (no conversion)
-                - str: Decodes UTF-8 text
-                - dict: Parses JSON (.json) or YAML (.yaml/.yml)
-                - list: Splits markdown (.md) or parses JSON/YAML
-                - BaseModel subclasses: Validates JSON/YAML into model
-
-        Returns:
-            Content parsed to the requested type.
-
-        Raises:
-            ValueError: If type is unsupported or parsing fails.
-
-        Extension Rules:
-            - .json → JSON parsing for dict/list/BaseModel
-            - .yaml/.yml → YAML parsing for dict/list/BaseModel
-            - .md + list → Split by markdown separator
-            - Any + str → UTF-8 decode
-            - Any + bytes → Raw content
-
-        Example:
-            >>> # String content
-            >>> doc = MyDocument(name="test.txt", content=b"Hello")
-            >>> doc.parse(str)
-            'Hello'
-
-            >>> # JSON content
-            >>> doc = MyDocument.create(name="data.json", content={"key": "value"})
-            >>> doc.parse(dict)  # Returns {'key': 'value'}
-
-            >>> # Markdown list
-            >>> items = ["Item 1", "Item 2"]
-            >>> content = "\n\n---\n\n".join(items).encode()
-            >>> doc = MyDocument(name="list.md", content=content)
-            >>> doc.parse(list)
-            ['Item 1', 'Item 2']
-        """
-        # Handle basic types
+        """Parse content to the requested type. Reverses create() conversion. Extension-based dispatch, no guessing."""
         if type_ is bytes:
             return self.content
-        elif type_ is str:
-            # Handle empty content specially
-            if len(self.content) == 0:
-                return ""
-            return self.text
-
-        # Handle structured data based on extension
-        name_lower = self.name.lower()
-
-        # JSON files
-        if name_lower.endswith(".json"):
-            if type_ is dict or type_ is list:
-                result = self.as_json()
-                # Ensure the result is the correct type
-                if type_ is dict and not isinstance(result, dict):
-                    raise ValueError(f"Expected dict but got {type(result).__name__}")
-                if type_ is list and not isinstance(result, list):
-                    raise ValueError(f"Expected list but got {type(result).__name__}")
-                return result
-            elif issubclass(type_, BaseModel):
-                return self.as_pydantic_model(type_)
-            else:
-                raise ValueError(f"Cannot parse JSON file to type {type_}")
-
-        # YAML files
-        elif name_lower.endswith((".yaml", ".yml")):
-            if type_ is dict or type_ is list:
-                result = self.as_yaml()
-                # Ensure the result is the correct type
-                if type_ is dict and not isinstance(result, dict):
-                    raise ValueError(f"Expected dict but got {type(result).__name__}")
-                if type_ is list and not isinstance(result, list):
-                    raise ValueError(f"Expected list but got {type(result).__name__}")
-                return result
-            elif issubclass(type_, BaseModel):
-                return self.as_pydantic_model(type_)
-            else:
-                raise ValueError(f"Cannot parse YAML file to type {type_}")
+        if type_ is str:
+            return self.text if self.content else ""
+        if type_ is dict or type_ is list:
+            data = self._parse_structured()
+            if not isinstance(data, type_):
+                raise ValueError(f"Expected {type_.__name__} but got {type(data).__name__}")
+            return data  # pyright: ignore[reportUnknownVariableType]
+        if isinstance(type_, type) and issubclass(type_, BaseModel):  # pyright: ignore[reportUnnecessaryIsInstance]
+            return self.as_pydantic_model(type_)
+        raise ValueError(f"Unsupported parse type: {type_}")
 
-        # Markdown files with lists
-        elif name_lower.endswith(".md") and type_ is list:
-            return self.as_markdown_list()
+    @property
+    def source_documents(self) -> tuple[str, ...]:
+        """Document SHA256 hashes from sources (filtered by is_document_sha256)."""
+        return tuple(src for src in self.sources if is_document_sha256(src))
 
-        # Default: try to return as requested basic type
-        elif type_ is dict or type_ is list:
-            # Try JSON first, then YAML
-            try:
-                result = self.as_json()
-                # Ensure the result is the correct type
-                if type_ is dict and not isinstance(result, dict):
-                    raise ValueError(f"Expected dict but got {type(result).__name__}")
-                if type_ is list and not isinstance(result, list):
-                    raise ValueError(f"Expected list but got {type(result).__name__}")
-                return result
-            except (json.JSONDecodeError, ValueError):
-                try:
-                    result = self.as_yaml()
-                    # Ensure the result is the correct type
-                    if type_ is dict and not isinstance(result, dict):
-                        raise ValueError(f"Expected dict but got {type(result).__name__}")
-                    if type_ is list and not isinstance(result, list):
-                        raise ValueError(f"Expected list but got {type(result).__name__}")
-                    return result
-                except Exception as e:
-                    raise ValueError(f"Cannot parse content to {type_}") from e
-
-        raise ValueError(f"Unsupported type {type_} for file {self.name}")
+    @property
+    def source_references(self) -> tuple[str, ...]:
+        """Non-hash reference strings from sources (URLs, file paths, etc.)."""
+        return tuple(src for src in self.sources if not is_document_sha256(src))
+
+    def has_source(self, source: "Document | str") -> bool:
+        """Check if a source (Document or string) is in this document's sources."""
+        if isinstance(source, str):
+            return source in self.sources
+        if isinstance(source, Document):  # type: ignore[misc]
+            return source.sha256 in self.sources
+        raise TypeError(f"Invalid source type: {type(source)}")  # pyright: ignore[reportUnreachable]
 
     @final
     def serialize_model(self) -> dict[str, Any]:
-        """Serialize document to dictionary for storage or transmission.
-
-        Creates a complete JSON-serializable representation of the document
-        with all metadata and properly encoded content. Automatically chooses
-        the most appropriate encoding (UTF-8 for text, base64 for binary).
-
-        Returns:
-            Dictionary with the following keys:
-                - name: Document filename (str)
-                - description: Optional description (str | None)
-                - base_type: Persistence type - "flow", "task", or "temporary" (str)
-                - size: Content size in bytes (int)
-                - id: Short hash identifier, first 6 chars of SHA256 (str)
-                - sha256: Full SHA256 hash in base32 encoding (str)
-                - mime_type: Detected MIME type (str)
-                - content: Encoded content (str)
-                - content_encoding: Either "utf-8" or "base64" (str)
-
-        Encoding Strategy:
-            - Text files (text/*, application/json, etc.) → UTF-8 string
-            - Binary files (images, PDFs, etc.) → Base64 string
-            - Invalid UTF-8 in text files → UTF-8 with replacement chars
-
-        Example:
-            >>> doc = MyDocument.create(name="data.json", content={"key": "value"})
-            >>> serialized = doc.serialize_model()
-            >>> serialized["content_encoding"]  # "utf-8"
-            >>> serialized["mime_type"]  # "application/json"
-        """
-        result = {
+        """Serialize to JSON-compatible dict for storage/transmission. Roundtrips with from_dict()."""
+        result: dict[str, Any] = {  # nosemgrep: mutable-field-on-frozen-pydantic-model
             "name": self.name,
             "description": self.description,
-            "base_type": self.get_base_type(),
             "size": self.size,
             "id": self.id,
             "sha256": self.sha256,
+            "content_sha256": self.content_sha256,
             "mime_type": self.mime_type,
+            "sources": list(self.sources),
+            "origins": list(self.origins),
+            "canonical_name": canonical_name_key(self.__class__),
+            "class_name": self.__class__.__name__,
         }
 
-        # Try to encode content as UTF-8, fall back to base64
-        if self.is_text or self.mime_type.startswith("text/"):
+        if self.is_text:
             try:
                 result["content"] = self.content.decode("utf-8")
                 result["content_encoding"] = "utf-8"
             except UnicodeDecodeError:
-                # For text files with encoding issues, use UTF-8 with replacement
                 result["content"] = self.content.decode("utf-8", errors="replace")
                 result["content_encoding"] = "utf-8"
         else:
-            # Binary content - use base64
             result["content"] = base64.b64encode(self.content).decode("ascii")
             result["content_encoding"] = "base64"
 
+        serialized_attachments: list[dict[str, Any]] = []  # nosemgrep: mutable-field-on-frozen-pydantic-model
+        for att in self.attachments:
+            att_data: dict[str, Any] = {"name": att.name, "description": att.description}  # nosemgrep: mutable-field-on-frozen-pydantic-model
+            if att.is_text:
+                att_data["content"] = att.content.decode("utf-8", errors="replace")
+                att_data["content_encoding"] = "utf-8"
+            else:
+                att_data["content"] = base64.b64encode(att.content).decode("ascii")
+                att_data["content_encoding"] = "base64"
+            att_data["mime_type"] = att.mime_type
+            att_data["size"] = att.size
+            serialized_attachments.append(att_data)
+        result["attachments"] = serialized_attachments
+
         return result
 
     @final
     @classmethod
     def from_dict(cls, data: dict[str, Any]) -> Self:
-        r"""Deserialize document from dictionary (inverse of serialize_model).
-
-        Reconstructs a Document instance from the dictionary format produced
-        by serialize_model(). Automatically handles content decoding based on
-        the content_encoding field.
-
-        Args:
-            data: Dictionary containing serialized document. Required keys:
-                - name: Document filename (str)
-                - content: Encoded content (str or bytes)
-                Optional keys:
-                - description: Document description (str | None)
-                - content_encoding: "utf-8" or "base64" (defaults to "utf-8")
-
-        Returns:
-            New Document instance with restored content.
-
-        Raises:
-            ValueError: If content type is invalid or base64 decoding fails
-            KeyError: If required keys are missing from data dictionary
-
-        Note:
-            Provides roundtrip guarantee with serialize_model().
-            Content and name are preserved exactly.
-
-        Example:
-            >>> data = {
-            ...     "name": "config.yaml",
-            ...     "content": "key: value\n",
-            ...     "content_encoding": "utf-8",
-            ...     "description": "Config file"
-            ... }
-            >>> doc = MyDocument.from_dict(data)
-        """
-        # Extract content and encoding
+        """Deserialize from dict produced by serialize_model(). Roundtrip guarantee."""
         content_raw = data.get("content", "")
         content_encoding = data.get("content_encoding", "utf-8")
 
-        # Decode content based on encoding
         content: bytes
         if content_encoding == "base64":
-            assert isinstance(content_raw, str), "base64 content must be string"
+            if not isinstance(content_raw, str):
+                raise ValueError("base64 content must be string")
             content = base64.b64decode(content_raw)
         elif isinstance(content_raw, str):
-            # Default to UTF-8
             content = content_raw.encode("utf-8")
         elif isinstance(content_raw, bytes):
             content = content_raw
         else:
             raise ValueError(f"Invalid content type: {type(content_raw)}")
 
-        # Create document with the required fields
+        attachments: tuple[Attachment, ...] | None = None
+        if attachments_raw := data.get("attachments"):
+            att_list: list[Attachment] = []  # nosemgrep: mutable-field-on-frozen-pydantic-model
+            for att_data in attachments_raw:
+                att_content_raw = att_data.get("content", "")
+                if att_data.get("content_encoding") == "base64":
+                    att_content = base64.b64decode(att_content_raw)
+                elif isinstance(att_content_raw, str):
+                    att_content = att_content_raw.encode("utf-8")
+                else:
+                    att_content = att_content_raw
+                att_list.append(Attachment(name=att_data["name"], content=att_content, description=att_data.get("description")))
+            attachments = tuple(att_list)
+
         return cls(
             name=data["name"],
             content=content,
             description=data.get("description"),
+            sources=tuple(data.get("sources") or ()),
+            origins=tuple(data.get("origins") or ()),
+            attachments=attachments,
+        )
+
+    @final
+    def model_convert(self, new_type: type[TDocument], *, update: dict[str, Any] | None = None) -> TDocument:
+        """Convert to a different Document subclass with optional field overrides."""
+        try:
+            if not isinstance(new_type, type):  # pyright: ignore[reportUnnecessaryIsInstance]
+                raise TypeError(f"new_type must be a class, got {new_type}")  # pyright: ignore[reportUnreachable]
+            if not issubclass(new_type, Document):  # pyright: ignore[reportUnnecessaryIsInstance]
+                raise TypeError(f"new_type must be a subclass of Document, got {new_type}")  # pyright: ignore[reportUnreachable]
+        except (TypeError, AttributeError) as err:
+            raise TypeError(f"new_type must be a subclass of Document, got {new_type}") from err
+
+        if new_type is Document:
+            raise TypeError("Cannot instantiate Document directly — use a concrete subclass")
+
+        data: dict[str, Any] = {  # nosemgrep: mutable-field-on-frozen-pydantic-model
+            "name": self.name,
+            "content": self.content,
+            "description": self.description,
+            "sources": self.sources,
+            "origins": self.origins,
+            "attachments": self.attachments,
+        }
+
+        if update:
+            data.update(update)
+
+        return new_type(
+            name=data["name"],
+            content=data["content"],
+            description=data.get("description"),
+            sources=data.get("sources"),
+            origins=data.get("origins"),
+            attachments=data.get("attachments"),
         )