chunk-metadata-adapter 1.0.0__py3-none-any.whl

chunk_metadata_adapter/__init__.py

@@ -0,0 +1,30 @@
+"""
+Chunk Metadata Adapter - A package for managing metadata for chunked content.
+
+This package provides tools for creating, managing, and converting metadata 
+for chunks of content in various systems, including RAG pipelines, document 
+processing, and machine learning training datasets.
+"""
+
+from .metadata_builder import ChunkMetadataBuilder
+from .models import (
+    SemanticChunk,
+    FlatSemanticChunk,
+    ChunkType,
+    ChunkRole,
+    ChunkStatus,
+    ChunkMetrics,
+    FeedbackMetrics
+)
+
+__version__ = "1.0.0"
+__all__ = [
+    "ChunkMetadataBuilder",
+    "SemanticChunk",
+    "FlatSemanticChunk",
+    "ChunkType",
+    "ChunkRole",
+    "ChunkStatus",
+    "ChunkMetrics",
+    "FeedbackMetrics",
+]

chunk_metadata_adapter/examples.py

@@ -0,0 +1,173 @@
+"""
+Examples of how to use the Chunk Metadata Adapter.
+
+This module contains practical examples for various use cases.
+"""
+import uuid
+
+from chunk_metadata_adapter import (
+    ChunkMetadataBuilder,
+    SemanticChunk,
+    FlatSemanticChunk,
+    ChunkType,
+    ChunkRole,
+    ChunkStatus
+)
+
+
+def example_basic_flat_metadata():
+    """Example of creating basic flat metadata for a chunk."""
+    # Create a builder instance for a specific project
+    builder = ChunkMetadataBuilder(project="MyProject", unit_id="chunker-service-1")
+    
+    # Generate UUID for the source document
+    source_id = str(uuid.uuid4())
+    
+    # Create metadata for a piece of code
+    metadata = builder.build_flat_metadata(
+        text="def hello_world():\n    print('Hello, World!')",
+        source_id=source_id,
+        ordinal=1,  # First chunk in the document
+        type=ChunkType.CODE_BLOCK,  # Using enum
+        language="python",
+        source_path="src/hello.py",
+        source_lines_start=10,
+        source_lines_end=12,
+        tags="example,hello",
+        role=ChunkRole.DEVELOPER
+    )
+    
+    # Access the metadata
+    print(f"Generated UUID: {metadata['uuid']}")
+    print(f"SHA256: {metadata['sha256']}")
+    print(f"Created at: {metadata['created_at']}")
+    
+    return metadata
+
+
+def example_structured_chunk():
+    """Example of creating a structured SemanticChunk instance."""
+    # Create a builder for a project with task
+    builder = ChunkMetadataBuilder(
+        project="DocumentationProject",
+        unit_id="docs-generator"
+    )
+    
+    # Generate a source document ID
+    source_id = str(uuid.uuid4())
+    
+    # Create a structured chunk 
+    chunk = builder.build_semantic_chunk(
+        text="# Introduction\n\nThis is the documentation for the system.",
+        language="markdown",
+        type=ChunkType.DOC_BLOCK,
+        source_id=source_id,
+        summary="Project introduction section",
+        role=ChunkRole.DEVELOPER,
+        source_path="docs/intro.md",
+        source_lines=[1, 3],
+        ordinal=0,
+        task_id="DOC-123",
+        subtask_id="DOC-123-A",
+        tags=["introduction", "documentation", "overview"],
+        links=[f"parent:{str(uuid.uuid4())}"]
+    )
+    
+    # Access the data
+    print(f"Chunk UUID: {chunk.uuid}")
+    print(f"Content summary: {chunk.summary}")
+    print(f"Links: {chunk.links}")
+    
+    return chunk
+
+
+def example_conversion_between_formats():
+    """Example of converting between structured and flat formats."""
+    # Create a builder instance
+    builder = ChunkMetadataBuilder(project="ConversionExample")
+    
+    # Start with a structured chunk
+    structured_chunk = builder.build_semantic_chunk(
+        text="This is a sample chunk for conversion demonstration.",
+        language="text",
+        type=ChunkType.COMMENT,
+        source_id=str(uuid.uuid4()),
+        role=ChunkRole.REVIEWER
+    )
+    
+    # Convert to flat dictionary
+    flat_dict = builder.semantic_to_flat(structured_chunk)
+    print(f"Flat representation has {len(flat_dict)} fields")
+    
+    # Convert back to structured format
+    restored_chunk = builder.flat_to_semantic(flat_dict)
+    print(f"Restored structured chunk: {restored_chunk.uuid}")
+    
+    # Verify they're equivalent
+    assert restored_chunk.uuid == structured_chunk.uuid
+    assert restored_chunk.text == structured_chunk.text
+    assert restored_chunk.type == structured_chunk.type
+    
+    return {
+        "original": structured_chunk,
+        "flat": flat_dict,
+        "restored": restored_chunk
+    }
+
+
+def example_chain_processing():
+    """Example of a chain of processing for document chunks."""
+    # Create a document with multiple chunks
+    builder = ChunkMetadataBuilder(project="ChainExample", unit_id="processor")
+    source_id = str(uuid.uuid4())
+    
+    # Create a sequence of chunks from a document
+    chunks = []
+    for i, text in enumerate([
+        "# Document Title",
+        "## Section 1\n\nThis is the content of section 1.",
+        "## Section 2\n\nThis is the content of section 2.",
+        "## Conclusion\n\nFinal thoughts on the topic."
+    ]):
+        chunk = builder.build_semantic_chunk(
+            text=text,
+            language="markdown",
+            type=ChunkType.DOC_BLOCK,
+            source_id=source_id,
+            ordinal=i,
+            summary=f"Section {i}" if i > 0 else "Title"
+        )
+        chunks.append(chunk)
+    
+    # Create explicit links between chunks (parent-child relationships)
+    for i in range(1, len(chunks)):
+        # Add parent link to the title chunk
+        chunks[i].links.append(f"parent:{chunks[0].uuid}")
+        # Update status to show progress
+        chunks[i].status = ChunkStatus.INDEXED
+    
+    # Simulate processing and updating metrics
+    for chunk in chunks:
+        # Update metrics based on some processing
+        chunk.metrics.quality_score = 0.95
+        chunk.metrics.used_in_generation = True
+        chunk.metrics.matches = 3
+        
+        # Add feedback
+        chunk.metrics.feedback.accepted = 2
+        
+    # Print the processed chain
+    print(f"Processed {len(chunks)} chunks from source {source_id}")
+    for i, chunk in enumerate(chunks):
+        print(f"Chunk {i}: {chunk.summary} - Status: {chunk.status.value}")
+    
+    return chunks
+
+
+if __name__ == "__main__":
+    print("Running examples...")
+    example_basic_flat_metadata()
+    example_structured_chunk()
+    example_conversion_between_formats()
+    example_chain_processing()
+    print("All examples completed.") 

chunk_metadata_adapter/metadata_builder.py

@@ -0,0 +1,249 @@
+"""
+Builder for chunk metadata.
+
+This module provides tools for creating and manipulating chunk metadata
+for semantic chunking systems.
+"""
+import uuid
+import hashlib
+from typing import Optional, Dict, Any, List, Union
+from datetime import datetime, timezone
+
+from .models import (
+    SemanticChunk, 
+    FlatSemanticChunk, 
+    ChunkType, 
+    ChunkRole, 
+    ChunkStatus,
+    ChunkMetrics,
+    FeedbackMetrics
+)
+
+
+class ChunkMetadataBuilder:
+    """
+    Builder for universal chunk metadata.
+    
+    Used after chunk text is formed to augment it with metadata fields:
+    - uuid, sha256, created_at
+    - project, status, role, tags, etc.
+    
+    Supports both flat and structured formats.
+    """
+    def __init__(
+        self, 
+        project: Optional[str] = None, 
+        unit_id: Optional[str] = None,
+        chunking_version: str = "1.0"
+    ):
+        """
+        Initialize a new metadata builder.
+        
+        Args:
+            project: Optional project identifier
+            unit_id: Optional identifier for the chunking unit/service
+            chunking_version: Version of chunking algorithm used
+        """
+        self.project = project
+        self.unit_id = unit_id or "chunker"
+        self.chunking_version = chunking_version
+
+    def generate_uuid(self) -> str:
+        """Generate a new UUIDv4 string"""
+        return str(uuid.uuid4())
+
+    def compute_sha256(self, text: str) -> str:
+        """Compute SHA256 hash of the given text"""
+        return hashlib.sha256(text.encode("utf-8")).hexdigest()
+        
+    def _get_iso_timestamp(self) -> str:
+        """Get current timestamp in ISO8601 format with UTC timezone"""
+        return datetime.now(timezone.utc).isoformat()
+
+    def build_flat_metadata(
+        self, *,
+        text: str,
+        source_id: str,
+        ordinal: int,
+        type: Union[str, ChunkType],
+        language: str,
+        source_path: Optional[str] = None,
+        source_lines_start: Optional[int] = None,
+        source_lines_end: Optional[int] = None,
+        summary: Optional[str] = None,
+        tags: Optional[str] = None,
+        role: Optional[Union[str, ChunkRole]] = None,
+        task_id: Optional[str] = None,
+        subtask_id: Optional[str] = None,
+        link_parent: Optional[str] = None,
+        link_related: Optional[str] = None,
+        status: Union[str, ChunkStatus] = ChunkStatus.NEW,
+    ) -> Dict[str, Any]:
+        """
+        Build chunk metadata in a flat dictionary format.
+        
+        Args:
+            text: The text content of the chunk
+            source_id: Identifier of the source (file, document, dialogue)
+            ordinal: Order of the chunk within the source
+            type: Type of the chunk (see ChunkType)
+            language: Programming or natural language of the content
+            source_path: Optional path to the source file
+            source_lines_start: Start line in source file
+            source_lines_end: End line in source file
+            summary: Brief summary of the chunk content
+            tags: Comma-separated tags
+            role: Role of the content creator
+            task_id: Task identifier
+            subtask_id: Subtask identifier
+            link_parent: UUID of the parent chunk
+            link_related: UUID of a related chunk
+            status: Processing status of the chunk
+            
+        Returns:
+            Dictionary with flat metadata
+        """
+        # Verify UUIDs
+        if not isinstance(source_id, str) or not uuid.UUID(source_id, version=4):
+            raise ValueError(f"source_id must be a valid UUIDv4 string: {source_id}")
+            
+        if link_parent is not None and (not isinstance(link_parent, str) or not uuid.UUID(link_parent, version=4)):
+            raise ValueError(f"link_parent must be a valid UUIDv4 string: {link_parent}")
+            
+        if link_related is not None and (not isinstance(link_related, str) or not uuid.UUID(link_related, version=4)):
+            raise ValueError(f"link_related must be a valid UUIDv4 string: {link_related}")
+        
+        # Convert enum types to string values if needed
+        if isinstance(type, ChunkType):
+            type = type.value
+        if isinstance(role, ChunkRole):
+            role = role.value
+        if isinstance(status, ChunkStatus):
+            status = status.value
+            
+        return {
+            "uuid": self.generate_uuid(),
+            "source_id": source_id,
+            "ordinal": ordinal,
+            "sha256": self.compute_sha256(text),
+            "text": text,
+            "summary": summary,
+            "language": language,
+            "type": type,
+            "source_path": source_path,
+            "source_lines_start": source_lines_start,
+            "source_lines_end": source_lines_end,
+            "project": self.project,
+            "task_id": task_id,
+            "subtask_id": subtask_id,
+            "status": status,
+            "unit_id": self.unit_id,
+            "created_at": self._get_iso_timestamp(),
+            "tags": tags,
+            "role": role,
+            "link_parent": link_parent,
+            "link_related": link_related,
+            "quality_score": None,
+            "used_in_generation": False,
+            "feedback_accepted": 0,
+            "feedback_rejected": 0
+        }
+        
+    # Alias for backward compatibility
+    build_metadata = build_flat_metadata
+
+    def build_semantic_chunk(
+        self, *,
+        text: str,
+        language: str,
+        type: Union[str, ChunkType],
+        source_id: Optional[str] = None,
+        summary: Optional[str] = None,
+        role: Optional[Union[str, ChunkRole]] = None,
+        source_path: Optional[str] = None,
+        source_lines: Optional[List[int]] = None,
+        ordinal: Optional[int] = None,
+        task_id: Optional[str] = None,
+        subtask_id: Optional[str] = None,
+        links: Optional[List[str]] = None,
+        tags: Optional[List[str]] = None,
+        status: Union[str, ChunkStatus] = ChunkStatus.NEW,
+    ) -> SemanticChunk:
+        """
+        Build a fully-structured SemanticChunk object.
+        
+        Args:
+            text: The text content of the chunk
+            language: Programming or natural language of the content
+            type: Type of the chunk
+            source_id: Optional identifier of the source
+            summary: Brief summary of the chunk content
+            role: Role of the content creator
+            source_path: Optional path to the source file
+            source_lines: List of line numbers [start, end]
+            ordinal: Order of the chunk within the source
+            task_id: Task identifier
+            subtask_id: Subtask identifier
+            links: List of links to other chunks (format: "relation:uuid")
+            tags: List of tags
+            status: Processing status of the chunk
+            
+        Returns:
+            Fully populated SemanticChunk instance
+        """
+        # Verify UUIDs
+        if source_id is not None and (not isinstance(source_id, str) or not uuid.UUID(source_id, version=4)):
+            raise ValueError(f"source_id must be a valid UUIDv4 string: {source_id}")
+            
+        # Validate links format and UUIDs
+        if links:
+            for link in links:
+                parts = link.split(":", 1)
+                if len(parts) != 2 or not parts[0] or not parts[1]:
+                    raise ValueError(f"Link must follow 'relation:uuid' format: {link}")
+                try:
+                    uuid.UUID(parts[1], version=4)
+                except (ValueError, AttributeError):
+                    raise ValueError(f"Invalid UUID4 in link: {link}")
+        
+        # Convert string types to enums if needed
+        if isinstance(type, str):
+            type = ChunkType(type)
+        if isinstance(role, str) and role:
+            role = ChunkRole(role)
+        if isinstance(status, str):
+            status = ChunkStatus(status)
+            
+        return SemanticChunk(
+            uuid=self.generate_uuid(),
+            source_id=source_id,
+            project=self.project,
+            task_id=task_id,
+            subtask_id=subtask_id,
+            unit_id=self.unit_id,
+            type=type,
+            role=role,
+            language=language,
+            text=text,
+            summary=summary,
+            source_path=source_path,
+            source_lines=source_lines,
+            ordinal=ordinal,
+            sha256=self.compute_sha256(text),
+            chunking_version=self.chunking_version,
+            status=status,
+            links=links or [],
+            tags=tags or [],
+            metrics=ChunkMetrics(),
+            created_at=self._get_iso_timestamp()
+        )
+
+    def flat_to_semantic(self, flat_chunk: Dict[str, Any]) -> SemanticChunk:
+        """Convert flat dictionary metadata to SemanticChunk model"""
+        flat_model = FlatSemanticChunk(**flat_chunk)
+        return flat_model.to_semantic_chunk()
+    
+    def semantic_to_flat(self, chunk: SemanticChunk) -> Dict[str, Any]:
+        """Convert SemanticChunk model to flat dictionary format"""
+        flat_model = FlatSemanticChunk.from_semantic_chunk(chunk)
+        return flat_model.model_dump()

chunk_metadata_adapter/models.py

@@ -0,0 +1,364 @@
+"""
+Models for chunk metadata representation using Pydantic.
+"""
+from enum import Enum
+from typing import List, Dict, Optional, Union, Any, Pattern
+import re
+import uuid
+from datetime import datetime, timezone
+from pydantic import BaseModel, Field, validator, field_validator
+
+
+# UUID4 регулярное выражение для валидации
+UUID4_PATTERN: Pattern = re.compile(
+    r'^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$',
+    re.IGNORECASE
+)
+
+# ISO 8601 с таймзоной
+ISO8601_PATTERN: Pattern = re.compile(
+    r'^([0-9]{4})-(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])T([2][0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(\.[0-9]+)?(Z|[+-][0-9]{2}:[0-9]{2})$'
+)
+
+
+class ChunkType(str, Enum):
+    """Types of semantic chunks"""
+    DOC_BLOCK = "DocBlock"
+    CODE_BLOCK = "CodeBlock"
+    MESSAGE = "Message"
+    DRAFT = "Draft"
+    TASK = "Task"
+    SUBTASK = "Subtask"
+    TZ = "TZ"
+    COMMENT = "Comment"
+    LOG = "Log"
+    METRIC = "Metric"
+
+
+class ChunkRole(str, Enum):
+    """Roles in the system"""
+    SYSTEM = "system"
+    USER = "user"
+    ASSISTANT = "assistant"
+    TOOL = "tool"
+    REVIEWER = "reviewer"
+    DEVELOPER = "developer"
+
+
+class ChunkStatus(str, Enum):
+    """Status of a chunk processing"""
+    NEW = "new"
+    INDEXED = "indexed"
+    VERIFIED = "verified"
+    OBSOLETE = "obsolete"
+    REJECTED = "rejected"
+    IN_PROGRESS = "in_progress"
+
+
+class FeedbackMetrics(BaseModel):
+    """Feedback metrics for a chunk"""
+    accepted: int = Field(default=0, description="How many times the chunk was accepted")
+    rejected: int = Field(default=0, description="How many times the chunk was rejected")
+    modifications: int = Field(default=0, description="Number of modifications made after generation")
+
+
+class ChunkMetrics(BaseModel):
+    """Metrics related to chunk quality and usage"""
+    quality_score: Optional[float] = Field(default=None, ge=0, le=1, description="Quality score between 0 and 1")
+    coverage: Optional[float] = Field(default=None, ge=0, le=1, description="Coverage score between 0 and 1")
+    matches: Optional[int] = Field(default=None, ge=0, description="How many times matched in retrieval")
+    used_in_generation: bool = Field(default=False, description="Whether used in generation")
+    used_as_input: bool = Field(default=False, description="Whether used as input")
+    used_as_context: bool = Field(default=False, description="Whether used as context")
+    feedback: FeedbackMetrics = Field(default_factory=FeedbackMetrics, description="Feedback metrics")
+
+
+class SemanticChunk(BaseModel):
+    """
+    Main model representing a universal semantic chunk with metadata.
+    
+    This is the full, structured representation based on the specification.
+    """
+    uuid: str = Field(..., description="Unique identifier (UUIDv4)")
+    type: ChunkType = Field(..., description="Type of chunk")
+    role: Optional[ChunkRole] = Field(default=None, description="Role of the creator")
+    project: Optional[str] = Field(default=None, description="Project identifier")
+    task_id: Optional[str] = Field(default=None, description="Task identifier")
+    subtask_id: Optional[str] = Field(default=None, description="Subtask identifier")
+    unit_id: Optional[str] = Field(default=None, description="Unit/service identifier")
+    
+    text: str = Field(..., description="Chunk content")
+    summary: Optional[str] = Field(default=None, description="Brief summary")
+    language: str = Field(..., description="Content language/format")
+    
+    source_id: Optional[str] = Field(default=None, description="Source identifier (UUIDv4)")
+    source_path: Optional[str] = Field(default=None, description="Path to source")
+    source_lines: Optional[List[int]] = Field(default=None, description="Source line numbers")
+    ordinal: Optional[int] = Field(default=None, ge=0, description="Order in source")
+    
+    created_at: str = Field(default_factory=lambda: datetime.now(timezone.utc).isoformat(), description="Creation timestamp (ISO8601)")
+    status: ChunkStatus = Field(default=ChunkStatus.NEW, description="Processing status")
+    chunking_version: Optional[str] = Field(default="1.0", description="Chunking algorithm version")
+    
+    sha256: str = Field(..., description="Content hash for integrity")
+    embedding: Optional[Any] = Field(default=None, description="Vector embedding")
+    
+    links: List[str] = Field(default_factory=list, description="References to other chunks")
+    tags: List[str] = Field(default_factory=list, description="Categorical tags")
+    
+    metrics: ChunkMetrics = Field(default_factory=ChunkMetrics, description="Quality and usage metrics")
+    
+    @field_validator('uuid')
+    @classmethod
+    def validate_uuid(cls, v: str) -> str:
+        """Validate UUID format"""
+        if not UUID4_PATTERN.match(v):
+            try:
+                # Try to parse and see if it's a valid UUID
+                uuid_obj = uuid.UUID(v, version=4)
+                if str(uuid_obj) != v.lower():
+                    raise ValueError("UUID version or format doesn't match")
+            except (ValueError, AttributeError):
+                raise ValueError(f"Invalid UUID4 format: {v}")
+        return v
+    
+    @field_validator('source_id')
+    @classmethod
+    def validate_source_id(cls, v: Optional[str]) -> Optional[str]:
+        """Validate source_id UUID format if present"""
+        if v is not None and not UUID4_PATTERN.match(v):
+            try:
+                uuid_obj = uuid.UUID(v, version=4)
+                if str(uuid_obj) != v.lower():
+                    raise ValueError("Source ID UUID version or format doesn't match")
+            except (ValueError, AttributeError):
+                raise ValueError(f"Invalid source_id UUID4 format: {v}")
+        return v
+    
+    @field_validator('created_at')
+    @classmethod
+    def validate_created_at(cls, v: str) -> str:
+        """Validate that created_at is in ISO8601 format with timezone"""
+        if not ISO8601_PATTERN.match(v):
+            try:
+                # Try to parse and see if it's valid ISO format
+                dt = datetime.fromisoformat(v.replace('Z', '+00:00'))
+                # Ensure timezone is present
+                if dt.tzinfo is None:
+                    raise ValueError("Missing timezone information")
+                # Return canonicalized format
+                return dt.isoformat()
+            except (ValueError, AttributeError):
+                raise ValueError(f"Invalid ISO8601 format with timezone: {v}")
+        return v
+    
+    @field_validator('links')
+    @classmethod
+    def validate_links(cls, links: List[str]) -> List[str]:
+        """Validate that links follow the format 'relation:uuid'"""
+        for link in links:
+            parts = link.split(":", 1)
+            if len(parts) != 2 or not parts[0] or not parts[1]:
+                raise ValueError(f"Link must follow 'relation:uuid' format: {link}")
+            
+            # Validate that UUID part is valid
+            uuid_part = parts[1]
+            if not UUID4_PATTERN.match(uuid_part):
+                try:
+                    uuid_obj = uuid.UUID(uuid_part, version=4)
+                    if str(uuid_obj) != uuid_part.lower():
+                        raise ValueError("Link UUID version or format doesn't match")
+                except (ValueError, AttributeError):
+                    raise ValueError(f"Invalid UUID4 format in link: {link}")
+        return links
+
+
+class FlatSemanticChunk(BaseModel):
+    """
+    Flat representation of the semantic chunk with all fields in a flat structure.
+    
+    Used for storage systems that prefer flat structures.
+    """
+    # Core identifiers
+    uuid: str
+    source_id: Optional[str] = None
+    project: Optional[str] = None
+    task_id: Optional[str] = None
+    subtask_id: Optional[str] = None
+    unit_id: Optional[str] = None
+    
+    # Content info
+    type: str
+    role: Optional[str] = None
+    language: str
+    text: str
+    summary: Optional[str] = None
+    
+    # Source tracking
+    ordinal: Optional[int] = None
+    sha256: str
+    created_at: str
+    status: str = ChunkStatus.NEW.value
+    source_path: Optional[str] = None
+    source_lines_start: Optional[int] = None
+    source_lines_end: Optional[int] = None
+    
+    # Tags & Links
+    tags: Optional[str] = None  # Comma-separated tags
+    link_related: Optional[str] = None
+    link_parent: Optional[str] = None
+    
+    # Flat metrics
+    quality_score: Optional[float] = None
+    used_in_generation: bool = False
+    feedback_accepted: int = 0
+    feedback_rejected: int = 0
+    
+    @field_validator('uuid')
+    @classmethod
+    def validate_uuid(cls, v: str) -> str:
+        """Validate UUID format"""
+        if not UUID4_PATTERN.match(v):
+            try:
+                uuid_obj = uuid.UUID(v, version=4)
+                if str(uuid_obj) != v.lower():
+                    raise ValueError("UUID version or format doesn't match")
+            except (ValueError, AttributeError):
+                raise ValueError(f"Invalid UUID4 format: {v}")
+        return v
+    
+    @field_validator('source_id')
+    @classmethod
+    def validate_source_id(cls, v: Optional[str]) -> Optional[str]:
+        """Validate source_id UUID format if present"""
+        if v is not None and not UUID4_PATTERN.match(v):
+            try:
+                uuid_obj = uuid.UUID(v, version=4)
+                if str(uuid_obj) != v.lower():
+                    raise ValueError("Source ID UUID version or format doesn't match")
+            except (ValueError, AttributeError):
+                raise ValueError(f"Invalid source_id UUID4 format: {v}")
+        return v
+    
+    @field_validator('created_at')
+    @classmethod
+    def validate_created_at(cls, v: str) -> str:
+        """Validate that created_at is in ISO8601 format with timezone"""
+        if not ISO8601_PATTERN.match(v):
+            try:
+                dt = datetime.fromisoformat(v.replace('Z', '+00:00'))
+                if dt.tzinfo is None:
+                    raise ValueError("Missing timezone information")
+                return dt.isoformat()
+            except (ValueError, AttributeError):
+                raise ValueError(f"Invalid ISO8601 format with timezone: {v}")
+        return v
+    
+    @field_validator('link_related', 'link_parent')
+    @classmethod
+    def validate_link_uuid(cls, v: Optional[str]) -> Optional[str]:
+        """Validate link UUIDs if present"""
+        if v is not None and not UUID4_PATTERN.match(v):
+            try:
+                uuid_obj = uuid.UUID(v, version=4)
+                if str(uuid_obj) != v.lower():
+                    raise ValueError("Link UUID version or format doesn't match")
+            except (ValueError, AttributeError):
+                raise ValueError(f"Invalid UUID4 format in link: {v}")
+        return v
+    
+    @classmethod
+    def from_semantic_chunk(cls, chunk: SemanticChunk) -> 'FlatSemanticChunk':
+        """Convert a full SemanticChunk to flat representation"""
+        source_lines_start = None
+        source_lines_end = None
+        if chunk.source_lines and len(chunk.source_lines) >= 2:
+            source_lines_start = chunk.source_lines[0]
+            source_lines_end = chunk.source_lines[1]
+            
+        # Extract link references
+        link_parent = None
+        link_related = None
+        for link in chunk.links:
+            if link.startswith("parent:"):
+                link_parent = link.split(":", 1)[1]
+            elif link.startswith("related:"):
+                link_related = link.split(":", 1)[1]
+                
+        return cls(
+            uuid=chunk.uuid,
+            source_id=chunk.source_id,
+            project=chunk.project,
+            task_id=chunk.task_id,
+            subtask_id=chunk.subtask_id,
+            unit_id=chunk.unit_id,
+            type=chunk.type.value,
+            role=chunk.role.value if chunk.role else None,
+            language=chunk.language,
+            text=chunk.text,
+            summary=chunk.summary,
+            ordinal=chunk.ordinal,
+            sha256=chunk.sha256,
+            created_at=chunk.created_at,
+            status=chunk.status.value,
+            source_path=chunk.source_path,
+            source_lines_start=source_lines_start,
+            source_lines_end=source_lines_end,
+            tags=",".join(chunk.tags) if chunk.tags else None,
+            link_related=link_related,
+            link_parent=link_parent,
+            quality_score=chunk.metrics.quality_score,
+            used_in_generation=chunk.metrics.used_in_generation,
+            feedback_accepted=chunk.metrics.feedback.accepted,
+            feedback_rejected=chunk.metrics.feedback.rejected
+        )
+        
+    def to_semantic_chunk(self) -> SemanticChunk:
+        """Convert flat representation to full SemanticChunk"""
+        # Prepare links
+        links = []
+        if self.link_parent:
+            links.append(f"parent:{self.link_parent}")
+        if self.link_related:
+            links.append(f"related:{self.link_related}")
+            
+        # Prepare tags
+        tags = self.tags.split(",") if self.tags else []
+        
+        # Prepare source lines
+        source_lines = None
+        if self.source_lines_start is not None and self.source_lines_end is not None:
+            source_lines = [self.source_lines_start, self.source_lines_end]
+            
+        # Prepare metrics
+        metrics = ChunkMetrics(
+            quality_score=self.quality_score,
+            used_in_generation=self.used_in_generation,
+            feedback=FeedbackMetrics(
+                accepted=self.feedback_accepted,
+                rejected=self.feedback_rejected
+            )
+        )
+        
+        return SemanticChunk(
+            uuid=self.uuid,
+            type=self.type,
+            role=self.role,
+            project=self.project,
+            task_id=self.task_id,
+            subtask_id=self.subtask_id,
+            unit_id=self.unit_id,
+            text=self.text,
+            summary=self.summary,
+            language=self.language,
+            source_id=self.source_id,
+            source_path=self.source_path,
+            source_lines=source_lines,
+            ordinal=self.ordinal,
+            created_at=self.created_at,
+            status=self.status,
+            sha256=self.sha256,
+            links=links,
+            tags=tags,
+            metrics=metrics
+        ) 

chunk_metadata_adapter-1.0.0.dist-info/METADATA

@@ -0,0 +1,218 @@
+Metadata-Version: 2.4
+Name: chunk_metadata_adapter
+Version: 1.0.0
+Summary: Reusable metadata builder for chunk-based systems
+Author: SmartAssistant
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/yourusername/chunk_metadata_adapter
+Project-URL: Bug Tracker, https://github.com/yourusername/chunk_metadata_adapter/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+Requires-Dist: pydantic>=2.0.0
+
+# Chunk Metadata Adapter
+
+[![PyPI version](https://badge.fury.io/py/chunk-metadata-adapter.svg)](https://badge.fury.io/py/chunk-metadata-adapter)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+A library for creating, transforming, and managing universal metadata for content chunks for RAG systems, documentation processing, and machine learning training.
+
+[Русская версия](README.md)
+
+## Purpose
+
+The package provides tools for:
+
+1. Creating structured metadata for content chunks (text, code, messages)
+2. Converting between flat and structured formats
+3. Safely storing and validating connections between chunks
+4. Tracking quality and usage metrics of chunks
+5. Ensuring data integrity through SHA256 hashing
+6. Strict validation of identifiers (UUID4) and timestamps (ISO8601 with timezone)
+
+## Installation
+
+```bash
+pip install chunk-metadata-adapter
+```
+
+## Data Models
+
+The package provides two main data formats:
+
+### SemanticChunk
+
+A fully structured model with nested objects, convenient for programmatic processing:
+
+```python
+chunk = SemanticChunk(
+    uuid="550e8400-e29b-41d4-a716-446655440000",
+    type=ChunkType.DOC_BLOCK,
+    role=ChunkRole.DEVELOPER,
+    project="MyProject",
+    task_id="TASK-123",
+    subtask_id="TASK-123-A",
+    unit_id="chunker-service",
+    text="# Introduction\n\nThis is the system documentation.",
+    summary="Documentation introduction",
+    language="markdown",
+    source_id="48bb4273-1f56-4015-8b14-3d685b8cc9ae",
+    source_path="docs/intro.md",
+    source_lines=[1, 3],
+    ordinal=0,
+    created_at="2023-10-15T12:34:56.789+00:00",
+    status=ChunkStatus.INDEXED,
+    chunking_version="1.0",
+    sha256="abcdef1234567890...",
+    links=["parent:a1b2c3d4-e5f6-4a5b-8c7d-9e0f1a2b3c4d"],
+    tags=["introduction", "documentation"],
+    metrics=ChunkMetrics(quality_score=0.95, used_in_generation=True)
+)
+```
+
+### FlatSemanticChunk
+
+A flat structure for storage systems that prefer simple key-value formats:
+
+```python
+flat_chunk = FlatSemanticChunk(
+    uuid="550e8400-e29b-41d4-a716-446655440000",
+    source_id="48bb4273-1f56-4015-8b14-3d685b8cc9ae",
+    project="MyProject",
+    task_id="TASK-123",
+    subtask_id="TASK-123-A",
+    unit_id="chunker-service",
+    type="DocBlock",
+    role="developer",
+    language="markdown",
+    text="# Introduction\n\nThis is the system documentation.",
+    summary="Documentation introduction",
+    ordinal=0,
+    sha256="abcdef1234567890...",
+    created_at="2023-10-15T12:34:56.789+00:00",
+    status="indexed",
+    source_path="docs/intro.md",
+    source_lines_start=1,
+    source_lines_end=3,
+    tags="introduction,documentation",
+    link_parent="a1b2c3d4-e5f6-4a5b-8c7d-9e0f1a2b3c4d",
+    quality_score=0.95,
+    used_in_generation=True
+)
+```
+
+## Usage
+
+### Creating Chunk Metadata
+
+```python
+import uuid
+from chunk_metadata_adapter import ChunkMetadataBuilder, ChunkType, ChunkRole
+
+# Create a metadata builder for the project
+builder = ChunkMetadataBuilder(project="MyProject", unit_id="chunker-service-1")
+
+# Generate a UUID for the source document
+source_id = str(uuid.uuid4())
+
+# Create metadata for a code chunk
+metadata = builder.build_flat_metadata(
+    text="def hello_world():\n    print('Hello, World!')",
+    source_id=source_id,
+    ordinal=1,
+    type=ChunkType.CODE_BLOCK,
+    language="python",
+    source_path="src/hello.py",
+    source_lines_start=10,
+    source_lines_end=12,
+    tags="example,hello",
+    role=ChunkRole.DEVELOPER
+)
+
+print(f"Chunk UUID: {metadata['uuid']}")
+print(f"SHA256: {metadata['sha256']}")
+```
+
+### Creating a Structured Chunk
+
+```python
+from chunk_metadata_adapter import ChunkMetadataBuilder, ChunkType, ChunkRole
+
+# Create a builder for a documentation project
+builder = ChunkMetadataBuilder(
+    project="DocumentationProject",
+    unit_id="docs-generator"
+)
+
+# Create a structured chunk
+chunk = builder.build_semantic_chunk(
+    text="# Introduction\n\nThis is the system documentation.",
+    language="markdown",
+    type=ChunkType.DOC_BLOCK,
+    source_id=str(uuid.uuid4()),
+    summary="Project introduction section",
+    role=ChunkRole.DEVELOPER,
+    source_path="docs/intro.md",
+    source_lines=[1, 3],
+    ordinal=0,
+    task_id="DOC-123",
+    subtask_id="DOC-123-A",
+    tags=["introduction", "documentation", "overview"],
+    links=[f"parent:{str(uuid.uuid4())}"]
+)
+
+print(f"Chunk UUID: {chunk.uuid}")
+print(f"Summary: {chunk.summary}")
+```
+
+### Converting Between Formats
+
+```python
+# Convert from structured to flat format
+flat_dict = builder.semantic_to_flat(chunk)
+
+# Convert from flat to structured format
+restored_chunk = builder.flat_to_semantic(flat_dict)
+
+# Verify they are equivalent
+assert restored_chunk.uuid == chunk.uuid
+assert restored_chunk.text == chunk.text
+assert restored_chunk.type == chunk.type
+```
+
+## Features
+
+- **Strict Type Validation:** All UUIDs are validated against the UUIDv4 format, dates are validated against ISO8601 with timezone.
+- **Flexible Formats:** Support for both structured and flat data representations for different storage systems.
+- **Built-in Integrity:** Automatic calculation of SHA256 hashes for integrity verification.
+- **Compatibility:** Designed to work in the RAG and ML system ecosystem.
+- **Type Safety:** Full typing support for IDEs and static analysis.
+
+## Documentation
+
+Documentation is available in English and Russian:
+
+### English Documentation
+- [Documentation Overview](docs/README.md)
+- [Metadata Structure](docs/Metadata.md)
+- [Usage Guide](docs/Usage.md)
+- [Component Interaction](docs/Component_Interaction.md)
+
+### Russian Documentation
+- [Обзор документации](docs/README.ru.md)
+- [Структура метаданных](docs/Metadata.ru.md)
+- [Руководство по использованию](docs/Usage.ru.md)
+- [Взаимодействие компонентов](docs/Component_Interaction.ru.md)
+
+## License
+
+MIT
+
+## Author
+
+SmartAssistant Team 

chunk_metadata_adapter-1.0.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+chunk_metadata_adapter/__init__.py,sha256=SgB0vjmvwkT29GOLYShHHeEvcHys1sZIRJkepFU3_3M,703
+chunk_metadata_adapter/examples.py,sha256=HBS0coKebzCqs43k90jzwd0aWqlahgN7tEJpplsplRE,5518
+chunk_metadata_adapter/metadata_builder.py,sha256=kQfWnaIt8mM-LKLv5Q6ijKd8gO37nIPpQjCW1_9jzo0,9213
+chunk_metadata_adapter/models.py,sha256=0tYn7PWvLQbqBnV0l4TiK_DVg3N4jAz21yiMv9Hahso,14396
+chunk_metadata_adapter-1.0.0.dist-info/METADATA,sha256=5_vFkZyZdaCGr47pju70F1JaQBI2RZd6Z9M4yfXwuoQ,6663
+chunk_metadata_adapter-1.0.0.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
+chunk_metadata_adapter-1.0.0.dist-info/top_level.txt,sha256=HKjlcPxdFwCG406zu8eysl3hd2x3cGY6N2JlNJ8y704,23
+chunk_metadata_adapter-1.0.0.dist-info/RECORD,,

chunk_metadata_adapter-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.7.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

chunk_metadata_adapter-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+chunk_metadata_adapter