django-cfg 1.4.20__py3-none-any.whl → 1.4.23__py3-none-any.whl

django_cfg/apps/knowbase/models/document.py

@@ -2,15 +2,17 @@
 Document models with pgvector support.
 """
 
+from typing import List, Optional
+
 from django.db import models
 from pgvector.django import VectorField
-from typing import Optional, List
-from .base import UserScopedModel, ProcessingStatus, TimestampedModel
+
+from .base import ProcessingStatus, TimestampedModel, UserScopedModel
 
 
 class DocumentCategory(TimestampedModel):
     """Document category for organization and access control."""
-    
+
     name = models.CharField(
         max_length=255,
         unique=True,
@@ -24,24 +26,24 @@ class DocumentCategory(TimestampedModel):
         default=True,
         help_text="Whether documents in this category are publicly accessible"
     )
-    
+
     class Meta:
         db_table = 'django_cfg_knowbase_document_categories'
         verbose_name = 'Document Category'
         verbose_name_plural = 'Document Categories'
         ordering = ['name']
-    
+
     def __str__(self) -> str:
         return f"{self.name} ({'Public' if self.is_public else 'Private'})"
 
 
 class Document(UserScopedModel):
     """Knowledge document with processing status tracking."""
-    
+
     # Custom managers
     from ..managers.document import DocumentManager
     objects = DocumentManager()
-    
+
     title = models.CharField(
         max_length=512,
         help_text="Document title"
@@ -74,7 +76,7 @@ class Document(UserScopedModel):
         default=0,
         help_text="Original file size in bytes"
     )
-    
+
     # Processing status
     processing_status = models.CharField(
         max_length=20,
@@ -85,17 +87,17 @@ class Document(UserScopedModel):
     processing_started_at = models.DateTimeField(null=True, blank=True)
     processing_completed_at = models.DateTimeField(null=True, blank=True)
     processing_error = models.TextField(blank=True, default="")
-    
+
     # Chunk statistics
     chunks_count = models.PositiveIntegerField(default=0)
     total_tokens = models.PositiveIntegerField(default=0)
-    
+
     # Cost tracking for monitoring
     total_cost_usd = models.FloatField(
         default=0.0,
         help_text="Total processing cost in USD"
     )
-    
+
     # Metadata
     metadata = models.JSONField(
         default=dict,
@@ -103,7 +105,7 @@ class Document(UserScopedModel):
         null=True,
         help_text="Additional document metadata"
     )
-    
+
     class Meta:
         db_table = 'django_cfg_knowbase_documents'
         indexes = [
@@ -118,27 +120,27 @@ class Document(UserScopedModel):
                 name='unique_user_document'
             )
         ]
-    
+
     def save(self, *args, **kwargs):
         """Override save to generate content_hash if not provided."""
         if not self.content_hash and self.content:
             import hashlib
             self.content_hash = hashlib.sha256(self.content.encode()).hexdigest()
-        
+
         # Set file_size if not provided
         if not self.file_size and self.content:
             self.file_size = len(self.content.encode('utf-8'))
-        
+
         super().save(*args, **kwargs)
-    
+
     def __str__(self) -> str:
         return f"{self.title} ({self.user.username})"
-    
+
     @property
     def is_processed(self) -> bool:
         """Check if document processing is completed."""
         return self.processing_status == ProcessingStatus.COMPLETED
-    
+
     @property
     def processing_duration(self) -> Optional[float]:
         """Calculate processing duration in seconds."""
@@ -146,32 +148,32 @@ class Document(UserScopedModel):
             delta = self.processing_completed_at - self.processing_started_at
             return delta.total_seconds()
         return None
-    
+
     @property
     def is_publicly_accessible(self) -> bool:
         """Check if document is publicly accessible (document and at least one category must be public)."""
         if not self.is_public:
             return False
-        
+
         # If document has categories, at least one must be public
         if self.categories.exists():
             return self.categories.filter(is_public=True).exists()
-        
+
         # If no categories assigned, document is public by default
         return True
-    
+
     def get_all_categories(self):
         """Get all categories for this document."""
         return list(self.categories.all())
-    
+
     def add_category(self, category):
         """Add a category to this document."""
         self.categories.add(category)
-    
+
     def remove_category(self, category):
         """Remove a category from this document."""
         self.categories.remove(category)
-    
+
     def set_categories(self, categories_list):
         """Set multiple categories for this document."""
         self.categories.set(categories_list)
@@ -179,11 +181,11 @@ class Document(UserScopedModel):
 
 class DocumentChunk(UserScopedModel):
     """Text chunk with vector embedding for semantic search."""
-    
+
     # Custom managers
     from ..managers.document import DocumentChunkManager
     objects = DocumentChunkManager()
-    
+
     document = models.ForeignKey(
         Document,
         on_delete=models.CASCADE,
@@ -196,13 +198,13 @@ class DocumentChunk(UserScopedModel):
     chunk_index = models.PositiveIntegerField(
         help_text="Sequential chunk number within document"
     )
-    
+
     # Vector embedding (1536 dimensions for OpenAI text-embedding-ada-002)
     embedding = VectorField(
         dimensions=1536,
         help_text="Vector embedding for semantic search"
     )
-    
+
     # Chunk statistics
     token_count = models.PositiveIntegerField(
         default=0,
@@ -212,7 +214,7 @@ class DocumentChunk(UserScopedModel):
         default=0,
         help_text="Number of characters in chunk"
     )
-    
+
     # Processing metadata
     embedding_model = models.CharField(
         max_length=100,
@@ -223,7 +225,7 @@ class DocumentChunk(UserScopedModel):
         default=0.0,
         help_text="Cost in USD for embedding generation"
     )
-    
+
     # Additional metadata
     metadata = models.JSONField(
         default=dict,
@@ -231,7 +233,7 @@ class DocumentChunk(UserScopedModel):
         null=True,
         help_text="Chunk-specific metadata"
     )
-    
+
     class Meta:
         db_table = 'django_cfg_knowbase_document_chunks'
         indexes = [
@@ -245,10 +247,10 @@ class DocumentChunk(UserScopedModel):
             )
         ]
         ordering = ['document', 'chunk_index']
-    
+
     def __str__(self) -> str:
         return f"Chunk {self.chunk_index} of {self.document.title}"
-    
+
     @classmethod
     def semantic_search(
         cls,
@@ -259,7 +261,7 @@ class DocumentChunk(UserScopedModel):
     ):
         """Perform semantic search using pgvector."""
         from pgvector.django import CosineDistance
-        
+
         return cls.objects.filter(user=user).annotate(
             similarity=1 - CosineDistance('embedding', query_embedding)
         ).filter(

django_cfg/apps/knowbase/models/external_data.py

@@ -4,14 +4,13 @@ External Data models for integrating external Django models into django_cfg.apps
 Provides a unified way to vectorize and search any external data source.
 """
 
-from django.db import models
+from typing import Any, Dict
+
 from django.conf import settings
+from django.db import models
 from pgvector.django import VectorField
-from typing import Optional, List, Dict, Any
-import json
-from enum import Enum
 
-from .base import UserScopedModel, ProcessingStatus, TimestampedModel
+from .base import TimestampedModel, UserScopedModel
 from .document import DocumentCategory
 
 
@@ -40,11 +39,11 @@ class ExternalData(UserScopedModel):
     This model represents any external data source (Django models, APIs, etc.)
     that has been integrated into the knowledge base for semantic search.
     """
-    
+
     # Manager will be set after class definition
     from ..managers.external_data import ExternalDataManager
     objects = ExternalDataManager()
-    
+
     # Basic information
     title = models.CharField(
         max_length=512,
@@ -54,7 +53,7 @@ class ExternalData(UserScopedModel):
         blank=True,
         help_text="Description of what this external data contains"
     )
-    
+
     # Source configuration
     source_type = models.CharField(
         max_length=20,
@@ -72,7 +71,7 @@ class ExternalData(UserScopedModel):
         blank=True,
         help_text="Configuration for data extraction (fields, filters, etc.)"
     )
-    
+
     # Content and metadata
     content = models.TextField(
         blank=True,
@@ -89,7 +88,7 @@ class ExternalData(UserScopedModel):
         blank=True,
         help_text="Additional metadata from the source"
     )
-    
+
     # Processing status
     status = models.CharField(
         max_length=20,
@@ -101,7 +100,7 @@ class ExternalData(UserScopedModel):
         blank=True,
         help_text="Error message if processing failed"
     )
-    
+
     # Vectorization settings
     chunk_size = models.PositiveIntegerField(
         default=1000,
@@ -116,13 +115,13 @@ class ExternalData(UserScopedModel):
         default="text-embedding-ada-002",
         help_text="Embedding model used for vectorization"
     )
-    
+
     # Search settings
     similarity_threshold = models.FloatField(
         default=0.5,
         help_text="Similarity threshold for this external data (0.0-1.0). Lower = more results, higher = more precise"
     )
-    
+
     # Processing timestamps
     processed_at = models.DateTimeField(
         null=True, blank=True,
@@ -132,7 +131,7 @@ class ExternalData(UserScopedModel):
         null=True, blank=True,
         help_text="When the source data was last updated"
     )
-    
+
     # Statistics
     total_chunks = models.PositiveIntegerField(
         default=0,
@@ -146,7 +145,7 @@ class ExternalData(UserScopedModel):
         default=0.0,
         help_text="Total cost for processing this data (USD)"
     )
-    
+
     # Organization
     category = models.ForeignKey(
         DocumentCategory,
@@ -159,7 +158,7 @@ class ExternalData(UserScopedModel):
         blank=True,
         help_text="Tags for categorization and filtering"
     )
-    
+
     # Access control
     is_active = models.BooleanField(
         default=True,
@@ -169,7 +168,7 @@ class ExternalData(UserScopedModel):
         default=False,
         help_text="Whether this data is publicly searchable"
     )
-    
+
     class Meta:
         db_table = 'django_cfg_knowbase_external_data'
         verbose_name = 'External Data'
@@ -184,7 +183,7 @@ class ExternalData(UserScopedModel):
             models.Index(fields=['content_hash']),
         ]
         ordering = ['-processed_at', '-created_at']
-    
+
     def save(self, *args, **kwargs):
         """Override save to generate content_hash if not provided."""
         # Store original hash for comparison in signals
@@ -196,7 +195,7 @@ class ExternalData(UserScopedModel):
                 self._original_content_hash = None
         else:
             self._original_content_hash = None
-            
+
         # Generate hash if not provided
         if not self.content_hash and self.content:
             import hashlib
@@ -207,51 +206,51 @@ class ExternalData(UserScopedModel):
             new_hash = hashlib.sha256(self.content.encode()).hexdigest()
             if self.content_hash != new_hash:
                 self.content_hash = new_hash
-                
+
         super().save(*args, **kwargs)
-    
+
     def __str__(self) -> str:
         return f"{self.title} ({self.source_type})"
-    
+
     @property
     def full_name(self) -> str:
         """Full name including source type."""
         return f"{self.title} ({self.get_source_type_display()})"
-    
+
     @property
     def is_processed(self) -> bool:
         """Check if data has been successfully processed."""
         return self.status == ExternalDataStatus.COMPLETED
-    
+
     @property
     def is_outdated(self) -> bool:
         """Check if data needs reprocessing."""
         return (
             self.status == ExternalDataStatus.OUTDATED or
-            (self.source_updated_at and self.processed_at and 
+            (self.source_updated_at and self.processed_at and
              self.source_updated_at > self.processed_at)
         )
-    
+
     @property
     def chunks_count(self) -> int:
         """Get the number of chunks for this external data."""
         return self.chunks.count()
-    
+
     def get_config_value(self, key: str, default=None):
         """Get a configuration value."""
         return self.source_config.get(key, default)
-    
+
     def set_config_value(self, key: str, value):
         """Set a configuration value."""
         self.source_config[key] = value
         self.save(update_fields=['source_config'])
-    
+
     def add_tag(self, tag: str):
         """Add a tag if it doesn't exist."""
         if tag not in self.tags:
             self.tags.append(tag)
             self.save(update_fields=['tags'])
-    
+
     def remove_tag(self, tag: str):
         """Remove a tag if it exists."""
         if tag in self.tags:
@@ -265,10 +264,10 @@ class ExternalDataChunk(TimestampedModel):
     
     Similar to DocumentChunk but for external data sources.
     """
-    
+
     # Manager will be set after class definition
     objects = models.Manager()  # Temporary default manager
-    
+
     # Relationships
     user = models.ForeignKey(
         settings.AUTH_USER_MODEL,
@@ -282,7 +281,7 @@ class ExternalDataChunk(TimestampedModel):
         related_name='chunks',
         help_text="External data this chunk belongs to"
     )
-    
+
     # Content
     content = models.TextField(
         blank=True,
@@ -292,7 +291,7 @@ class ExternalDataChunk(TimestampedModel):
         default=0,
         help_text="Sequential index of this chunk within the external data"
     )
-    
+
     # Vector embedding
     embedding = VectorField(
         dimensions=1536,  # OpenAI ada-002 default
@@ -304,7 +303,7 @@ class ExternalDataChunk(TimestampedModel):
         default="text-embedding-ada-002",
         help_text="Model used for embedding generation"
     )
-    
+
     # Metrics
     token_count = models.PositiveIntegerField(
         default=0,
@@ -318,14 +317,14 @@ class ExternalDataChunk(TimestampedModel):
         default=0.0,
         help_text="Cost for generating this embedding (USD)"
     )
-    
+
     # Context information
     chunk_metadata = models.JSONField(
         default=dict,
         blank=True,
         help_text="Additional metadata for this specific chunk"
     )
-    
+
     class Meta:
         db_table = 'django_cfg_knowbase_external_data_chunk'
         verbose_name = 'External Data Chunk'
@@ -338,10 +337,10 @@ class ExternalDataChunk(TimestampedModel):
             models.Index(fields=['chunk_index']),
         ]
         ordering = ['external_data', 'chunk_index']
-    
+
     def __str__(self) -> str:
         return f"{self.external_data.title} - Chunk {self.chunk_index}"
-    
+
     @property
     def content_preview(self) -> str:
         """Get a preview of the chunk content."""
@@ -350,19 +349,19 @@ class ExternalDataChunk(TimestampedModel):
         if len(self.content) <= 100:
             return self.content
         return self.content[:100] + "..."
-    
+
     @property
     def embedding_info(self) -> str:
         """Get embedding information."""
         if not self.embedding:
             return "No embedding"
         return f"{len(self.embedding)} dimensions"
-    
+
     @property
     def similarity_search_ready(self) -> bool:
         """Check if chunk is ready for similarity search."""
         return self.embedding is not None
-    
+
     @property
     def source_info(self) -> Dict[str, Any]:
         """Get source information for this chunk."""

django_cfg/apps/knowbase/serializers/__init__.py

@@ -4,11 +4,11 @@ Knowledge Base Serializers
 Pydantic v2 models for API request/response validation.
 """
 
-from .document_serializers import *
-from .chat_serializers import *
-from .public_serializers import *
 from .archive_serializers import *
+from .chat_serializers import *
+from .document_serializers import *
 from .external_data_serializers import *
+from .public_serializers import *
 
 __all__ = [
     # Document serializers
@@ -16,19 +16,19 @@ __all__ = [
     'DocumentSerializer',
     'DocumentStatsSerializer',
     'DocumentProcessingStatusSerializer',
-    
+
     # Public serializers
     'PublicCategorySerializer',
     'PublicDocumentListSerializer',
     'PublicDocumentSerializer',
-    
+
     # Chat serializers
     'ChatSessionCreateRequest',
     'ChatSessionResponse',
     'ChatQueryRequest',
     'ChatResponse',
     'ChatHistoryResponse',
-    
+
     # Archive serializers
     'DocumentArchiveCreateSerializer',
     'DocumentArchiveSerializer',
@@ -46,7 +46,7 @@ __all__ = [
     'ArchiveUploadSerializer',
     'ChunkRevectorizationRequestSerializer',
     'VectorizationResultSerializer',
-    
+
     # External data serializers
     'ExternalDataCreateRequest',
     'ExternalDataResponse',