django-cfg 1.3.11__py3-none-any.whl → 1.4.0__py3-none-any.whl

django_cfg/apps/knowbase/services/archive/context/__init__.py

@@ -0,0 +1,14 @@
+"""
+Context building for chunks.
+
+Models and builders for chunk context metadata.
+"""
+
+from .models import ChunkContextMetadata, ChunkData
+from .builders import ChunkContextBuilder
+
+__all__ = [
+    'ChunkContextMetadata',
+    'ChunkData',
+    'ChunkContextBuilder',
+]

django_cfg/apps/knowbase/services/archive/context/builders.py

@@ -0,0 +1,220 @@
+"""
+Context metadata builders for chunks.
+
+Builds rich context metadata for different types of content chunks.
+"""
+
+from typing import Dict, Any, Optional, List
+from ....models.archive import ArchiveItem
+
+
+class ChunkContextBuilder:
+    """Build context metadata for different chunk types."""
+
+    @staticmethod
+    def build_code_context(
+        item: ArchiveItem,
+        chunk_index: int,
+        content: str,
+        start_line: int,
+        end_line: int,
+        code_info: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        """
+        Build context metadata for code chunk.
+
+        Args:
+            item: Archive item being chunked
+            chunk_index: Index of this chunk
+            content: Chunk content
+            start_line: Starting line number
+            end_line: Ending line number
+            code_info: Code analysis info (element_name, type, etc.)
+
+        Returns:
+            Context metadata dictionary
+        """
+        from ..analyzers import TagGenerator
+
+        return {
+            'archive_info': {
+                'id': str(item.archive.id),
+                'title': item.archive.title,
+                'description': item.archive.description,
+            },
+            'item_info': {
+                'id': str(item.id),
+                'relative_path': item.relative_path,
+                'item_name': item.item_name,
+                'content_type': item.content_type,
+                'language': item.language,
+            },
+            'position_info': {
+                'chunk_index': chunk_index,
+                'start_line': start_line + 1,
+                'end_line': end_line,
+                'total_lines': len(item.raw_content.split('\n')),
+            },
+            'structure_info': {
+                'element_name': code_info.get('element_name'),
+                'element_type': code_info.get('element_type'),
+                'is_async': code_info.get('is_async', False),
+                'has_docstring': bool(code_info.get('docstring')),
+            },
+            'semantic_info': {
+                'chunk_type': 'code',
+                'content_purpose': code_info.get('purpose', 'implementation'),
+                'complexity_score': code_info.get('complexity_score', 0.0),
+                'technical_tags': TagGenerator.generate_code_tags(content, code_info),
+            },
+            'processing_info': {
+                'extraction_method': 'ast_parser',
+                'chunking_strategy': 'logical_units',
+                'quality_score': code_info.get('quality_score', 0.5),
+            }
+        }
+
+    @staticmethod
+    def build_document_context(
+        item: ArchiveItem,
+        chunk_index: int,
+        content: str,
+        section_info: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        """
+        Build context metadata for document chunk.
+
+        Args:
+            item: Archive item being chunked
+            chunk_index: Index of this chunk
+            content: Chunk content
+            section_info: Section info (title, level, etc.)
+
+        Returns:
+            Context metadata dictionary
+        """
+        from ..analyzers import TagGenerator
+
+        return {
+            'archive_info': {
+                'id': str(item.archive.id),
+                'title': item.archive.title,
+            },
+            'item_info': {
+                'id': str(item.id),
+                'relative_path': item.relative_path,
+                'content_type': item.content_type,
+                'language': item.language,
+            },
+            'position_info': {
+                'chunk_index': chunk_index,
+            },
+            'structure_info': {
+                'section_title': section_info.get('title'),
+                'section_level': section_info.get('level', 0),
+            },
+            'semantic_info': {
+                'chunk_type': 'heading' if section_info.get('title') else 'text',
+                'content_purpose': 'documentation',
+                'topic_tags': TagGenerator.generate_document_tags(content),
+            },
+            'processing_info': {
+                'extraction_method': 'markdown_parser',
+                'chunking_strategy': 'heading_based',
+            }
+        }
+
+    @staticmethod
+    def build_data_context(
+        item: ArchiveItem,
+        chunk_index: int,
+        content: str,
+        data_type: str,
+        key_name: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """
+        Build context metadata for data chunk.
+
+        Args:
+            item: Archive item being chunked
+            chunk_index: Index of this chunk
+            content: Chunk content
+            data_type: Type of data (e.g., 'json_object')
+            key_name: Optional key name for object property
+
+        Returns:
+            Context metadata dictionary
+        """
+        return {
+            'archive_info': {
+                'id': str(item.archive.id),
+                'title': item.archive.title,
+            },
+            'item_info': {
+                'id': str(item.id),
+                'relative_path': item.relative_path,
+                'content_type': item.content_type,
+            },
+            'position_info': {
+                'chunk_index': chunk_index,
+            },
+            'structure_info': {
+                'data_key': key_name,
+                'data_type': data_type,
+            },
+            'semantic_info': {
+                'chunk_type': 'metadata',
+                'content_purpose': 'data_definition',
+            },
+            'processing_info': {
+                'extraction_method': 'json_parser',
+                'chunking_strategy': 'object_properties',
+            }
+        }
+
+    @staticmethod
+    def build_generic_context(
+        item: ArchiveItem,
+        chunk_index: int,
+        content: str,
+        start_pos: int,
+        end_pos: int
+    ) -> Dict[str, Any]:
+        """
+        Build context metadata for generic text chunk.
+
+        Args:
+            item: Archive item being chunked
+            chunk_index: Index of this chunk
+            content: Chunk content
+            start_pos: Starting character position
+            end_pos: Ending character position
+
+        Returns:
+            Context metadata dictionary
+        """
+        return {
+            'archive_info': {
+                'id': str(item.archive.id),
+                'title': item.archive.title,
+            },
+            'item_info': {
+                'id': str(item.id),
+                'relative_path': item.relative_path,
+                'content_type': item.content_type,
+            },
+            'position_info': {
+                'chunk_index': chunk_index,
+                'start_char': start_pos,
+                'end_char': end_pos,
+                'relative_position': start_pos / len(item.raw_content) if item.raw_content else 0.0,
+            },
+            'semantic_info': {
+                'chunk_type': 'text',
+                'content_purpose': 'content',
+            },
+            'processing_info': {
+                'extraction_method': 'text_splitting',
+                'chunking_strategy': 'fixed_size_overlap',
+            }
+        }

django_cfg/apps/knowbase/services/archive/context/models.py

@@ -0,0 +1,38 @@
+"""
+Pydantic models for chunking context metadata.
+
+Data structures for chunk data and context metadata.
+"""
+
+from pydantic import BaseModel
+from typing import Dict, Any
+
+
+class ChunkContextMetadata(BaseModel):
+    """Rich context metadata for chunks."""
+
+    # Parent hierarchy
+    archive_info: Dict[str, Any]
+    item_info: Dict[str, Any]
+
+    # Position and structure
+    position_info: Dict[str, Any]
+    structure_info: Dict[str, Any]
+
+    # Semantic context
+    semantic_info: Dict[str, Any]
+
+    # Relational context (optional)
+    relationship_info: Dict[str, Any] = {}
+
+    # Processing provenance
+    processing_info: Dict[str, Any]
+
+
+class ChunkData(BaseModel):
+    """Data structure for created chunk."""
+
+    content: str
+    chunk_index: int
+    chunk_type: str
+    context_metadata: Dict[str, Any]

django_cfg/apps/knowbase/services/embedding/models.py

@@ -6,7 +6,7 @@ the embedding processing pipeline using Pydantic for type safety.
 """
 
 from typing import List, Dict, Any, Optional
-from pydantic import BaseModel, Field, validator
+from pydantic import BaseModel, Field, field_validator, ValidationInfo
 from enum import Enum
 
 
@@ -35,7 +35,8 @@ class ChunkData(BaseModel):
         description="Type of parent content"
     )
     
-    @validator('content')
+    @field_validator('content')
+    @classmethod
     def content_must_not_be_empty(cls, v):
         if not v or not v.strip():
             raise ValueError('Content cannot be empty')
@@ -76,7 +77,8 @@ class EmbeddingResult(BaseModel):
         description="Time taken to process this chunk in seconds"
     )
     
-    @validator('embedding')
+    @field_validator('embedding')
+    @classmethod
     def validate_embedding_dimension(cls, v):
         if v is not None and len(v) > 0 and len(v) not in [1536, 3072]:  # Common OpenAI embedding dimensions
             # Warning, not error - allow different dimensions
@@ -153,22 +155,24 @@ class BatchProcessingResult(BaseModel):
             return 0.0
         return self.total_tokens / self.successful_chunks
     
-    @validator('successful_chunks', 'failed_chunks')
-    def validate_chunk_counts(cls, v, values):
-        if 'total_chunks' in values:
-            total = values['total_chunks']
+    @field_validator('successful_chunks', 'failed_chunks')
+    @classmethod
+    def validate_chunk_counts(cls, v, info: ValidationInfo):
+        if info.data and 'total_chunks' in info.data:
+            total = info.data['total_chunks']
             if v > total:
                 raise ValueError(f'Chunk count cannot exceed total chunks ({total})')
         return v
-    
-    @validator('failed_chunks')
-    def validate_total_consistency(cls, v, values):
-        if 'total_chunks' in values and 'successful_chunks' in values:
-            expected_failed = values['total_chunks'] - values['successful_chunks']
+
+    @field_validator('failed_chunks')
+    @classmethod
+    def validate_total_consistency(cls, v, info: ValidationInfo):
+        if info.data and 'total_chunks' in info.data and 'successful_chunks' in info.data:
+            expected_failed = info.data['total_chunks'] - info.data['successful_chunks']
             if v != expected_failed:
                 raise ValueError(
-                    f'Failed chunks ({v}) + successful chunks ({values["successful_chunks"]}) '
-                    f'must equal total chunks ({values["total_chunks"]})'
+                    f'Failed chunks ({v}) + successful chunks ({info.data["successful_chunks"]}) '
+                    f'must equal total chunks ({info.data["total_chunks"]})'
                 )
         return v
     

django_cfg/apps/knowbase/services/embedding/processors.py

@@ -86,14 +86,17 @@ class ArchiveChunkProcessor:
         if len(enhanced_content) > max_length:
             if context_parts:
                 context_prefix_len = len(context_prefix) + 2  # +2 for \n\n
-                available_content_len = max_length - context_prefix_len
+                # Account for "..." suffix (3 chars)
+                available_content_len = max_length - context_prefix_len - 3
                 if available_content_len > 100:  # Ensure we have meaningful content
                     truncated_content = content[:available_content_len] + "..."
                     enhanced_content = f"{context_prefix}\n\n{truncated_content}"
                 else:
-                    enhanced_content = content[:max_length] + "..."
+                    # Subtract 3 for "..." suffix
+                    enhanced_content = content[:max_length - 3] + "..."
             else:
-                enhanced_content = content[:max_length] + "..."
+                # Subtract 3 for "..." suffix
+                enhanced_content = content[:max_length - 3] + "..."
         
         return enhanced_content.strip()
     

django_cfg/apps/knowbase/tasks/document_processing.py

@@ -127,14 +127,22 @@ def process_document_async(
             
             # Step 5: Update document status
             processing_time = time.time() - start_time
-            document.processing_status = ProcessingStatus.COMPLETED
+
+            # Check if embedding generation failed
+            if embedding_result.successful_chunks == 0 and embedding_result.total_chunks > 0:
+                # All embeddings failed
+                document.processing_status = ProcessingStatus.FAILED
+                document.processing_error = "; ".join(embedding_result.errors) if embedding_result.errors else "All embeddings failed to generate"
+            else:
+                document.processing_status = ProcessingStatus.COMPLETED
+
             document.processing_completed_at = timezone.now()
             document.chunks_count = embedding_result.total_chunks
             document.total_tokens = total_tokens
             document.total_cost_usd = total_cost
             document.save(update_fields=[
-                'processing_status', 'processing_completed_at', 'chunks_count',
-                'total_tokens', 'total_cost_usd'
+                'processing_status', 'processing_completed_at', 'processing_error',
+                'chunks_count', 'total_tokens', 'total_cost_usd'
             ])
             
             return {

django_cfg/apps/leads/tests.py

@@ -112,7 +112,7 @@ class LeadAPITest(APITestCase):
     
     def setUp(self):
         """Set up test data."""
-        self.url = reverse('leads:lead-submit')
+        self.url = reverse('cfg_leads:lead-submit')
         self.valid_data = {
             'name': 'John Doe',
             'email': 'john@example.com',

django_cfg/apps/payments/admin/api_keys_admin.py

@@ -23,7 +23,7 @@ from django_cfg.modules.django_admin import (
     ActionVariant
 )
 from django_cfg.modules.django_admin.utils.badges import StatusBadge
-from django_cfg.modules.django_logger import get_logger
+from django_cfg.modules.django_logging import get_logger
 
 from ..models import APIKey
 

django_cfg/apps/payments/admin/balance_admin.py

@@ -24,7 +24,7 @@ from django_cfg.modules.django_admin import (
     action,
     ActionVariant
 )
-from django_cfg.modules.django_logger import get_logger
+from django_cfg.modules.django_logging import get_logger
 
 from ..models import UserBalance, Transaction
 from .filters import BalanceRangeFilter, RecentActivityFilter

django_cfg/apps/payments/admin/currencies_admin.py

@@ -25,7 +25,7 @@ from django_cfg.modules.django_admin import (
     action
 )
 from django_cfg.modules.django_admin.utils.badges import StatusBadge
-from django_cfg.modules.django_logger import get_logger
+from django_cfg.modules.django_logging import get_logger
 from ..models import Currency
 from .filters import CurrencyTypeFilter, CurrencyProviderFilter, CurrencyRateStatusFilter
 

django_cfg/apps/payments/admin/payments_admin.py

@@ -24,7 +24,7 @@ from django_cfg.modules.django_admin import (
     ActionVariant
 )
 from django_cfg.modules.django_admin.utils.badges import StatusBadge
-from django_cfg.modules.django_logger import get_logger
+from django_cfg.modules.django_logging import get_logger
 
 from ..models import UniversalPayment
 

django_cfg/apps/payments/admin/subscriptions_admin.py

@@ -20,7 +20,7 @@ from django_cfg.modules.django_admin import (
     action
 )
 from django_cfg.modules.django_admin.utils.badges import StatusBadge
-from django_cfg.modules.django_logger import get_logger
+from django_cfg.modules.django_logging import get_logger
 from ..models import Subscription
 
 logger = get_logger("subscriptions_admin")

django_cfg/apps/payments/admin_interface/templates/payments/base.html

@@ -1,127 +1,60 @@
+{% extends 'django_tailwind/app.html' %}
 {% load static %}
-<!DOCTYPE html>
-<html lang="en" class="h-full">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>{% block title %}Payment Admin - Django CFG{% endblock %}</title>
-    
-    <!-- Material Icons -->
-    <link href="https://fonts.googleapis.com/icon?family=Material+Icons+Outlined" rel="stylesheet">
-    
-    <!-- Tailwind CSS -->
-    <script src="https://cdn.tailwindcss.com"></script>
-    
-    <!-- Alpine.js -->
-    <script defer src="https://unpkg.com/alpinejs@3.x.x/dist/cdn.min.js"></script>
-    
-    <style>
-        [x-cloak] { display: none !important; }
-    </style>
-    
-    <!-- Dark mode initialization script to prevent flash -->
-    <script>
-        // Initialize dark mode before Alpine.js loads
-        if (localStorage.getItem('darkMode') === 'true' || 
-            (!localStorage.getItem('darkMode') && window.matchMedia('(prefers-color-scheme: dark)').matches)) {
-            document.documentElement.classList.add('dark');
-        }
-    </script>
-    
-    {% block extra_head %}{% endblock %}
-</head>
-<body class="h-full bg-gray-50 dark:bg-gray-900" x-data="{ 
-    darkMode: localStorage.getItem('darkMode') === 'true' || 
-              (!localStorage.getItem('darkMode') && window.matchMedia('(prefers-color-scheme: dark)').matches),
-    showNgrokHelp: false
-}" x-init="$watch('darkMode', val => { 
-    localStorage.setItem('darkMode', val);
-    document.documentElement.classList.toggle('dark', val);
-})">
-    
-    <!-- Navigation -->
-    <nav class="bg-white dark:bg-gray-800 shadow border-b border-gray-200 dark:border-gray-700">
-        <div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8">
-            <div class="flex justify-between h-16">
-                <!-- Logo -->
-                <div class="flex items-center">
-                    <span class="material-icons-outlined text-blue-600 mr-2">payment</span>
-                    <h1 class="text-xl font-semibold text-gray-900 dark:text-white">Payment Admin</h1>
-                </div>
-                
-                <!-- Navigation Links -->
-                <div class="flex items-center space-x-4">
-                    <a href="{% url 'cfg_payments_admin:dashboard' %}" 
-                       class="text-gray-600 hover:text-gray-900 dark:text-gray-300 dark:hover:text-white px-3 py-2 rounded-md text-sm font-medium">
-                        Dashboard
-                    </a>
-                    <a href="{% url 'cfg_payments_admin:payment-list' %}" 
-                       class="text-gray-600 hover:text-gray-900 dark:text-gray-300 dark:hover:text-white px-3 py-2 rounded-md text-sm font-medium">
-                        Payments
-                    </a>
-                    <a href="{% url 'cfg_payments_admin:webhook-dashboard' %}" 
-                       class="text-gray-600 hover:text-gray-900 dark:text-gray-300 dark:hover:text-white px-3 py-2 rounded-md text-sm font-medium">
-                        Webhooks
-                    </a>
-                    
-                    <!-- Dark Mode Toggle -->
-                    <button @click="darkMode = !darkMode" 
-                            class="p-2 text-gray-600 hover:text-gray-900 dark:text-gray-300 dark:hover:text-white transition-colors">
-                        <span x-show="!darkMode" x-cloak class="material-icons-outlined">dark_mode</span>
-                        <span x-show="darkMode" x-cloak class="material-icons-outlined">light_mode</span>
-                    </button>
-                </div>
-            </div>
-        </div>
-    </nav>
-    
-    <!-- Main Content -->
-    <main class="max-w-7xl mx-auto py-6 px-4 sm:px-6 lg:px-8">
-        <!-- Page Header -->
-        {% block header %}
-        <div class="mb-8">
-            <h2 class="text-2xl font-bold text-gray-900 dark:text-white">
-                {% block page_title %}Dashboard{% endblock %}
-            </h2>
-            {% block page_subtitle %}
-            <p class="mt-1 text-sm text-gray-600 dark:text-gray-400">
-                Manage payments and webhooks
-            </p>
-            {% endblock %}
-        </div>
-        {% endblock %}
-        
-        <!-- Messages -->
-        {% if messages %}
-        <div class="mb-6 space-y-2">
-            {% for message in messages %}
-            <div class="rounded-md p-4 {% if message.tags == 'error' %}bg-red-50 text-red-800 border border-red-200 dark:bg-red-900/20 dark:text-red-400 dark:border-red-800{% elif message.tags == 'warning' %}bg-yellow-50 text-yellow-800 border border-yellow-200 dark:bg-yellow-900/20 dark:text-yellow-400 dark:border-yellow-800{% elif message.tags == 'success' %}bg-green-50 text-green-800 border border-green-200 dark:bg-green-900/20 dark:text-green-400 dark:border-green-800{% else %}bg-blue-50 text-blue-800 border border-blue-200 dark:bg-blue-900/20 dark:text-blue-400 dark:border-blue-800{% endif %}">
-                {{ message }}
-            </div>
-            {% endfor %}
-        </div>
-        {% endif %}
-        
-        <!-- Page Content -->
-        {% block content %}{% endblock %}
-    </main>
-    
-    <!-- Footer -->
-    <footer class="bg-white dark:bg-gray-800 border-t border-gray-200 dark:border-gray-700 mt-12">
-        <div class="max-w-7xl mx-auto py-4 px-4 sm:px-6 lg:px-8">
-            <div class="text-center text-sm text-gray-600 dark:text-gray-400">
-                Payment Admin - Django CFG
-            </div>
-        </div>
-    </footer>
-    
-    <!-- Global Dialogs -->
-    {% include 'payments/components/ngrok_help_dialog.html' %}
-    
-    <!-- Payment System JS Components -->
-    <script src="{% static 'payments/js/api-client.js' %}"></script>
-    <script src="{% static 'payments/js/ngrok-status.js' %}"></script>
-    
-    {% block extra_js %}{% endblock %}
-</body>
-</html>
+
+{% block title %}{% block page_title_text %}Dashboard{% endblock %} - Payment Admin - Django CFG{% endblock %}
+
+{# Navbar configuration #}
+{% block header %}
+{% with navbar_items=payment_nav_items %}
+{% include 'django_tailwind/components/navbar.html' with title="Payment Admin" icon='<span class="material-icons-outlined text-blue-600">payment</span>' nav_items=navbar_items %}
+{% endwith %}
+{% endblock %}
+
+{% block extra_head %}
+{{ block.super }}
+<!-- Material Icons -->
+<link href="https://fonts.googleapis.com/icon?family=Material+Icons+Outlined" rel="stylesheet">
+{% endblock %}
+
+{% block alpine_data_extra %}, showNgrokHelp: false{% endblock %}
+
+{% block content %}
+<!-- Page Header -->
+<div class="mb-8">
+    <h2 class="text-2xl font-bold text-gray-900 dark:text-white">
+        {% block page_title %}Dashboard{% endblock %}
+    </h2>
+    {% block page_subtitle %}
+    <p class="mt-1 text-sm text-gray-600 dark:text-gray-400">
+        Manage payments and webhooks
+    </p>
+    {% endblock %}
+</div>
+
+<!-- Messages -->
+{% if messages %}
+<div class="mb-6 space-y-2">
+    {% for message in messages %}
+    <div class="rounded-md p-4 {% if message.tags == 'error' %}bg-red-50 text-red-800 border border-red-200 dark:bg-red-900/20 dark:text-red-400 dark:border-red-800{% elif message.tags == 'warning' %}bg-yellow-50 text-yellow-800 border border-yellow-200 dark:bg-yellow-900/20 dark:text-yellow-400 dark:border-yellow-800{% elif message.tags == 'success' %}bg-green-50 text-green-800 border border-green-200 dark:bg-green-900/20 dark:text-green-400 dark:border-green-800{% else %}bg-blue-50 text-blue-800 border border-blue-200 dark:bg-blue-900/20 dark:text-blue-400 dark:border-blue-800{% endif %}">
+        {{ message }}
+    </div>
+    {% endfor %}
+</div>
+{% endif %}
+
+<!-- Page Content -->
+{% block page_content %}{% endblock %}
+{% endblock %}
+
+{% block body %}
+{{ block.super }}
+
+<!-- Global Dialogs -->
+{% include 'payments/components/ngrok_help_dialog.html' %}
+{% endblock %}
+
+{% block extra_js %}
+<!-- Payment System JS Components -->
+<script src="{% static 'payments/js/api-client.js' %}"></script>
+<script src="{% static 'payments/js/ngrok-status.js' %}"></script>
+{% endblock %}

django_cfg/apps/payments/admin_interface/views/api/payments.py

@@ -23,7 +23,7 @@ from django_cfg.apps.payments.admin_interface.serializers import (
     AdminPaymentUpdateSerializer,
     AdminPaymentStatsSerializer,
 )
-from django_cfg.modules.django_logger import get_logger
+from django_cfg.modules.django_logging import get_logger
 
 logger = get_logger("admin_payment_api")
 

django_cfg/apps/payments/admin_interface/views/api/stats.py

@@ -12,7 +12,7 @@ from django.utils import timezone
 from datetime import timedelta
 
 
-from django_cfg.modules.django_logger import get_logger
+from django_cfg.modules.django_logging import get_logger
 from django_cfg.apps.payments.models import UniversalPayment
 from ..base import AdminReadOnlyViewSet
 from django_cfg.apps.payments.admin_interface.serializers import AdminPaymentStatsSerializer, WebhookStatsSerializer