django-cfg 1.1.82__py3-none-any.whl → 1.2.1__py3-none-any.whl

django_cfg/apps/agents/toolsets/orm_toolset.py

@@ -0,0 +1,319 @@
+"""
+Django ORM toolset for database operations.
+"""
+
+import logging
+from typing import Dict, Any, List, Optional, Union
+from pydantic_ai.toolsets import AbstractToolset
+from pydantic_ai import RunContext
+from django.db import models
+from django.apps import apps
+
+from ..core.dependencies import DjangoDeps
+
+logger = logging.getLogger(__name__)
+
+
+class ORMToolset(AbstractToolset[DjangoDeps]):
+    """
+    Django ORM toolset for safe database operations.
+    
+    Provides tools for:
+    - Model queries (read-only by default)
+    - Data aggregation
+    - Relationship traversal
+    - Safe filtering
+    """
+    
+    def __init__(self, read_only: bool = True, allowed_models: Optional[List[str]] = None):
+        """
+        Initialize ORM toolset.
+        
+        Args:
+            read_only: If True, only allow read operations
+            allowed_models: List of allowed models in format "app_label.model_name"
+        """
+        self.read_only = read_only
+        self.allowed_models = set(allowed_models) if allowed_models else None
+    
+    @property
+    def id(self) -> str:
+        return "django_orm"
+    
+    def _check_model_access(self, app_label: str, model_name: str) -> bool:
+        """Check if model access is allowed."""
+        if self.allowed_models is None:
+            return True
+        
+        model_key = f"{app_label}.{model_name}"
+        return model_key in self.allowed_models
+    
+    def _get_model(self, app_label: str, model_name: str) -> models.Model:
+        """Get Django model class with access check."""
+        if not self._check_model_access(app_label, model_name):
+            raise PermissionError(f"Access to model '{app_label}.{model_name}' not allowed")
+        
+        try:
+            return apps.get_model(app_label, model_name)
+        except LookupError:
+            raise ValueError(f"Model '{app_label}.{model_name}' not found")
+    
+    async def count_objects(
+        self, 
+        ctx: RunContext[DjangoDeps], 
+        app_label: str, 
+        model_name: str,
+        filters: Optional[Dict[str, Any]] = None
+    ) -> int:
+        """Count objects in model with optional filters."""
+        model = self._get_model(app_label, model_name)
+        
+        queryset = model.objects.all()
+        
+        if filters:
+            # Apply safe filters
+            safe_filters = self._sanitize_filters(filters)
+            queryset = queryset.filter(**safe_filters)
+        
+        return await queryset.acount()
+    
+    async def get_object(
+        self, 
+        ctx: RunContext[DjangoDeps], 
+        app_label: str, 
+        model_name: str,
+        object_id: Union[int, str]
+    ) -> Optional[Dict[str, Any]]:
+        """Get single object by ID."""
+        model = self._get_model(app_label, model_name)
+        
+        try:
+            obj = await model.objects.aget(pk=object_id)
+            return self._serialize_object(obj)
+        except model.DoesNotExist:
+            return None
+    
+    async def list_objects(
+        self, 
+        ctx: RunContext[DjangoDeps], 
+        app_label: str, 
+        model_name: str,
+        filters: Optional[Dict[str, Any]] = None,
+        limit: int = 10,
+        offset: int = 0,
+        order_by: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """List objects with pagination and filtering."""
+        model = self._get_model(app_label, model_name)
+        
+        queryset = model.objects.all()
+        
+        # Apply filters
+        if filters:
+            safe_filters = self._sanitize_filters(filters)
+            queryset = queryset.filter(**safe_filters)
+        
+        # Apply ordering
+        if order_by:
+            # Sanitize order_by field
+            order_field = order_by.lstrip('-')
+            if hasattr(model, order_field):
+                queryset = queryset.order_by(order_by)
+        
+        # Get total count
+        total_count = await queryset.acount()
+        
+        # Apply pagination
+        queryset = queryset[offset:offset + limit]
+        
+        # Serialize objects
+        objects = []
+        async for obj in queryset:
+            objects.append(self._serialize_object(obj))
+        
+        return {
+            'objects': objects,
+            'total_count': total_count,
+            'limit': limit,
+            'offset': offset,
+            'has_next': offset + limit < total_count,
+            'has_previous': offset > 0,
+        }
+    
+    async def aggregate_data(
+        self, 
+        ctx: RunContext[DjangoDeps], 
+        app_label: str, 
+        model_name: str,
+        aggregations: Dict[str, str],
+        filters: Optional[Dict[str, Any]] = None,
+        group_by: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """Perform aggregations on model data."""
+        from django.db.models import Count, Sum, Avg, Max, Min
+        
+        model = self._get_model(app_label, model_name)
+        
+        queryset = model.objects.all()
+        
+        # Apply filters
+        if filters:
+            safe_filters = self._sanitize_filters(filters)
+            queryset = queryset.filter(**safe_filters)
+        
+        # Prepare aggregation functions
+        agg_functions = {
+            'count': Count,
+            'sum': Sum,
+            'avg': Avg,
+            'max': Max,
+            'min': Min,
+        }
+        
+        agg_kwargs = {}
+        for alias, agg_spec in aggregations.items():
+            if ':' in agg_spec:
+                func_name, field_name = agg_spec.split(':', 1)
+            else:
+                func_name = agg_spec
+                field_name = 'id'  # Default field
+            
+            if func_name.lower() in agg_functions:
+                agg_func = agg_functions[func_name.lower()]
+                if func_name.lower() == 'count':
+                    agg_kwargs[alias] = agg_func(field_name)
+                else:
+                    agg_kwargs[alias] = agg_func(field_name)
+        
+        # Perform aggregation
+        if group_by and hasattr(model, group_by):
+            # Group by field
+            result = []
+            async for item in queryset.values(group_by).annotate(**agg_kwargs):
+                result.append(item)
+            return {'grouped_results': result}
+        else:
+            # Simple aggregation
+            result = await queryset.aaggregate(**agg_kwargs)
+            return result
+    
+    async def search_objects(
+        self, 
+        ctx: RunContext[DjangoDeps], 
+        app_label: str, 
+        model_name: str,
+        search_fields: List[str],
+        query: str,
+        limit: int = 10
+    ) -> List[Dict[str, Any]]:
+        """Search objects using icontains on specified fields."""
+        from django.db.models import Q
+        
+        model = self._get_model(app_label, model_name)
+        
+        # Build search query
+        search_q = Q()
+        for field in search_fields:
+            if hasattr(model, field):
+                search_q |= Q(**{f"{field}__icontains": query})
+        
+        if not search_q:
+            return []
+        
+        # Execute search
+        queryset = model.objects.filter(search_q)[:limit]
+        
+        results = []
+        async for obj in queryset:
+            results.append(self._serialize_object(obj))
+        
+        return results
+    
+    async def get_related_objects(
+        self, 
+        ctx: RunContext[DjangoDeps], 
+        app_label: str, 
+        model_name: str,
+        object_id: Union[int, str],
+        relation_name: str,
+        limit: int = 10
+    ) -> List[Dict[str, Any]]:
+        """Get related objects through foreign key or many-to-many."""
+        model = self._get_model(app_label, model_name)
+        
+        try:
+            obj = await model.objects.aget(pk=object_id)
+        except model.DoesNotExist:
+            return []
+        
+        # Check if relation exists
+        if not hasattr(obj, relation_name):
+            return []
+        
+        relation = getattr(obj, relation_name)
+        
+        # Handle different relation types
+        if hasattr(relation, 'all'):
+            # Many-to-many or reverse foreign key
+            related_objects = relation.all()[:limit]
+        else:
+            # Single related object
+            if relation:
+                return [self._serialize_object(relation)]
+            else:
+                return []
+        
+        results = []
+        async for related_obj in related_objects:
+            results.append(self._serialize_object(related_obj))
+        
+        return results
+    
+    def _sanitize_filters(self, filters: Dict[str, Any]) -> Dict[str, Any]:
+        """Sanitize filters to prevent dangerous operations."""
+        safe_filters = {}
+        
+        # Allowed lookup types
+        allowed_lookups = {
+            'exact', 'iexact', 'contains', 'icontains', 'startswith', 'istartswith',
+            'endswith', 'iendswith', 'gt', 'gte', 'lt', 'lte', 'in', 'isnull',
+            'date', 'year', 'month', 'day', 'week_day', 'hour', 'minute', 'second'
+        }
+        
+        for key, value in filters.items():
+            # Check for dangerous operations
+            if '__' in key:
+                field, lookup = key.rsplit('__', 1)
+                if lookup in allowed_lookups:
+                    safe_filters[key] = value
+            else:
+                # Direct field lookup (exact)
+                safe_filters[key] = value
+        
+        return safe_filters
+    
+    def _serialize_object(self, obj: models.Model) -> Dict[str, Any]:
+        """Serialize Django model object to dictionary."""
+        data = {}
+        
+        for field in obj._meta.fields:
+            value = getattr(obj, field.name)
+            
+            # Handle special field types
+            if hasattr(value, 'isoformat'):
+                # DateTime fields
+                data[field.name] = value.isoformat()
+            elif hasattr(value, '__dict__') and hasattr(value, 'pk'):
+                # Related objects - just include ID and str representation
+                data[field.name] = {
+                    'id': value.pk,
+                    'str': str(value)
+                }
+            else:
+                data[field.name] = value
+        
+        # Add primary key and string representation
+        data['pk'] = obj.pk
+        data['str'] = str(obj)
+        
+        return data

django_cfg/apps/agents/urls.py

@@ -0,0 +1,46 @@
+"""
+Django Orchestrator URLs.
+
+Provides API endpoints for agent management and execution.
+"""
+
+from django.urls import path, include
+from rest_framework.routers import DefaultRouter
+
+# Import views when they're created
+# from .api.views import AgentViewSet, WorkflowViewSet, ExecutionViewSet
+
+app_name = 'django_orchestrator'
+
+# Create router for API endpoints
+router = DefaultRouter()
+
+# Register viewsets when they're created
+# router.register(r'agents', AgentViewSet, basename='agent')
+# router.register(r'workflows', WorkflowViewSet, basename='workflow')
+# router.register(r'executions', ExecutionViewSet, basename='execution')
+
+urlpatterns = [
+    # API endpoints
+    path('api/', include(router.urls)),
+    
+    # Health check endpoint
+    path('health/', lambda request: JsonResponse({'status': 'ok', 'service': 'django-orchestrator'}), name='health'),
+]
+
+# For now, create a simple health endpoint
+from django.http import JsonResponse
+
+def health_check(request):
+    """Health check endpoint for Django Orchestrator."""
+    return JsonResponse({
+        'status': 'ok',
+        'service': 'django-orchestrator',
+        'version': '0.1.0'
+    })
+
+# Update urlpatterns with the actual function
+urlpatterns = [
+    path('api/', include(router.urls)),
+    path('health/', health_check, name='health'),
+]

django_cfg/apps/knowbase/README.md

@@ -0,0 +1,150 @@
+# Django CFG Knowledge Base
+
+Enterprise-grade RAG (Retrieval-Augmented Generation) application built on Django with pgvector semantic search, Dramatiq background processing, and comprehensive API endpoints for document management and AI-powered chat.
+
+## Features
+
+- 📄 **Document Management**: Upload, process, and manage documents with automatic chunking
+- 🔍 **Semantic Search**: pgvector-powered cosine similarity search
+- 🤖 **AI Chat**: RAG-powered conversational interface with context retrieval
+- 👥 **Multi-tenant**: Complete user isolation and access control
+- ⚡ **Background Processing**: Async document processing with Dramatiq
+- 💰 **Cost Tracking**: Monitor LLM usage and costs
+- 🎨 **Admin Interface**: Beautiful Unfold-styled admin with statistics
+- 🔒 **Type Safety**: Pydantic v2 validation throughout
+- ✅ **Full Testing**: Comprehensive test coverage
+
+## Quick Start
+
+### 1. Enable in Configuration
+
+```python
+from django_cfg import DjangoConfig
+
+class MyConfig(DjangoConfig):
+    enable_knowbase: bool = True
+    
+    # Required for AI features
+    openai_api_key: str = "${OPENAI_API_KEY}"
+    
+    # Optional: Configure similarity thresholds
+    knowbase_document_threshold: float = 0.7
+    knowbase_archive_threshold: float = 0.6
+```
+
+### 2. Add to INSTALLED_APPS
+
+```python
+INSTALLED_APPS = [
+    # ... other apps
+    'django_cfg.apps.knowbase',
+]
+```
+
+### 3. Run Migrations
+
+```bash
+python manage.py migrate
+```
+
+### 4. Start Using
+
+The Knowledge Base will be available at `/cfg/knowbase/` with:
+
+- **Admin Interface**: Document and chat management
+- **API Endpoints**: RESTful API for integration
+- **Chat Interface**: AI-powered conversational search
+
+## Architecture
+
+### Models
+- **Document**: File storage and metadata
+- **DocumentChunk**: Text chunks with embeddings
+- **ChatSession**: Conversation management
+- **ChatMessage**: Individual messages with context
+
+### Services
+- **DocumentService**: Document processing and management
+- **ChatService**: AI chat with RAG capabilities
+- **SearchService**: Semantic search across all content types
+
+### Background Tasks
+- Document processing and vectorization
+- Embedding generation and optimization
+- Maintenance and cleanup tasks
+
+## Configuration Options
+
+```python
+class MyConfig(DjangoConfig):
+    # Enable/disable the app
+    enable_knowbase: bool = True
+    
+    # AI Configuration
+    openai_api_key: str = "${OPENAI_API_KEY}"
+    
+    # Search Thresholds
+    knowbase_document_threshold: float = 0.7  # Document similarity
+    knowbase_archive_threshold: float = 0.6   # Code similarity
+    
+    # Processing Settings
+    knowbase_chunk_size: int = 1000           # Text chunk size
+    knowbase_overlap_size: int = 200          # Chunk overlap
+    knowbase_batch_size: int = 50             # Embedding batch size
+```
+
+## API Usage
+
+### Document Upload
+
+```python
+import requests
+
+response = requests.post('/cfg/knowbase/api/documents/', {
+    'title': 'My Document',
+    'file': open('document.pdf', 'rb')
+})
+```
+
+### Chat Query
+
+```python
+response = requests.post('/cfg/knowbase/api/chat/sessions/{session_id}/query/', {
+    'query': 'What is machine learning?',
+    'max_tokens': 500
+})
+```
+
+### Search Documents
+
+```python
+response = requests.get('/cfg/knowbase/api/search/', {
+    'query': 'artificial intelligence',
+    'limit': 10
+})
+```
+
+## Integration with External Apps
+
+The Knowledge Base supports integration with external Django apps through the `ExternalDataMixin`:
+
+```python
+from django_cfg.apps.knowbase.mixins import ExternalDataMixin
+
+class MyModel(ExternalDataMixin, models.Model):
+    name = models.CharField(max_length=100)
+    description = models.TextField()
+    
+    # That's it! Auto AI integration enabled
+```
+
+## Requirements
+
+- **PostgreSQL** with pgvector extension
+- **Redis** for Dramatiq task queue
+- **OpenAI API Key** for embeddings and chat
+- **Python 3.11+** and **Django 5.0+**
+
+## License
+
+Part of Django CFG - MIT License

django_cfg/apps/knowbase/__init__.py

@@ -0,0 +1,27 @@
+"""
+Django SaaS Knowledge Assistant
+
+Enterprise-grade RAG (Retrieval-Augmented Generation) application built on Django 5.2
+with pgvector semantic search, Dramatiq background processing, and comprehensive
+API endpoints for document management and AI-powered chat.
+
+Key Features:
+- Document ingestion with automatic chunking and embedding generation
+- Semantic search using pgvector cosine similarity
+- RAG-powered chat with context retrieval
+- Multi-tenant user isolation
+- Background processing with Dramatiq
+- Cost tracking for LLM usage monitoring
+- Comprehensive admin interface with Unfold styling
+- Type-safe APIs with Pydantic v2 validation
+- Full test coverage
+
+Architecture:
+- Models: Document, DocumentChunk, ChatSession, ChatMessage
+- Services: DocumentService, ChatService, SearchService
+- Tasks: Async document processing, maintenance, optimization
+- APIs: REST endpoints with DRF and OpenAPI documentation
+- Admin: Unfold-optimized interfaces with statistics
+"""
+
+default_app_config = 'django_cfg.apps.knowbase.apps.KnowbaseConfig'

django_cfg/apps/knowbase/admin/__init__.py

@@ -0,0 +1,23 @@
+"""
+Knowledge Base Admin Configuration
+
+Unfold-optimized admin interfaces for knowledge management.
+"""
+
+from .document_admin import *
+from .chat_admin import *
+from .archive_admin import *
+from .external_data_admin import *
+
+__all__ = [
+    'DocumentCategoryAdmin',
+    'DocumentAdmin',
+    'DocumentChunkAdmin',
+    'DocumentArchiveAdmin',
+    'ArchiveItemAdmin',
+    'ArchiveItemChunkAdmin',
+    'ExternalDataAdmin',
+    'ExternalDataChunkAdmin',
+    'ChatSessionAdmin',
+    'ChatMessageAdmin',
+]