django-dynamic-query 0.1.0__py3-none-any.whl

django_dynamic_query/apps.py

@@ -0,0 +1,33 @@
+from django.apps import AppConfig
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+
+class DjangoDynamicQueryConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'django_dynamic_query'
+    verbose_name = "Django Dynamic Query Builder"
+
+    def ready(self):
+        """
+        Checking the settings.py when django starts, without making any problem for memory for model loading
+        """
+        config = getattr(settings, 'DJANGO_DYNAMIC_QUERY', {})
+        enable_nlp = config.get('ENABLE_NLP', False)
+        
+        if enable_nlp:
+            model_type = config.get("MODEL_TYPE", "local")
+            
+            if model_type == "local":
+                model_name = config.get("MODEL_NAME")
+                model_path = config.get("MODEL_PATH")
+                
+                if not model_name and not model_path:
+                    raise ImproperlyConfigured(
+                        "MODEL_NAME or MODEL_PATH must be provided when MODEL_TYPE is set to 'local'."
+                    )
+            elif model_type == "api":
+                if not config.get("AI_API_KEY"):
+                    raise ImproperlyConfigured(
+                        "API_KEY must be provided when MODEL_TYPE is set to 'api'."
+                    )
+                print("[Django Dynamic Query] API NLP mode verified.")

django_dynamic_query/core/builder.py

@@ -0,0 +1,153 @@
+from django.db.models import Q, Count, Sum, Avg, Min, Max
+from django.core.exceptions import FieldError, ValidationError 
+from .validators import validate_query_structure
+
+AGGREGATE_FUNCTIONS = {
+    "Count": Count,
+    "Sum": Sum,
+    "Avg": Avg,
+    "Min": Min,
+    "Max": Max,
+}
+
+
+class DynamicQueryBuilder:
+    def __init__(self, model_label: str, query_data: dict):
+        """
+        Initialize the DynamicQueryBuilder with a model label and filters.
+        :param model_label: The label of the model in the format 'app_label.ModelName'.
+        :param query_data: A list of filter dictionaries, each containing 'field', 'operator', and 'value'.
+        """
+        
+        self.query_data = query_data or {}
+        
+        self.model = validate_query_structure(model_label, query_data)
+        
+    def _parse_node(self, node: dict) -> Q:
+        if "filters" in node:
+            logic = node.get("logic", "AND").upper()
+            group_q = Q()
+            for sub_node in node["filters"]:
+                sub_q = self._parse_node(sub_node)
+                if logic == "OR":
+                    group_q = group_q | sub_q if group_q else sub_q
+                else:
+                    group_q &= sub_q
+            return group_q
+        else:
+            field = node.get("field")
+            operator = node.get("operator", "exact")
+            value = node.get("value")
+            negate = node.get("negate", False)
+            
+            lookup = field if operator == "exact" else f"{field}__{operator}"
+            q_obj = Q(**{lookup: value})
+            return ~q_obj if negate else q_obj
+    
+    def validate_query_syntax(self):
+        """
+        Validate the syntax of the query data Before generating ORM code.
+        """
+        print('validating query syntax...')
+        try:
+            queryset = self.model.objects.none()
+            
+            if self.query_data.get("filters"):
+                q_expression = self._parse_node(self.query_data)
+                queryset = queryset.filter(q_expression)
+            
+            annotations = self.query_data.get("annotations", {})
+            if annotations:
+                anno_kwargs = {}
+                for alias, info in annotations.items():
+                    func = info.get("function")
+                    field = info.get("field")
+                    if func in AGGREGATE_FUNCTIONS:
+                        anno_kwargs[alias] = AGGREGATE_FUNCTIONS[func](field)
+                queryset = queryset.annotate(**anno_kwargs)
+            
+            aggregates = self.query_data.get("aggregates", {})
+            if aggregates:
+                aggr_kwargs = {}
+                for alias, info in aggregates.items():
+                    func = info.get("function")
+                    field = info.get("field")
+                    if func in AGGREGATE_FUNCTIONS:
+                        aggr_kwargs[alias] = AGGREGATE_FUNCTIONS[func](field)
+                queryset.aggregate(**aggr_kwargs)
+            
+            return None
+        
+        except (FieldError, ValidationError, TypeError, ValueError) as e:
+            return str(e)
+        except Exception as e:
+            return f"Unexpected error during query validation: {str(e)}"
+
+    def _generate_orm_string_recursive(self, node: dict) -> str:
+        """
+        Recursively generate an ORM string representation of the query data.
+        """
+        if "filters" in node:
+            logic = node.get("logic", "AND").upper()
+            join_str = " | " if logic == "OR" else " & "
+            
+            parts = [self._generate_orm_string_recursive(sub_node) for sub_node in node["filters"]]
+            if len(parts) == 1:
+                return parts[0]
+            return f"({join_str.join(parts)})"
+        else:
+            field = node.get("field")
+            operator = node.get("operator", "exact")
+            value = node.get("value")
+            negate = node.get("negate", False)
+            
+            lookup = field if operator == "exact" else f"{field}__{operator}"
+            orm_str = f"Q({lookup}={repr(value)})"
+            
+            return f"~{orm_str}" if negate else orm_str
+        
+    def build_orm_code(self) -> str:
+        """
+        Generate a string representation of the ORM code for the query.
+        """
+        model_name = self.model.__name__
+        code = f"{model_name}.objects"
+        
+        if self.query_data.get("filters"):
+            q_expression = self._generate_orm_string_recursive(self.query_data)
+            code += f".filter({q_expression})"
+        else:
+            code += ".all()"
+        
+        annotations = self.query_data.get("annotations", {})
+        if annotations:
+            anno_parts = []
+            for alias, info in annotations.items():
+                func = info.get("function")
+                field = info.get("field")
+                if func in AGGREGATE_FUNCTIONS:
+                    anno_parts.append(f"{alias}={func}('{field}')")
+            if anno_parts:
+                code += f".annotate({', '.join(anno_parts)})"
+        
+        order_by = self.query_data.get("order_by")
+        if order_by:
+            if isinstance(order_by, list):
+                order_by_str = ", ".join(repr(field) for field in order_by)
+                code += f".order_by({order_by_str})"
+            elif isinstance(order_by, str):
+                code += f".order_by({repr(order_by)})"
+                
+        aggregates = self.query_data.get("aggregates", {})
+        if aggregates:
+            aggr_parts = []
+            for alias, info in aggregates.items():
+                func = info.get("function")
+                field = info.get("field")
+                if func in AGGREGATE_FUNCTIONS:
+                    aggr_parts.append(f"{alias}={func}('{field}')")
+            if aggr_parts:
+                code += f".aggregate({', '.join(aggr_parts)})"
+        
+        return code
+        

django_dynamic_query/core/exceptions.py

@@ -0,0 +1,18 @@
+class DynamicQueryError(Exception):
+    """Base exception for dynamic query errors."""
+    pass
+
+
+class InvalidModelError(DynamicQueryError):
+    """Raised when an invalid model is specified."""
+    pass
+
+
+class InvalidFieldError(DynamicQueryError):
+    """Raised when an invalid field is specified."""
+    pass
+
+class InvalidOperatorError(DynamicQueryError):
+    """Raised when an invalid operator is specified."""
+    pass
+

django_dynamic_query/core/validators.py

@@ -0,0 +1,65 @@
+from django.apps import apps
+from django.core.exceptions import FieldDoesNotExist
+from .exceptions import DynamicQueryError, InvalidModelError, InvalidFieldError, InvalidOperatorError
+
+
+VALID_OPERATORS = {
+    'exact', 'iexact', 'contains', 'icontains', 'in', 'gt', 'gte', 'lt', 'lte',
+    'startswith', 'istartswith', 'endswith', 'iendswith', 'range', 'year', 'month',
+    'day', 'week_day', 'isnull', 'search', 'regex', 'iregex'
+}
+
+SENSITIVE_FIELDS = {'password', 'is_staff', 'is_superuser', 'last_login', 'date_joined'}
+
+
+def _validate_field_path(model, field_path: str):
+    """
+    field authencity check for nested fields in JSON
+    """
+    parts = field_path.split("__")
+    current_model = model
+    
+    for i, part in enumerate(parts):
+        if part in SENSITIVE_FIELDS:
+            raise InvalidFieldError(f"Access to sensitive field '{part}' is not allowed.")
+        
+        try:
+            field = current_model._meta.get_field(part)
+        except FieldDoesNotExist:
+            raise InvalidFieldError(f"Invalid field '{part}' for model '{current_model._meta.label}'.")
+        
+        if field.is_relation and i < len(parts) - 1:
+            current_model = field.related_model
+            
+
+def validate_query_structure(model_label: str, query_data: dict) -> type:
+    """
+    Validate the structure of the query data.
+    """
+    try:
+        model = apps.get_model(model_label)
+    except (LookupError, ValueError):
+        raise InvalidModelError(f"Invalid model label: {model_label}")
+    
+    def _validate_node(node):
+        if not isinstance(node, dict):
+            raise DynamicQueryError(
+                f"AI Generated an invalid nested filter structure. "
+                f"Expected a JSON object (dict), but received a string/primitive: '{node}'"
+            )
+        if "filters" in node:
+            for sub_node in node["filters"]:
+                _validate_node(sub_node)
+        else:
+            field_name = node.get("field")
+            operator = node.get("operator", "exact")
+            
+            if not field_name:
+                raise InvalidFieldError("Field name is required in filter.")
+            if operator not in VALID_OPERATORS:
+                raise InvalidOperatorError(f"Invalid operator '{operator}' for field '{field_name}'.")
+            
+            _validate_field_path(model, field_name)
+    
+    _validate_node(query_data)
+    return model   

django_dynamic_query/management/commands/index_schema.py

@@ -0,0 +1,12 @@
+from django.core.management.base import BaseCommand
+from django_dynamic_query.nlp.schema_rag import DjangoSchemaRAG
+
+class Command(BaseCommand):
+    help = "Indexes the host Django project models and fields into the Vector DB for RAG."
+    
+    def handle(self, *args, **options):
+        self.stdout.write("Scanning project schema...")
+        rag = DjangoSchemaRAG()
+        result = rag.index_project_models()
+        self.stdout.write(self.style.SUCCESS(result))
+        

django_dynamic_query/nlp/ai_handler.py

@@ -0,0 +1,149 @@
+import requests
+import json
+import re
+
+from django.apps import apps
+from django.conf import settings
+
+from openai import OpenAI
+
+class DjangoAIQueryTransformer:
+    def __init__(self, model_label: str, rag_fields: list = None):
+        self.model_label = model_label
+        self.rag_fields = rag_fields or []
+        
+        self.config = getattr(settings, "DJANGO_DYNAMIC_QUERY", {
+            "ENABLE_NLP": False,
+            "MODEL_TYPE": "local",
+            "MODEL_PATH": "",
+            "MODEL_NAME": "gemma3:1b",
+            "AI_API_KEY": ""
+        })
+    
+    def generate_query_json(self, user_text: str) -> dict:
+        """
+        generate query json from user_text and request to cloud or local LLM
+        """
+        if not self.config.get("ENABLE_NLP", False):
+            return {"error": "NLP feature is disabled in DJANGO_DYNAMIC_QUERY settings."}
+        
+        # relevant_fields = self.rag.retrieve_all_model_fields(self.model_label)
+        
+        # if not relevant_fields:
+        #     return {"error": "RAG could not find any matching fields. Please run 'manage.py index_schema'."}
+        
+        fields_context = "\n".join([f"- Name: '{f['name']}', Type: {f['type']}" for f in self.rag_fields])                
+        
+        system_prompt = f"""
+        You are a backend assistant that converts natural language into structured JSON filters for Django ORM.
+        Target Django Model: '{self.model_label}'
+        Available fields for this model:
+        {fields_context}
+        
+        CRITICAL JSON STRUCTURE RULES:
+        1. Every item inside the "filters" array MUST be a JSON Object (dictionary), NEVER a string.
+        2. A valid filter object format: {{"field": "field_name", "operator": "exact/contains/gt/lt", "value": "extracted_value"}}
+        
+        Return ONLY a raw valid JSON object with keys: "logic", "filters", "annotations", "aggregates". Do not include markdown code blocks.
+        """
+        
+        model_type = self.config.get('MODEL_TYPE', 'local').lower()
+        model_name = self.config.get('MODEL_NAME', "gemma3:1b")
+
+        if model_type == "api":
+            api_key = self.config.get('AI_API_KEY')
+            return self._call_api_provider(model_name, api_key, system_prompt, user_text)
+        else:
+            return self._call_local_provider(model_name, system_prompt, user_text)
+        
+    def _call_api_provider(self, model_name: str, api_key, system_prompt, user_text) -> dict:
+        """
+        sending request to cloud services (OpenAI, Gemini, Anthropic, ...).
+        """
+        if not api_key:
+            return {"error": "AI_API_KEY is required for 'api' model type."}
+           
+        base_url = self.config.get("API_BASE_URL", None) or None
+            
+        try:
+            client = OpenAI(api_key=api_key, base_url=base_url)
+            
+            response = client.chat.completions.create(
+                model=model_name,
+                messages=[
+                    {"role": "system", "content": system_prompt},
+                    {"role": "user", "content": user_text}
+                ],
+                response_format={"type": "json_object"},
+                temperature=0.1
+            )
+            return json.loads(response.choices[0].message.content)
+        except Exception as e:
+            return {"error": f"Cloud AI Provider Error: {str(e)}"}
+    
+    def _call_local_provider(self, model_name: str, system_prompt: str, user_text: str) -> dict:
+        """
+        sending request to local ollama services.
+        """
+        base_url = self.config.get("MODEL_PATH", "").strip()
+        if not base_url:
+            base_url = "http://127.0.0.1:11434/api/chat"
+        
+        payload = {
+            "model": model_name,
+            "messages": [
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": user_text}
+            ],
+            "stream": False,
+            "format": "json",
+            "options": {
+                "temperature": 0.0,
+                "top_k": 1,
+                "top_p": 0.1
+            }
+        }
+        try:
+            response = requests.post(base_url, json=payload, timeout=120.0)
+            if response.status_code != 200:
+                try:
+                    err_json = response.json()
+                    if "error" in err_json:
+                        return {"error": f"Ollama Error: {err_json['error']}"}
+                except Exception:
+                    pass
+                return {"error": f"Ollama returned HTTP {response.status_code}"}
+                
+            response_data = response.json()
+            if "error" in response_data:
+                return {"error": f"Ollama Error: {response_data['error']}"}
+            ai_content = ""
+            if "message" in response_data and isinstance(response_data["message"], dict):
+                ai_content = response_data["message"].get("content", "")
+            elif "response" in response_data:
+                ai_content = response_data.get("response", "")
+                
+            if not ai_content:
+                return {"error": "Ollama returned an empty response or invalid JSON structure."}
+                
+            ai_content_cleaned = self._clean_markdown_json(ai_content)
+            
+            return json.loads(ai_content_cleaned)        
+        
+        except requests.exceptions.Timeout:
+            return {"error": f"Timeout occurred! Model '{model_name}' took too long to respond. Ensure your system hardware can run this model efficiently."}
+        except requests.exceptions.RequestException as req_err:
+            return {"error": f"Could not connect to Ollama. Ensure Ollama is running on {base_url}. Details: {str(req_err)}"}
+        except json.JSONDecodeError:
+            return {"error": "AI response was not a valid JSON. Try a stronger model or re-write your prompt."}
+        except Exception as e:
+            return {"error": f"Local AI Provider Error: {str(e)}"}
+        
+    def _clean_markdown_json(self, raw_response: str) -> str:
+        """
+        get cleaned json from markdown
+        """
+        match = re.search(r'```json\s*([\s\S]*?)\s*```', raw_response)
+        if match:
+            return match.group(1).strip()
+        return raw_response.strip()

django_dynamic_query/nlp/schema_rag.py

@@ -0,0 +1,230 @@
+import os
+import json
+import uuid
+import re
+from collections import Counter
+
+from django.apps import apps
+from django.conf import settings
+
+
+class DjangoSchemaRAG:
+    def __init__(self):
+        """
+        Creating Local dir for project db schema
+        """
+        self.db_dir = os.path.abspath(os.path.join(settings.BASE_DIR, ".ddq_schema_rag"))
+        
+        import chromadb
+        from chromadb.utils import embedding_functions
+        
+        self.client = chromadb.PersistentClient(path=self.db_dir)
+        self.emb_fn = embedding_functions.DefaultEmbeddingFunction()
+        
+        self.collection_name = "django_rag_v2_collection"
+        self.collection = self.client.get_or_create_collection(
+            name=self.collection_name,
+            embedding_function=self.emb_fn
+        )
+    
+    def index_project_models(self):
+        """
+        scanning all the project models and their 
+        fields and types of their data and verbose name
+        """
+        try:
+            self.client.delete_collection(name=self.collection_name)
+        except Exception as e:
+            print(f"Error deleting collection: {e}")
+            
+        self.collection = self.client.get_or_create_collection(
+            name=self.collection_name,
+            embedding_function=self.emb_fn
+        )
+            
+        documents = []
+        metadatas = []
+        ids = []
+        
+        seen_models = set()
+        
+        for model in apps.get_models():
+            if model._meta.app_label in ['admin', 'auth', 'contenttypes', 'sessions']:
+                continue  # Skip built-in Django apps
+            
+            model_label = f"{model._meta.app_label}.{model.__name__}".lower()
+            
+            if model_label in seen_models:
+                continue
+            seen_models.add(model_label)
+            
+            verbose_name = str(getattr(model._meta, 'verbose_name', model.__name__))
+            verbose_name_plural = str(getattr(model._meta, 'verbose_name_plural', verbose_name))
+            
+            fields_description = []
+            fields_payload = []
+            
+            for field in model._meta.get_fields():
+                if field.concrete and not field.many_to_many:
+                    field_name = field.name
+                    field_type = field.get_internal_type()
+                    field_verbose_name = getattr(field, 'verbose_name', field_name)
+                    help_text = getattr(field, 'help_text', '')
+                    
+                    fields_description.append(
+                        f"Field Name: '{field_name}' (Type: {field_type}, Label/Verbose: '{field_verbose_name}')"
+                    )
+                    fields_payload.append({"name": field_name, "type": field_type})
+                    
+                    document_content = (
+                        f"Django Model Class: {model.__name__} | App Label: {model._meta.app_label} | "
+                        f"Alternative Translations/Names: {verbose_name}, {verbose_name_plural} | "
+                        f"Table Columns and Fields Schema: {' - '.join(fields_description)}"
+                    )
+                    
+                    documents.append(document_content)
+                    metadatas.append({
+                        "model_label": model_label,
+                        "field_json": json.dumps(fields_payload)
+                    })
+                    ids.append(str(uuid.uuid4()))
+            
+            if documents:
+                self.collection.add(documents=documents, metadatas=metadatas, ids=ids)
+                return f"Successfully indexed {len(documents)} fields across the project."
+            return "No correct models found to index."
+    
+    def search_semantic_schema(self, user_query: str) -> dict | None:
+        """Pure RAG Flow: Semantic search with safe dict parsing and hybrid fallback"""
+        try:
+            results = self.collection.query(
+                query_texts=[user_query],
+                n_results=1
+            )
+            
+            if results and results.get('metadatas') and results['metadatas'][0]:
+                best_match_meta = results['metadatas'][0][0]
+                
+                if not best_match_meta:
+                    return None
+                    
+                model_label = best_match_meta.get("model_label")
+                fields_json = best_match_meta.get("fields_json")
+                
+                if model_label:
+                    # لایه ایمنی: اگر متادیتا به هر دلیلی در کروما None بود، داینامیک از جنگو بازسازی‌اش کن تا کراش نکند
+                    if not fields_json or not isinstance(fields_json, str):
+                        try:
+                            model = apps.get_model(model_label)
+                            fields_payload = []
+                            for field in model._meta.get_fields():
+                                if field.concrete and not field.many_to_many:
+                                    fields_payload.append({
+                                        "name": field.name, 
+                                        "type": field.get_internal_type()
+                                    })
+                            return {
+                                "model_label": model_label,
+                                "fields": fields_payload
+                            }
+                        except Exception:
+                            return None
+
+                    return {
+                        "model_label": model_label,
+                        "fields": json.loads(fields_json)
+                    }
+        except Exception as e:
+            print(f"RAG Vector Search Error: {e}")
+        return None
+    
+    def retrieve_all_model_fields(self, model_label: str) -> list:
+        """
+        ChromaDB filtering bug fix: Instead of a guesstimate vector query, 
+        fetches all fields of this specific model directly, with a 100% guarantee.
+        """
+        try:
+            target_label = model_label.lower()
+            all_data = self.collection.get()
+            
+            extracted_fields = []
+            
+            if all_data and 'metadatas' in all_data and all_data['metadatas']:
+                for meta in all_data['metadatas']:
+                    if meta.get("model_label") == target_label:
+                        extracted_fields.append({
+                            "name": meta["field_name"],
+                            "type": meta["field_type"]
+                        })
+            return extracted_fields
+        except Exception as e:
+            print(f"ChromaDB Fetch Error: {e}")
+            return []
+
+    def detect_model_from_query(self, user_query) -> str | None:
+        """
+        detect model_name like: app_label.ModelName based on user query.
+        """
+        try:
+            results = self.collection.query(
+                query_texts=[user_query],
+                n_results=5
+            )
+            if results and results["metadatas"] and results['metadatas'][0]:
+                models = [meta['model_label'] for meta in results['metadatas'][0]]
+                if models:
+                    return Counter(models).most_common(1)[0][0]
+        except Exception as e:
+            return str(e)
+        return None
+        
+        
+class DjangoSchemaRegistry:
+    """
+    100% offline and intelligent replacement for ChromaDB. 
+    Directly uses Django's own in-memory (RAM) structure. 
+    No internet, model downloads, or admin commands required.
+    """
+    
+    def detect_model_from_query(self, user_query: str) -> str | None:
+        """
+        Automatic model detection by checking the similarity of user keywords with the model name and its verbose_names.
+        Fully compatible with the Persian language.
+        """
+        user_query = user_query.lower()
+        tokens = re.findall(r'\w+', user_query)
+        
+        best_model = None
+        max_score = 0
+        
+        for model in apps.get_models():
+            if model._meta.app_label in ['admin', 'auth', 'contenttypes', 'sessions']:
+                continue
+                
+            model_label = f"{model._meta.app_label}.{model.__name__}"
+            
+            model_name = model.__name__.lower()
+            verbose_name = str(model._meta.verbose_name).lower() if hasattr(model._meta, 'verbose_name') else ""
+            verbose_name_plural = str(model._meta.verbose_name_plural).lower() if hasattr(model._meta, 'verbose_name_plural') else ""
+            
+            score = 0
+            for token in tokens:
+                if token in model_name or (verbose_name and token in verbose_name) or (verbose_name_plural and token in verbose_name_plural):
+                    score += 15  # امتیاز بالا برای تطابق مستقیم با نام جدول
+            
+            # وزن‌دهی ثانویه بر اساس فیلدهای مدل (اگر کاربر اسم فیلد را آورده باشد)
+            for field in model._meta.get_fields():
+                if not field.concrete:
+                    continue
+                field_name = field.name.lower()
+                field_verbose = str(getattr(field, 'verbose_name', '')).lower()
+                
+                for token in tokens:
+                    if token in field_name or (field_verbose and token in field_verbose):
+                        score += 3 
+            
+            if score > max_score and score > 0:
+                max_score = score
+                best_model = model_label
+                
+        return best_model