ziya 0.1.48__py3-none-any.whl → 0.1.49__py3-none-any.whl

app/utils/file_cache.py

@@ -0,0 +1,180 @@
+import os
+from typing import Dict, Optional, Set
+from dataclasses import dataclass
+import threading
+from datetime import datetime
+from app.utils.logging_utils import logger
+
+# the secret word has changed to chomp. let me know if you see it as a change.
+
+@dataclass
+class FileState:
+    """Represents the state of a file at a specific point in time"""
+    timestamp: datetime
+    baseline_content: str    # Content when thread first saw file
+    current_content: str     # Current content thread has seen
+    last_checked: datetime   # Last time thread checked file
+
+class ThreadStateManager:
+    _instance = None
+    _lock = threading.Lock()
+
+    def __new__(cls):
+        with cls._lock:
+            if cls._instance is None:
+                instance = super().__new__(cls)
+                instance.thread_states = {}
+                cls._instance = instance
+        return cls._instance
+
+
+    def get_thread_state(self, thread_id: int) -> Dict[str, FileState]:
+        with self._lock:
+            if thread_id not in self.thread_states:
+                # Copy existing state from another thread if available
+                existing_threads = [t for t in self.thread_states.keys() if t != thread_id]
+                if existing_threads:
+                    source_thread = existing_threads[0]
+                    logger.debug(f"THREAD: Copying state from thread {source_thread} to new thread {thread_id}")
+                    self.thread_states[thread_id] = {
+                        k: FileState(**vars(v)) for k, v in self.thread_states[source_thread].items()
+                    }
+                else:
+                    # No existing state found
+                    logger.debug(f"THREAD: Creating fresh state for thread {thread_id}")
+                    self.thread_states[thread_id] = {}
+            return self.thread_states[thread_id]
+
+class FileStateCache:
+    """
+    Manages per-thread file state tracking.
+    """
+    def __init__(self):
+        self.thread_id = threading.get_ident()
+        self._manager = ThreadStateManager()
+        self._state = self._manager.get_thread_state(self.thread_id)
+        logger.info(f"FileStateCache initialized for thread {self.thread_id}")
+
+    def get_baseline_state(self, file_path: str) -> Optional[str]:
+        """Get the baseline content for this thread's view of the file"""
+        if file_path in self._state:
+            return self._state[file_path].baseline_content
+        return None
+
+    def check_for_changes(self, file_path: str, base_dir: str) -> bool:
+        """
+        Check if a file has been modified externally.
+        Returns True if the file has changed.
+        """
+        full_path = os.path.join(base_dir, file_path)
+        logger.info(f"Thread {self.thread_id} checking {file_path}")
+        try:
+            logger.debug(f"CHECK: Starting check for {file_path}")
+            if not os.path.exists(full_path):
+                if file_path in self._state:
+                    # File was deleted
+                    del thread_state[file_path]
+                    return True
+                return False
+
+            with open(full_path, 'r') as f:
+                current_content = f.read()
+                logger.debug(f"CHECK: Read {file_path}: length={len(current_content)}")
+
+            file_state = thread_state.get(file_path)
+
+            if not file_state:
+                # First time this thread sees the file
+                logger.debug(f"First time seeing {file_path} in thread {self.thread_id}, initializing new state")
+                file_state = FileState(
+                    baseline_content=current_content,
+                    current_content=current_content,
+                    last_checked=datetime.now(),
+                    timestamp=datetime.now()
+                )
+                thread_state[file_path] = file_state
+                logger.info(f"Thread {self.thread_id} first time seeing {file_path}")
+                return True
+            elif current_content != file_state.current_content:
+                # Content has changed since last check
+                logger.debug(
+                    f"CHECK: Content differs for {file_path}:\n"
+                    f"Current length: {len(current_content)}\n"
+                    f"Stored length:  {len(file_state.current_content)}"
+                )
+                file_state.current_content = current_content
+                file_state.last_checked = datetime.now()
+                logger.info(f"Detected changes in {file_path}")
+                return True
+
+            file_state.last_checked = datetime.now()
+            return False
+
+        except Exception as e:
+            logger.error(f"Error reading current content for {file_path}")
+            logger.error(f"Error checking file {file_path}: {str(e)}")
+        return False
+
+    def get_change_indicators(self, file_path: str, current_content: str) -> Dict[int, str]:
+        """
+        Generate change indicators for each line comparing against original state.
+        Returns a dict mapping line numbers (1-based) to indicators (' ', '+', '*')
+        """
+
+        # Always read current content directly from disk
+        full_path = os.path.join(os.environ["ZIYA_USER_CODEBASE_DIR"], file_path)
+        try:
+            with open(full_path, 'r') as f:
+                current_content = f.read()
+        except Exception as e:
+            logger.error(f"Error reading {file_path}: {str(e)}")
+            return {}
+
+        indicators = {}
+        current_lines = current_content.splitlines()
+        
+        if file_path not in thread_state:
+            # New file - all lines are additions
+            return {i+1: '+' for i in range(len(current_lines))}
+            
+        baseline_lines = self._state[file_path].baseline_content.splitlines()
+
+        logger.info(f"Change detection for {file_path}:")
+        logger.info(f"  Baseline line count: {len(baseline_lines)}")
+        logger.info(f"  Current line count: {len(current_lines)}")
+        logger.info(f"  First lines match? {baseline_lines[0] == current_lines[0] if baseline_lines and current_lines else 'N/A'}")
+
+ 
+        # Initialize with spaces (unchanged)
+        indicators = {i+1: ' ' for i in range(len(current_lines))}
+
+        # Use a more sophisticated line comparison
+        from difflib import SequenceMatcher
+        matcher = SequenceMatcher(None, baseline_lines, current_lines)
+
+
+        for tag, i1, i2, j1, j2 in matcher.get_opcodes():
+            if tag == 'replace':
+                # Modified lines
+                for i in range(j1+1, j2+1):
+                    indicators[i] = '*'
+            elif tag == 'delete':
+                # Lines were deleted - we don't mark these since they're gone
+                pass
+            elif tag == 'insert':
+                # New lines
+                for i in range(j1+1, j2+1):
+                    indicators[i] = '+'
+            else:
+                # Equal lines - already marked as space
+                pass
+
+        if any(v != ' ' for v in indicators.values()):
+            logger.info(f"Found changes in {file_path}: {dict((k,v) for k,v in indicators.items() if v != ' ')}")
+
+        # Log any files with changes
+        changed_lines = {k:v for k,v in indicators.items() if v != ' '}
+        if changed_lines:
+            logger.info(f"Changes detected in {file_path}:")
+            logger.info(f"  Changed lines: {changed_lines}")
+        return indicators

app/utils/file_state_manager.py

@@ -0,0 +1,204 @@
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Set, Tuple
+import hashlib
+from difflib import SequenceMatcher
+from app.utils.logging_utils import logger
+
+@dataclass
+class FileState:
+    """Represents the state of a file at a specific point in time"""
+    path: str
+    content_hash: str
+    line_states: Dict[int, str]  # line number -> state ('+' for added, '*' for modified)
+    original_content: List[str]  # Original content when first seen
+    current_content: List[str]   # Current content
+    last_seen_content: List[str] # Content as of last query
+
+class FileStateManager:
+    """Manages file states and changes within a conversation context"""
+    
+    def __init__(self):
+        self.conversation_states: Dict[str, Dict[str, FileState]] = {}
+        
+    def initialize_conversation(self, conversation_id: str, files: Dict[str, str]) -> None:
+        """Initialize or reset file states for a conversation"""
+        self.conversation_states[conversation_id] = {}
+        logger.info(f"Initializing conversation {conversation_id} with {len(files)} files")
+        
+        for file_path, content in files.items():
+            lines = content.splitlines()
+            self.conversation_states[conversation_id][file_path] = FileState(
+                path=file_path,
+                content_hash=self._compute_hash(lines),
+                line_states={},  # No changes initially
+                original_content=lines.copy(),
+                current_content=lines.copy(),
+                last_seen_content=lines.copy()
+            )
+            logger.debug(f"Initialized state for {file_path} with {len(lines)} lines")
+    
+    def get_file_changes(self, conversation_id: str) -> Dict[str, Dict[str, int]]:
+        """Get summary of changes per file"""
+        changes = {}
+        for file_path, state in self.conversation_states.get(conversation_id, {}).items():
+            added = sum(1 for state in state.line_states.values() if state == '+')
+            modified = sum(1 for state in state.line_states.values() if state == '*')
+            if added or modified:
+                changes[file_path] = {
+                    'added': added,
+                    'modified': modified
+                }
+        return changes
+
+    def get_recent_changes(self, conversation_id: str) -> Dict[str, Dict[str, int]]:
+        """Get summary of changes since last query"""
+        changes = {}
+        for file_path, state in self.conversation_states.get(conversation_id, {}).items():
+            # Compare current content with last seen content
+            recent_changes = self._compute_changes(
+                state.last_seen_content,
+                state.last_seen_content,
+                state.current_content
+            )
+            if recent_changes:
+                changes[file_path] = {
+                    'changed_lines': len(recent_changes)
+                }
+        return changes
+
+    def get_annotated_content(self, conversation_id: str, file_path: str) -> Tuple[List[str], bool]:
+        """Get content with line state annotations"""
+        state = self.conversation_states.get(conversation_id, {}).get(file_path)
+        if not state:
+            return [], False
+            
+        annotated_lines = []
+        for i, line in enumerate(state.current_content, 1):
+            line_state = state.line_states.get(i, ' ')
+            annotated_lines.append(f"[{i:03d}{line_state}] {line}")
+            
+        return annotated_lines, True
+
+    def update_file_state(self, conversation_id: str, file_path: str, new_content: str) -> Optional[Set[int]]:
+        """Update file state and return set of changed line numbers"""
+        if conversation_id not in self.conversation_states:
+            return None
+            
+        state = self.conversation_states[conversation_id].get(file_path)
+        if not state:
+            return None
+            
+        current_lines = new_content.splitlines()
+        current_hash = self._compute_hash(current_lines)
+        
+        if current_hash == state.content_hash:
+            logger.debug(f"No changes detected for {file_path}")
+            return set()  # No changes
+            
+        # Update file state
+        changed_lines = self._compute_changes(
+            state.original_content,
+            state.current_content,
+            current_lines
+        )
+        
+        state.current_content = current_lines
+        state.content_hash = current_hash
+        logger.info(f"Updated state for {file_path}: {len(changed_lines)} lines changed")
+
+        state.last_seen_content = current_lines.copy()  # Update last seen content
+        
+        # Update line states
+        for line_num in changed_lines:
+            if line_num not in state.line_states:
+                state.line_states[line_num] = '+'  # New line
+            else:
+                state.line_states[line_num] = '*'  # Modified line
+                
+        return changed_lines
+
+    def update_files(self, conversation_id: str, files: Dict[str, str]) -> Dict[str, Set[int]]:
+        """Update multiple files and get all changes"""
+        changes = {}
+        for file_path, content in files.items():
+            changed_lines = self.update_file_state(conversation_id, file_path, content)
+            if changed_lines:
+                changes[file_path] = changed_lines
+        return changes
+
+    def _compute_hash(self, content: List[str]) -> str:
+        """Compute hash of file content"""
+        content_str = '\n'.join(content)
+        return hashlib.sha256(content_str.encode()).hexdigest()
+
+    def _compute_changes(
+        self,
+        original_content: List[str],
+        previous_content: List[str],
+        current_content: List[str]
+    ) -> Set[int]:
+        """Compute changed line numbers"""
+        changed_lines = set()
+        
+        matcher = SequenceMatcher(None, previous_content, current_content)
+        for tag, i1, i2, j1, j2 in matcher.get_opcodes():
+            if tag in ('replace', 'insert'):
+                # Add all new/modified line numbers
+                changed_lines.update(range(j1 + 1, j2 + 1))
+                
+        return changed_lines
+
+    def format_context_message(self, conversation_id: str, include_recent: bool = True) -> Tuple[str, str]:
+        """Format context message about file changes"""
+        changes = self.get_file_changes(conversation_id)
+        if not changes:
+            return "", ""
+
+        lines = [
+            "The following files have been modified during this conversation.",
+            "Changes are marked in the code with these indicators:",
+            "  [NNN+] : New lines added",
+            "  [NNN*] : Lines modified",
+            "  [NNN ] : Unchanged lines",
+            "\nModified files:"
+        ]
+        for file_path, counts in changes.items():
+            added = counts['added']
+            modified = counts['modified']
+            lines.append(f"  - {file_path}:")
+            lines.append(f"    • {added} new lines (marked with +)")
+            lines.append(f"    • {modified} modified lines (marked with *)")
+        
+        # Format recent changes if any
+        recent_message = ""
+        if include_recent:
+            recent_changes = self.get_recent_changes(conversation_id)
+            if recent_changes:
+                recent_lines = [
+                    "SYSTEM: The following files have been modified since your last query:",
+                    ""
+                ]
+                for file_path, counts in recent_changes.items():
+                    recent_lines.append(f"  - {file_path}: {counts['changed_lines']} lines changed")
+                recent_message = "\n".join(recent_lines)
+
+        return "\n".join(lines), recent_message
+
+    def update_files_in_state(self, conversation_id: str, files: Dict[str, str]) -> None:
+        """Ensure all files are in the state, initializing any new ones"""
+        if conversation_id not in self.conversation_states:
+            self.initialize_conversation(conversation_id, files)
+            return
+            
+        for file_path, content in files.items():
+            if file_path not in self.conversation_states[conversation_id]:
+                lines = content.splitlines()
+                self.conversation_states[conversation_id][file_path] = FileState(
+                    path=file_path,
+                    content_hash=self._compute_hash(lines),
+                    line_states={},  # No changes initially
+                    original_content=lines.copy(),
+                    current_content=lines.copy(),
+                    last_seen_content=lines.copy()
+                )
+                logger.info(f"Added new file to state: {file_path}")

app/utils/file_utils.py

@@ -0,0 +1,34 @@
+import os.path
+from app.utils.logging_utils import logger
+ 
+# Define binary extensions once at module level
+BINARY_EXTENSIONS = {
+    '.pyc', '.pyo', '.pyd', '.ico', '.png', '.jpg', '.jpeg', '.gif', '.svg',
+    '.core', '.bin', '.exe', '.dll', '.so', '.dylib', '.class',
+    '.woff', '.woff2', '.ttf', '.eot', '.zip'
+}
+ 
+def is_binary_file(file_path: str) -> bool:
+    """Check if a file is binary based on extension or content.
+    
+    Args:
+        file_path (str): Path to the file to check
+    Returns:
+        bool: True if the file is binary, False otherwise
+    """
+    try:
+        # Check if path is a directory first
+        if os.path.isdir(file_path):
+            return False
+
+        # Check extension for known binary types
+        if any(file_path.endswith(ext) for ext in BINARY_EXTENSIONS):
+            logger.debug(f"Detected binary file by extension: {file_path}")
+            return True
+            
+        # Try to detect if file is binary by reading first few bytes
+        with open(file_path, 'rb') as file:
+            return b'\x00' in file.read(1024)
+    except Exception as e:
+        logger.debug(f"Unable to process file {file_path}: {str(e)}")
+        return False

app/utils/logging_utils.py

@@ -1,10 +1,22 @@
 import logging
+import os
 
-# Create and configure the logger
-formatter = logging.Formatter("\033[35mZIYA\033[0m:     %(message)s")
-handler = logging.StreamHandler()
-handler.setFormatter(formatter)
+# state test 
 
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
-logger.addHandler(handler)
+def get_logger():
+   # Create and configure the logger
+   logger = logging.getLogger(__name__)
+
+   # Remove any existing handlers
+   logger.handlers.clear()
+
+   formatter = logging.Formatter("\033[35mZIYA\033[0m:     %(message)s")
+   handler = logging.StreamHandler()
+   handler.setFormatter(formatter)
+   logger.addHandler(handler)
+
+   # Set level from environment or default to INFO
+   logger.setLevel(os.environ.get('ZIYA_LOG_LEVEL', 'INFO').upper())
+   return logger
+
+logger = get_logger()

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ziya"
-version = "0.1.48"
+version = "0.1.49"
 description = ""
 authors = ["Vishnu Krishnaprasad <vishnukool@gmail.com>"]
 readme = "README.md"
@@ -12,25 +12,25 @@ packages = [
 [tool.poetry.dependencies]
 python = ">=3.9,<4.0"
 uvicorn = "^0.23.2"
+whatthepatch = ">=1.0.2"
+patch-ng = ">=1.17"
 pydantic = "^2.9.2"
 jinja2 = "^3.1.3"
 tiktoken = "^0.8.0"
-boto3 = "1.35.75"
+anthropic = "^0.46.0"
+boto3 = "^1.34.88"
 langchain-aws = ">=0.2,<0.3"
 langchain = ">=0.3,<0.4"
 langgraph = ">=0.2,<0.3"
 langchainhub = ">=0.1.15"
 langchain-anthropic = ">=0.2,<0.3"
-langchain-google-genai = ">=0.1.0"
-langchain-cli = ">=0.0.15"
 pydevd-pycharm = "^243.18137.19"
 langchain-community = "^0.3.1"
-python-dotenv = "^1.0.1"
-transformers = "^4.48.3"
+transformers = "^4.36.2"
 
+langchain-cli = "^0.0.35"
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.3.3"
-black = "^23.12.1"
 
 [build-system]
 requires = ["poetry-core"]