mito-ai 0.1.55__tar.gz → 0.1.56__tar.gz

mito_ai-0.1.56/.cursorrules

@@ -0,0 +1,17 @@
+# Cursor Rules for mito-ai Development
+
+## Jupyter Lab Access Token
+
+When testing Jupyter extension changes, use the token from `.cursor-jupyter-token` file.
+
+The Jupyter Lab URL format is:
+```
+http://localhost:8888/lab?token={token}
+```
+
+To get the token, read it from `.cursor-jupyter-token` in the repo root.
+
+**Important**: 
+1. This token configuration is development-only and will not affect production installations.
+The dev config file (`jupyter-config/jupyter_server_config.d/mito_ai_dev.json`) is gitignored.
+2. If the token is not there, tell the user to run dev/setup_jupyter_dev_token.py and relaunch Jupyter Lab.

{mito_ai-0.1.55 → mito_ai-0.1.56}/.gitignore

@@ -127,3 +127,7 @@ dmypy.json
 # Secrets
 .env
 secrets.tsx
+
+# Development-only Jupyter config (contains fixed token for Cursor testing)
+jupyter-config/jupyter_server_config.d/mito_ai_dev.py
+.cursor-jupyter-token

{mito_ai-0.1.55 → mito_ai-0.1.56}/CLAUDE.md

@@ -24,6 +24,10 @@ jlpm build
 jupyter labextension develop . --overwrite
 jupyter server extension enable --py mito_ai
 
+# (Optional) Setup fixed Jupyter token for Cursor agent testing
+# This creates a development-only config that allows Cursor to automatically test changes
+python dev/setup_jupyter_dev_token.py
+
 # Start development environment
 jlpm watch  # Terminal 1: auto-rebuild frontend
 jupyter lab --autoreload  # Terminal 2: run JupyterLab with backend hot reload

{mito_ai-0.1.55 → mito_ai-0.1.56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mito_ai
-Version: 0.1.55
+Version: 0.1.56
 Summary: AI chat for JupyterLab
 Project-URL: Homepage, https://trymito.io
 Project-URL: Bug Tracker, https://github.com/mito-ds/monorepo/issues
@@ -132,6 +132,10 @@ jupyter labextension develop . --overwrite
 # Start the jupyter server extension for development
 jupyter server extension enable --py mito_ai
 
+# (Optional) Setup fixed Jupyter token for Cursor agent testing
+# This creates a development-only config that allows Cursor to automatically test changes
+python dev/setup_jupyter_dev_token.py
+
 # Watch the source directory in one terminal, automatically rebuilding when needed
 # In case of Error: If this command fails because the lib directory was not created (the error will say something like
 # unable to find main entry point) then run `jlpm run clean:lib` first to get rid of the old buildcache 

{mito_ai-0.1.55 → mito_ai-0.1.56}/README.md

@@ -77,6 +77,10 @@ jupyter labextension develop . --overwrite
 # Start the jupyter server extension for development
 jupyter server extension enable --py mito_ai
 
+# (Optional) Setup fixed Jupyter token for Cursor agent testing
+# This creates a development-only config that allows Cursor to automatically test changes
+python dev/setup_jupyter_dev_token.py
+
 # Watch the source directory in one terminal, automatically rebuilding when needed
 # In case of Error: If this command fails because the lib directory was not created (the error will say something like
 # unable to find main entry point) then run `jlpm run clean:lib` first to get rid of the old buildcache 

{mito_ai-0.1.55 → mito_ai-0.1.56}/dev/README.md

@@ -1,5 +1,13 @@
 # Mito AI Dev Scripts
 
+### View Chat History
+
+A Streamlit app that helps you explore the chat history of a Mito AI chat file.
+
+```bash
+streamlit run dev/chat_history_viewer.py
+```
+
 ### Generate AI Chats
 
 A simple script that creates a bunch of fake Mito AI chat files so you can quickly generate a fully populated chat history.

mito_ai-0.1.56/dev/chat_history_viewer.py

@@ -0,0 +1,346 @@
+# Copyright (c) Saga Inc.
+# Distributed under the terms of the GNU Affero General Public License v3.0 License.
+
+"""
+Streamlit app to view and explore chat histories from .mito/ai-chats folder.
+"""
+
+import streamlit as st
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+from datetime import datetime
+from typing import Dict, List, Any
+
+# Try to import tiktoken for accurate token counting
+# If not available, attempt to install it automatically
+try:
+    import tiktoken
+    TIKTOKEN_AVAILABLE = True
+except ImportError:
+    TIKTOKEN_AVAILABLE = False
+    # Attempt to install tiktoken automatically
+    try:
+        print("Installing tiktoken...")
+        subprocess.check_call([sys.executable, "-m", "pip", "install", "tiktoken>=0.5.0", "--quiet"])
+        print("tiktoken installed successfully")
+        import tiktoken
+        TIKTOKEN_AVAILABLE = True
+    except (subprocess.CalledProcessError, ImportError):
+        # Installation failed or import still fails - will use fallback estimation
+        TIKTOKEN_AVAILABLE = False
+
+# Get the chat history directory
+HOME_FOLDER = os.path.expanduser("~")
+MITO_FOLDER = os.path.join(HOME_FOLDER, ".mito")
+CHATS_DIR = os.path.join(MITO_FOLDER, "ai-chats")
+
+
+def get_chat_files() -> List[str]:
+    """Get all JSON files from the ai-chats directory."""
+    if not os.path.exists(CHATS_DIR):
+        return []
+    
+    try:
+        files = [f for f in os.listdir(CHATS_DIR) if f.endswith('.json')]
+        return sorted(files, reverse=True)  # Most recent first
+    except Exception:
+        return []
+
+
+def get_most_recent_chat_file() -> str:
+    """Get the filename of the most recently edited chat file based on last_interaction_ts."""
+    chat_files = get_chat_files()
+    
+    if not chat_files:
+        return None
+    
+    most_recent_file = None
+    most_recent_ts = 0
+    
+    for filename in chat_files:
+        try:
+            chat_data = load_chat_file(filename)
+            last_ts = chat_data.get("last_interaction_ts", 0)
+            
+            # Use file modification time as fallback if last_interaction_ts is missing
+            if last_ts == 0:
+                filepath = os.path.join(CHATS_DIR, filename)
+                last_ts = os.path.getmtime(filepath)
+            
+            if last_ts > most_recent_ts:
+                most_recent_ts = last_ts
+                most_recent_file = filename
+        except Exception:
+            # If we can't load the file, skip it
+            continue
+    
+    return most_recent_file
+
+
+def load_chat_file(filename: str) -> Dict[str, Any]:
+    """Load a chat history JSON file."""
+    filepath = os.path.join(CHATS_DIR, filename)
+    with open(filepath, 'r', encoding='utf-8') as f:
+        return json.load(f)
+
+
+def format_timestamp(ts: float) -> str:
+    """Format a timestamp to a readable date string."""
+    try:
+        dt = datetime.fromtimestamp(ts)
+        return dt.strftime("%Y-%m-%d %H:%M:%S")
+    except:
+        return str(ts)
+
+
+def format_message_content(content: Any) -> str:
+    """Format message content for display."""
+    # Handle different content types
+    if isinstance(content, (dict, list)):
+        return json.dumps(content, indent=2)
+    elif isinstance(content, str):
+        # Try to parse as JSON if it looks like JSON
+        if content.strip().startswith('{') or content.strip().startswith('['):
+            try:
+                parsed = json.loads(content)
+                return json.dumps(parsed, indent=2)
+            except:
+                pass
+        return content
+    else:
+        return str(content)
+
+
+def display_message_content(content: Any):
+    """Display message content in the appropriate format."""
+    # Handle different content types
+    if isinstance(content, (dict, list)):
+        st.json(content)
+    elif isinstance(content, str):
+        # Try to display as JSON if it looks like JSON
+        if content.strip().startswith('{') or content.strip().startswith('['):
+            try:
+                parsed = json.loads(content)
+                st.json(parsed)
+            except:
+                st.code(content, language="text")
+        else:
+            st.markdown(content)
+    else:
+        st.code(str(content), language="text")
+
+
+def is_json_object(content: Any) -> bool:
+    """Check if content is or can be parsed as a JSON object (dict or list)."""
+    if isinstance(content, (dict, list)):
+        return True
+    elif isinstance(content, str):
+        stripped = content.strip()
+        if stripped.startswith('{') or stripped.startswith('['):
+            try:
+                parsed = json.loads(content)
+                return isinstance(parsed, (dict, list))
+            except:
+                return False
+    return False
+
+
+def display_user_message_content(content: Any):
+    """Display user message content: JSON objects as JSON, everything else as code."""
+    if is_json_object(content):
+        # If it's a JSON object, display as JSON
+        if isinstance(content, (dict, list)):
+            st.json(content)
+        else:
+            # Parse and display as JSON
+            try:
+                parsed = json.loads(content)
+                st.json(parsed)
+            except:
+                st.code(content, language="text")
+    else:
+        # If it's not JSON, display as code
+        if isinstance(content, str):
+            st.code(content, language="text")
+        else:
+            st.code(str(content), language="text")
+
+
+def count_tokens_in_content(content: Any) -> int:
+    """Count tokens in message content."""
+    if TIKTOKEN_AVAILABLE:
+        try:
+            encoding = tiktoken.get_encoding("cl100k_base")  # Used by GPT-4 and GPT-3.5-turbo
+            # Convert content to string for token counting
+            if isinstance(content, (dict, list)):
+                content_str = json.dumps(content)
+            elif isinstance(content, str):
+                content_str = content
+            else:
+                content_str = str(content)
+            return len(encoding.encode(content_str))
+        except Exception:
+            # Fallback to character-based estimation
+            pass
+    
+    # Fallback: rough estimation (1 token ≈ 3 characters)
+    if isinstance(content, (dict, list)):
+        content_str = json.dumps(content)
+    elif isinstance(content, str):
+        content_str = content
+    else:
+        content_str = str(content)
+    return int(len(content_str) / 3)
+
+
+def count_tokens_in_message(message: Dict[str, Any]) -> int:
+    """Count tokens in a single message."""
+    content = message.get("content", "")
+    return count_tokens_in_content(content)
+
+
+def count_total_tokens_in_history(messages: List[Dict[str, Any]]) -> int:
+    """Count total tokens in a message history."""
+    total = 0
+    for message in messages:
+        total += count_tokens_in_message(message)
+    return total
+
+
+def main():
+    st.set_page_config(
+        page_title="Chat History Viewer",
+        page_icon="💬",
+        layout="wide"
+    )
+    
+    st.title("💬 Chat History Viewer")
+    
+    # Get list of chat files
+    chat_files = get_chat_files()
+    most_recent = get_most_recent_chat_file()
+    
+    if not chat_files:
+        st.warning(f"No chat history files found in `{CHATS_DIR}`")
+        st.info("Make sure the directory exists and contains JSON files.")
+        return
+    
+    # Get the most recently edited file (only on first load or if not set)
+    if 'file_selector' not in st.session_state:
+        st.session_state.file_selector = most_recent if most_recent else chat_files[0]
+    
+    # Find the index of the selected file in the chat_files list
+    default_index = 0
+    if st.session_state.file_selector and st.session_state.file_selector in chat_files:
+        default_index = chat_files.index(st.session_state.file_selector)
+    
+    # Sidebar for file selection
+    with st.sidebar:
+        selected_file = st.selectbox(
+            "Select a chat file:",
+            options=chat_files,
+            index=default_index,
+            format_func=lambda x: x,
+            key="file_selector"
+        )
+        
+        if selected_file == most_recent:
+            st.success("✅ This is the most recently modified chat")
+    
+    if not selected_file:
+        st.info("Please select a chat file from the sidebar.")
+        return
+    
+    # Load the selected chat file
+    try:
+        chat_data = load_chat_file(selected_file)
+    except Exception as e:
+        st.error(f"Error loading file: {e}")
+        return
+    
+    # Display metadata
+    thread_id = chat_data.get("thread_id", "N/A")
+    creation_ts = chat_data.get("creation_ts", 0)
+    last_ts = chat_data.get("last_interaction_ts", 0)
+    name = chat_data.get("name", "N/A")
+    version = chat_data.get("chat_history_version", "N/A")
+    
+    metadata_text = f"""
+**Thread ID:** {thread_id} 
+
+**Created:** {format_timestamp(creation_ts)}
+
+**Last Interaction:** {format_timestamp(last_ts)}
+
+**Name:** {name}
+
+**Version:** {version}
+"""
+    st.info(metadata_text)
+    
+    st.divider()
+    
+    # Display AI Optimized History
+    st.header("🤖 AI Optimized History")
+    
+    ai_history = chat_data.get("ai_optimized_history", [])
+    
+    if not ai_history:
+        st.info("No AI optimized history found in this chat.")
+    else:
+        # Calculate total tokens for the AI Optimized History
+        total_tokens = count_total_tokens_in_history(ai_history)
+        token_info = f"**Total Tokens:** {total_tokens:,}"
+        if not TIKTOKEN_AVAILABLE:
+            token_info += " (estimated - install tiktoken for accurate counts)"
+        st.info(token_info)
+        
+        # Display all messages
+        for idx, message in enumerate(ai_history):
+            role = message.get("role", "unknown")
+            content = message.get("content", "")
+            message_tokens = count_tokens_in_message(message)
+            
+            # Color code by role
+            if role == "system":
+                st.markdown("---")
+                with st.expander(f"🔧 **System Message** (Message {idx + 1}/{len(ai_history)}) - {message_tokens:,} tokens", expanded=False):
+                    st.code(format_message_content(content), language="text")
+            elif role == "user":
+                st.markdown("---")
+                with st.expander(f"👤 **User Message** (Message {idx + 1}/{len(ai_history)}) - {message_tokens:,} tokens", expanded=False):
+                    st.markdown("**Content:**")
+                    display_user_message_content(content)
+            elif role == "assistant":
+                st.markdown("---")
+                with st.expander(f"🤖 **Assistant Message** (Message {idx + 1}/{len(ai_history)}) - {message_tokens:,} tokens", expanded=False):
+                    st.markdown("**Content:**")
+                    display_message_content(content)
+    
+    st.divider()
+    
+    # Raw JSON viewer
+    st.header("🔍 Raw JSON Data")
+    
+    with st.expander("View Full JSON", expanded=False):
+        st.json(chat_data)
+    
+    # Display history comparison
+    if "display_history" in chat_data:
+        st.divider()
+        st.header("📺 Display History")
+        st.caption("This is the display-optimized version of the history")
+        
+        display_history = chat_data.get("display_history", [])
+        st.info(f"Display history contains {len(display_history)} messages")
+        
+        with st.expander("View Display History", expanded=False):
+            st.json(display_history)
+
+
+if __name__ == "__main__":
+    main()
+

mito_ai-0.1.56/dev/setup_jupyter_dev_token.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+# Copyright (c) Saga Inc.
+# Distributed under the terms of the GNU Affero General Public License v3.0 License.
+
+"""
+Setup script to configure Jupyter Lab with a fixed token for Cursor agent testing.
+
+This script:
+1. Creates a development-only Jupyter config file with a fixed token
+2. Writes the token to a file for Cursor agent to read
+
+The config file uses conditional logic to only apply the token when running in
+development mode, so it's safe to run this script anytime.
+
+Note: The .cursorrules file is maintained in git and contains instructions for
+the Cursor agent on how to use the token.
+"""
+
+import sys
+from pathlib import Path
+
+# Fixed token for development testing
+DEV_TOKEN = "dev-token-for-cursor-testing-12345"
+
+# Paths relative to repo root
+REPO_ROOT = Path(__file__).parent.parent
+JUPYTER_CONFIG_DIR = REPO_ROOT / "jupyter-config" / "jupyter_server_config.d"
+DEV_CONFIG_FILE = JUPYTER_CONFIG_DIR / "mito_ai_dev.py"
+CURSOR_TOKEN_FILE = REPO_ROOT / ".cursor-jupyter-token"
+
+
+def main():
+    """Main setup function."""
+    print("Setting up Jupyter development token for Cursor agent...")
+    print()
+    
+    # Ensure jupyter config directory exists
+    JUPYTER_CONFIG_DIR.mkdir(parents=True, exist_ok=True)
+    print(f"✓ Config directory ready: {JUPYTER_CONFIG_DIR}")
+    
+    # Create dev config file with conditional logic
+    # This Python config file will only apply the token when in dev mode
+    dev_config_content = f'''# Copyright (c) Saga Inc.
+# Distributed under the terms of the GNU Affero General Public License v3.0 License.
+
+"""
+Development-only Jupyter Server configuration.
+
+This config file sets a fixed token for Cursor agent testing, but only when
+running in development mode. In production, this config file has no effect.
+
+This file is gitignored and should not be committed to the repository.
+"""
+
+try:
+    from mito_ai.utils.telemetry_utils import is_dev_mode
+    
+    # Only set the token if we're in development mode
+    if is_dev_mode():
+        c.ServerApp.token = "{DEV_TOKEN}"
+except ImportError:
+    # If mito_ai is not available, don't set anything
+    # This ensures the config file doesn't break if mito_ai isn't installed
+    pass
+except Exception:
+    # Silently fail if there's any error checking dev mode
+    # This ensures the config file is safe to use in any environment
+    pass
+'''
+    
+    with open(DEV_CONFIG_FILE, 'w') as f:
+        f.write(dev_config_content)
+    
+    print(f"✓ Created dev config: {DEV_CONFIG_FILE}")
+    print("  (Token will only be applied when running in development mode)")
+    
+    # Write token to file for Cursor
+    with open(CURSOR_TOKEN_FILE, 'w') as f:
+        f.write(DEV_TOKEN)
+    
+    print(f"✓ Created token file: {CURSOR_TOKEN_FILE}")
+    print()
+    print("Setup complete!")
+    print()
+    print("Next steps:")
+    print("  1. Restart Jupyter Lab to apply the new token configuration")
+    print("  2. In dev mode, access Jupyter at: http://localhost:8888/lab?token=" + DEV_TOKEN)
+    print("  3. Cursor agent can now automatically test changes using this token")
+    print()
+    print("Note: The dev config file is gitignored and will not be committed.")
+    print("      The token will only be active when running in development mode.")
+
+
+if __name__ == "__main__":
+    main()

{mito_ai-0.1.55 → mito_ai-0.1.56}/example-notebooks/geospatial/bike-violations.ipynb

@@ -1,5 +1,15 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "4ba557a2-1ef1-414d-aaf4-e0c4f0c802be",
+   "metadata": {
+    "include-cell-in-app": true
+   },
+   "source": [
+    "Made by Aaron Diamond-Reivich"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "a8acbb26-bf39-4cd3-8735-b7f24cd3ca04",
@@ -262,6 +272,16 @@
     "This mapping allows us to analyze violations by their actual meaning rather than just cryptic codes. "
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "15e12aaf-f5c7-44ec-b7fa-44d86aa4de24",
+   "metadata": {
+    "include-cell-in-app": true
+   },
+   "source": [
+    "Disclaimer: These codes were sourced from the internet and might not be exhaustive"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 2,
@@ -96952,6 +96972,36 @@
    },
    "outputs": [],
    "source": []
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6f91c84a-b52c-4fd3-9b95-461c56cbcab2",
+   "metadata": {
+    "include-cell-in-app": true
+   },
+   "source": [
+    "Disclaimer: This project is meant as a tutorial of Mito and might be innacturate"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "32615869-5a5b-4ce6-9be0-a94dd61fd246",
+   "metadata": {
+    "include-cell-in-app": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Fin\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"Fin\")"
+   ]
   }
  ],
  "metadata": {
@@ -96970,8 +97020,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.1"
-  }
+   "version": "3.12.12"
+  },
+  "mito-notebook-id": "mito-notebook-56a545c0dbb8"
  },
  "nbformat": 4,
  "nbformat_minor": 5

{mito_ai-0.1.55 → mito_ai-0.1.56}/mito_ai/_version.py

@@ -1,4 +1,4 @@
 # This file is auto-generated by Hatchling. As such, do not:
 #   - modify
 #   - track in version control e.g. be sure to add to .gitignore
-__version__ = VERSION = '0.1.55'
+__version__ = VERSION = '0.1.56'

{mito_ai-0.1.55 → mito_ai-0.1.56}/mito_ai/anthropic_client.py

@@ -7,7 +7,7 @@ from typing import Dict, Any, Optional, Tuple, Union, Callable, List, cast
 
 from anthropic.types import Message, MessageParam, TextBlockParam
 from mito_ai.completions.models import ResponseFormatInfo, CompletionReply, CompletionStreamChunk, CompletionItem, MessageType
-from mito_ai.constants import MESSAGE_HISTORY_TRIM_THRESHOLD
+from mito_ai.completions.prompt_builders.prompt_section_registry import get_max_trim_after_messages
 from openai.types.chat import ChatCompletionMessageParam
 from mito_ai.utils.anthropic_utils import get_anthropic_completion_from_mito_server, select_correct_model, stream_anthropic_completion_from_mito_server, get_anthropic_completion_function_params
 
@@ -15,6 +15,10 @@ from mito_ai.utils.anthropic_utils import get_anthropic_completion_from_mito_ser
 # We set it to a high number so that we can edit large code cells
 MAX_TOKENS = 64_000
 
+# Calculate the max trim threshold once at module load time.
+# This is used for cache boundary calculation - messages older than this threshold are stable.
+MAX_TRIM_THRESHOLD = get_max_trim_after_messages()
+
 def extract_and_parse_anthropic_json_response(response: Message) -> Union[object, Any]:
     """
     Extracts and parses the JSON response from the Claude API.
@@ -170,9 +174,6 @@ def get_anthropic_system_prompt_and_messages_with_caching(messages: List[ChatCom
     1. System prompt (static) → Always cached
     2. Stable conversation history → Cache at keep_recent boundary
     3. Recent messages → Never cached (always fresh)
-    
-    The keep_recent parameter determines which messages are stable and won't be trimmed.
-    We cache at the keep_recent boundary because those messages are guaranteed to be stable.
     """
     
     # Get the base system prompt and messages
@@ -189,14 +190,14 @@ def get_anthropic_system_prompt_and_messages_with_caching(messages: List[ChatCom
         }]
     
     # 2. Cache conversation history at the boundary where the messages are stable.
-    # Messages are stable after they are more than MESSAGE_HISTORY_TRIM_THRESHOLD old.
+    # Messages are stable after they are older than the max trim_after_messages threshold.
     # At this point, the messages are not edited anymore, so they will not invalidate the cache.
     # If we included the messages before the boundary in the cache, then every time we send a new
     # message, we would invalidate the cache and we would never get a cache hit except for the system prompt.
     messages_with_cache = []
     
     if len(anthropic_messages) > 0:
-        cache_boundary = len(anthropic_messages) - MESSAGE_HISTORY_TRIM_THRESHOLD - 1
+        cache_boundary = len(anthropic_messages) - MAX_TRIM_THRESHOLD - 1
         
         # Add all messages, but only add cache_control to the message at the boundary
         for i, msg in enumerate(anthropic_messages):

{mito_ai-0.1.55 → mito_ai-0.1.56}/mito_ai/completions/models.py

@@ -17,7 +17,7 @@ ThreadID = NewType('ThreadID', str)
     
 class CellUpdate(BaseModel):
     type: Literal['modification', 'new']
-    index: Optional[int]
+    after_cell_id: Optional[str]
     id: Optional[str]
     code: str
     code_summary: str