signalpilot-ai-internal 0.5.1__py3-none-any.whl → 0.7.6__py3-none-any.whl

signalpilot_ai_internal/handlers.py

@@ -12,6 +12,7 @@ from .cache_service import get_cache_service
 from .cache_handlers import ChatHistoriesHandler, AppValuesHandler, CacheInfoHandler
 from .unified_database_schema_service import UnifiedDatabaseSchemaHandler, UnifiedDatabaseQueryHandler
 from .snowflake_schema_service import SnowflakeSchemaHandler, SnowflakeQueryHandler
+from .file_scanner_service import get_file_scanner_service
 from .schema_search_service import SchemaSearchHandler
 
 
@@ -258,6 +259,447 @@ class ReadAllFilesHandler(APIHandler):
         return '\n'.join(lines)
 
 
+class SelectFolderHandler(APIHandler):
+    """Handler to open a native folder picker and return the selected absolute path"""
+    
+    # Class-level flag to prevent multiple dialogs
+    _dialog_open = False
+
+    @tornado.web.authenticated
+    def get(self):
+        # Check if a dialog is already open
+        if SelectFolderHandler._dialog_open:
+            self.set_status(409)  # Conflict status
+            self.finish(json.dumps({
+                "error": "A folder selection dialog is already open"
+            }))
+            return
+            
+        try:
+            import tkinter as tk
+            from tkinter import filedialog
+            import threading
+            import time
+
+            # Set flag to prevent multiple dialogs
+            SelectFolderHandler._dialog_open = True
+            
+            # Create a fresh tkinter instance
+            root = tk.Tk()
+            
+            # Position the root window in the center of the screen BEFORE withdrawing
+            try:
+                # Get screen dimensions
+                screen_width = root.winfo_screenwidth()
+                screen_height = root.winfo_screenheight()
+                
+                # Calculate center position
+                x = (screen_width // 2) - 200  # Dialog is roughly 400px wide
+                y = (screen_height // 2) - 150  # Dialog is roughly 300px tall
+                
+                # Set window position and make it visible briefly for positioning
+                root.geometry(f"400x300+{x}+{y}")
+                root.update_idletasks()
+                
+                # Now withdraw the window
+                root.withdraw()
+                
+                # Enhanced topmost settings for better visibility
+                root.attributes('-topmost', True)
+                root.lift()
+                root.focus_force()
+                
+                # Final positioning update
+                root.update_idletasks()
+            except Exception:
+                # Fallback: just withdraw if positioning fails
+                root.withdraw()
+
+            folder = None
+            
+            try:
+                # Show the dialog with proper positioning
+                folder = filedialog.askdirectory(
+                    parent=root,
+                    title="Select Folder",
+                    initialdir=os.path.expanduser("~")  # Start in user's home directory
+                )
+            except Exception as e:
+                raise e
+            finally:
+                # Comprehensive cleanup
+                try:
+                    # Force close and destroy all tkinter components
+                    root.quit()
+                    root.destroy()
+                    
+                    # Additional cleanup for macOS - ensure complete destruction
+                    try:
+                        root.update_idletasks()
+                        root.update()
+                        # Force garbage collection of tkinter objects
+                        import gc
+                        gc.collect()
+                    except Exception:
+                        pass
+                        
+                except Exception:
+                    pass
+                finally:
+                    # Reset flag and add small delay to ensure cleanup
+                    SelectFolderHandler._dialog_open = False
+                    time.sleep(0.1)  # Small delay to ensure cleanup completes
+
+            # Normalize and return absolute path or null
+            if folder:
+                folder_path = os.path.abspath(folder)
+                self.finish(json.dumps({"path": folder_path}))
+            else:
+                self.finish(json.dumps({"path": None}))
+                
+        except Exception as e:
+            # Reset flag on error
+            SelectFolderHandler._dialog_open = False
+            self.set_status(400)
+            self.finish(json.dumps({
+                "error": str(e)
+            }))
+
+class FileScanHandler(APIHandler):
+    """Handler for scanning directories for files"""
+    
+    @tornado.web.authenticated
+    async def post(self):
+        try:
+            data = json.loads(self.request.body.decode('utf-8'))
+            paths = data.get('paths', [])
+            
+            if not paths:
+                self.set_status(400)
+                self.finish(json.dumps({
+                    "error": "No paths provided"
+                }))
+                return
+            
+            file_scanner = get_file_scanner_service()
+            # Pass the current working directory as the workspace root for relative path calculation
+            result = await file_scanner.scan_directories(paths, workspace_root=os.getcwd())
+            
+            # Update scanned directories tracking
+            scanned_dirs = file_scanner.get_scanned_directories()
+            current_dirs = scanned_dirs.get('directories', [])
+            
+            # Update directory metadata
+            for new_dir in result['scanned_directories']:
+                # Check if directory already exists
+                existing_dir = None
+                for existing in current_dirs:
+                    if existing['path'] == new_dir['path']:
+                        existing_dir = existing
+                        break
+                
+                if existing_dir:
+                    # Update existing directory
+                    existing_dir['file_count'] = new_dir['file_count']
+                    existing_dir['scanned_at'] = new_dir['scanned_at']
+                else:
+                    # Add new directory
+                    current_dirs.append(new_dir)
+            
+            # Save updated directories list
+            file_scanner.update_scanned_directories(current_dirs)
+            
+            self.finish(json.dumps(result))
+            
+        except Exception as e:
+            self.set_status(500)
+            self.finish(json.dumps({
+                "error": str(e)
+            }))
+
+
+class ScannedDirectoriesHandler(APIHandler):
+    """Handler for getting scanned directories list"""
+    
+    @tornado.web.authenticated
+    def get(self):
+        try:
+            file_scanner = get_file_scanner_service()
+            result = file_scanner.get_scanned_directories()
+            
+            self.finish(json.dumps(result))
+            
+        except Exception as e:
+            self.set_status(500)
+            self.finish(json.dumps({
+                "error": str(e)
+            }))
+
+
+class WorkDirHandler(APIHandler):
+    """Handler for returning current working directory"""
+
+    @tornado.web.authenticated
+    def get(self):
+        try:
+            self.finish(json.dumps({"workdir": os.getcwd()}))
+        except Exception as e:
+            self.set_status(500)
+            self.finish(json.dumps({
+                "error": str(e)
+            }))
+
+
+class TerminalExecuteHandler(APIHandler):
+    """Handler for executing terminal commands"""
+
+    @tornado.web.authenticated
+    def post(self):
+        try:
+            data = json.loads(self.request.body.decode('utf-8'))
+            command = data.get('command')
+
+            if not command:
+                self.set_status(400)
+                self.finish(json.dumps({
+                    "error": "No command provided"
+                }))
+                return
+
+            import subprocess
+            result = subprocess.run(
+                command,
+                shell=True,
+                cwd=os.getcwd(),
+                capture_output=True,
+                text=True,
+                timeout=300
+            )
+
+            def truncate_output(output: str, max_lines: int = 50) -> str:
+                if not output:
+                    return output
+                lines = output.splitlines()
+                if len(lines) <= max_lines * 2:
+                    return output
+                first_lines = lines[:max_lines]
+                last_lines = lines[-max_lines:]
+                truncated_count = len(lines) - (max_lines * 2)
+                return '\n'.join(first_lines + [f'\n... {truncated_count} lines truncated ...\n'] + last_lines)
+
+            self.finish(json.dumps({
+                "stdout": truncate_output(result.stdout),
+                "stderr": truncate_output(result.stderr),
+                "exit_code": result.returncode
+            }))
+
+        except subprocess.TimeoutExpired:
+            self.set_status(408)
+            self.finish(json.dumps({
+                "error": "Command timed out after 300 seconds"
+            }))
+        except Exception as e:
+            self.set_status(500)
+            self.finish(json.dumps({
+                "error": str(e)
+            }))
+
+
+class DeleteScannedDirectoryHandler(APIHandler):
+    """Handler for deleting a scanned directory"""
+    
+    @tornado.web.authenticated
+    def post(self):
+        try:
+            data = json.loads(self.request.body.decode('utf-8'))
+            directory_path = data.get('path')
+            
+            if not directory_path:
+                self.set_status(400)
+                self.finish(json.dumps({
+                    "error": "No directory path provided"
+                }))
+                return
+            
+            file_scanner = get_file_scanner_service()
+            
+            # Get current scanned directories
+            current_directories = file_scanner.get_scanned_directories()
+            directories = current_directories.get('directories', [])
+            
+            # Filter out the directory to be deleted
+            filtered_directories = [
+                dir_info for dir_info in directories 
+                if dir_info.get('path') != directory_path
+            ]
+            
+            # Check if directory was actually found and removed
+            if len(filtered_directories) == len(directories):
+                self.set_status(404)
+                self.finish(json.dumps({
+                    "error": f"Directory '{directory_path}' not found in scanned directories"
+                }))
+                return
+            
+            # Update the scanned directories list
+            success = file_scanner.update_scanned_directories(filtered_directories)
+            
+            if success:
+                self.finish(json.dumps({
+                    "success": True,
+                    "message": f"Directory '{directory_path}' removed from scanning"
+                }))
+            else:
+                self.set_status(500)
+                self.finish(json.dumps({
+                    "error": "Failed to update scanned directories"
+                }))
+            
+        except Exception as e:
+            self.set_status(500)
+            self.finish(json.dumps({
+                "error": str(e)
+            }))
+
+
+class NotebookCellsHandler(APIHandler):
+    """Handler for reading specific cell ranges from notebooks"""
+
+    @tornado.web.authenticated
+    def post(self):
+        try:
+            data = json.loads(self.request.body.decode('utf-8'))
+            file_path = data.get('file_path')
+            start_cell = data.get('start_cell', 0)
+            end_cell = data.get('end_cell', 5)
+
+            if not file_path:
+                self.set_status(400)
+                self.finish(json.dumps({
+                    "error": "No file_path provided"
+                }))
+                return
+
+            file_scanner = get_file_scanner_service()
+            result = file_scanner.extract_schema(file_path, start_cell=start_cell, end_cell=end_cell)
+
+            self.finish(json.dumps(result))
+
+        except Exception as e:
+            self.set_status(500)
+            self.finish(json.dumps({
+                "error": str(e)
+            }))
+
+
+class NotebookToHTMLHandler(APIHandler):
+    """Handler for converting notebooks to HTML using nbconvert"""
+
+    @tornado.web.authenticated
+    def post(self):
+        try:
+            # Parse request body
+            try:
+                data = json.loads(self.request.body.decode('utf-8'))
+            except json.JSONDecodeError:
+                self.set_status(400)
+                self.finish(json.dumps({
+                    "error": "Invalid JSON in request body"
+                }))
+                return
+
+            # Get notebook path
+            notebook_path = data.get('notebook_path')
+            if not notebook_path:
+                self.set_status(400)
+                self.finish(json.dumps({
+                    "error": "No notebook path provided"
+                }))
+                return
+
+            # Get export options with defaults
+            include_input = data.get('include_input', True)
+            include_output = data.get('include_output', True)
+            include_images = data.get('include_images', True)
+
+            # Convert to nbconvert options
+            exclude_input = not include_input
+            exclude_output = not include_output
+
+            try:
+                # Import nbconvert and nbformat
+                from nbconvert import HTMLExporter
+                from nbconvert.preprocessors import ExtractOutputPreprocessor
+                from nbformat import read
+
+                # Read the notebook file using nbformat
+                notebook_file_path = Path(notebook_path)
+                if not notebook_file_path.exists():
+                    self.set_status(404)
+                    self.finish(json.dumps({
+                        "error": f"Notebook file not found: {notebook_path}"
+                    }))
+                    return
+
+                # Load notebook using nbformat.read (more robust)
+                with open(notebook_file_path, 'r', encoding='utf-8') as f:
+                    notebook_content = read(f, as_version=4)
+
+                # Get the custom template path (relative to this file)
+                template_dir = Path(__file__).parent / 'html_export_template'
+                template_path = str(template_dir.resolve())
+
+                # Configure the HTML exporter with custom template
+                exporter = HTMLExporter(template_name=template_path)
+
+                # Set export options
+                exporter.exclude_input = exclude_input
+                exporter.exclude_output = exclude_output
+                exporter.exclude_input_prompt = True
+                exporter.exclude_output_prompt = True
+
+                # Configure image handling
+                if not include_images:
+                    # Add preprocessor to exclude images
+                    exporter.register_preprocessor(ExtractOutputPreprocessor(), enabled=True)
+
+                # Convert notebook to HTML - nbconvert handles everything
+                html_content, _ = exporter.from_notebook_node(notebook_content)
+
+                # Get absolute path for the notebook
+                workspace_notebook_path = str(notebook_file_path.absolute())
+
+                # Return the HTML content
+                self.finish(json.dumps({
+                    "success": True,
+                    "html_content": html_content,
+                    "notebook_path": notebook_path,
+                    "workspace_notebook_path": workspace_notebook_path,
+                    "export_options": {
+                        "include_input": include_input,
+                        "include_output": include_output,
+                        "include_images": include_images
+                    }
+                }))
+
+            except ImportError as e:
+                self.set_status(500)
+                self.finish(json.dumps({
+                    "error": f"Required packages not installed: {str(e)}. Please install them with: pip install nbconvert nbformat"
+                }))
+            except Exception as e:
+                self.set_status(500)
+                self.finish(json.dumps({
+                    "error": f"Failed to convert notebook to HTML: {str(e)}"
+                }))
+
+        except Exception as e:
+            self.set_status(500)
+            self.finish(json.dumps({
+                "error": str(e)
+            }))
+
+
 def setup_handlers(web_app):
     host_pattern = ".*$"
     base_url = web_app.settings["base_url"]
@@ -268,6 +710,16 @@ def setup_handlers(web_app):
     # Read all files endpoint
     read_all_files_route = url_path_join(base_url, "signalpilot-ai-internal", "read-all-files")
     
+    # File scanning endpoints
+    file_scan_route = url_path_join(base_url, "signalpilot-ai-internal", "files", "scan")
+    scanned_directories_route = url_path_join(base_url, "signalpilot-ai-internal", "files", "directories")
+    select_folder_route = url_path_join(base_url, "signalpilot-ai-internal", "files", "select-folder")
+    workdir_route = url_path_join(base_url, "signalpilot-ai-internal", "files", "workdir")
+    delete_scanned_directory_route = url_path_join(base_url, "signalpilot-ai-internal", "files", "directories", "delete")
+
+    # Notebook endpoints
+    notebook_cells_route = url_path_join(base_url, "signalpilot-ai-internal", "notebook", "cells")
+
     # Cache service endpoints
     chat_histories_route = url_path_join(base_url, "signalpilot-ai-internal", "cache", "chat-histories")
     chat_history_route = url_path_join(base_url, "signalpilot-ai-internal", "cache", "chat-histories", "([^/]+)")
@@ -290,13 +742,32 @@ def setup_handlers(web_app):
     snowflake_schema_route = url_path_join(base_url, "signalpilot-ai-internal", "snowflake", "schema")
     snowflake_query_route = url_path_join(base_url, "signalpilot-ai-internal", "snowflake", "query")
     
+    # Notebook HTML export endpoint
+    notebook_html_route = url_path_join(base_url, "signalpilot-ai-internal", "notebook", "to-html")
+
+    # Terminal endpoint
+    terminal_execute_route = url_path_join(base_url, "signalpilot-ai-internal", "terminal", "execute")
+
     handlers = [
         # Original endpoint
         (hello_route, HelloWorldHandler),
-        
+
         # Read all files endpoint
         (read_all_files_route, ReadAllFilesHandler),
-        
+
+        # File scanning endpoints
+        (file_scan_route, FileScanHandler),
+        (scanned_directories_route, ScannedDirectoriesHandler),
+        (select_folder_route, SelectFolderHandler),
+        (workdir_route, WorkDirHandler),
+        (delete_scanned_directory_route, DeleteScannedDirectoryHandler),
+
+        # Terminal endpoint
+        (terminal_execute_route, TerminalExecuteHandler),
+
+        # Notebook endpoints
+        (notebook_cells_route, NotebookCellsHandler),
+
         # Chat histories endpoints
         (chat_histories_route, ChatHistoriesHandler),
         (chat_history_route, ChatHistoriesHandler),
@@ -320,6 +791,9 @@ def setup_handlers(web_app):
         # Snowflake service endpoints
         (snowflake_schema_route, SnowflakeSchemaHandler),
         (snowflake_query_route, SnowflakeQueryHandler),
+        
+        # Notebook HTML export endpoint
+        (notebook_html_route, NotebookToHTMLHandler),
     ]
     
     web_app.add_handlers(host_pattern, handlers)
@@ -347,3 +821,5 @@ def setup_handlers(web_app):
     print(f"  - MySQL Query: {mysql_query_route}")
     print(f"  - Snowflake Schema: {snowflake_schema_route}")
     print(f"  - Snowflake Query: {snowflake_query_route}")
+    print(f"  - Notebook Cells: {notebook_cells_route}")
+    print(f"  - Notebook to HTML: {notebook_html_route}")

signalpilot_ai_internal/html_export_template/README.md

@@ -0,0 +1,23 @@
+# HTML Export Template with Code Toggle
+
+This is a custom Jupyter NBConvert HTML template that adds a toggle to hide/show each code cell.
+
+## Usage
+
+The handler uses this template automatically. You can modify the notebook cells to control their visibility:
+
+- By default, all code cells are hidden with a "Show Code" button
+- To show a code cell by default, tag the cell with 'code_shown' in its metadata
+
+## Template Files
+
+- `index.html.j2` - The main template that extends lab/index.html.j2
+- `conf.json` - Configuration for the template
+- `README.md` - This file
+
+## Features
+
+- Clean presentation with toggleable code cells
+- Uses JQuery for smooth transitions
+- Removes input/output prompts for cleaner appearance
+- Responsive and modern styling

signalpilot_ai_internal/html_export_template/conf.json

@@ -0,0 +1,12 @@
+{
+  "base_template": "lab",
+  "mimetypes": {
+    "text/html": true
+  },
+  "preprocessors": {
+    "100-pygments": {
+      "type": "nbconvert.preprocessors.CSSHTMLHeaderPreprocessor",
+      "enabled": true
+    }
+  }
+}

signalpilot_ai_internal/html_export_template/index.html.j2

@@ -0,0 +1,140 @@
+{% extends 'lab/index.html.j2' %}
+
+{% block html_head %}
+{{ super() }}
+<script src="https://code.jquery.com/jquery-3.4.1.min.js"
+        integrity="sha256-CSXorXvZcTkaix6Yvo6HppcZGetbYMGWSFlBw8HfCJo="
+        crossorigin="anonymous"></script>
+
+<script>
+$(document).ready(function() {
+    // Add Show All / Hide All buttons above first code cell (if not already added)
+    if ($('.html-toggle-all-buttons').length === 0) {
+        // Only add buttons if there is at least one code_shower present
+        if ($('.code_shower').length === 0) {
+            return;
+        }
+        var $btnContainer = $('<div class="html-toggle-all-buttons"></div>');
+        var $showAllBtn = $('<button type="button" class="html-toggle-btn">Show All Code</button>');
+        var $hideAllBtn = $('<button type="button" class="html-toggle-btn">Hide All Code</button>');
+        $btnContainer.append($showAllBtn).append($hideAllBtn);
+
+        // Insert at the top of the <body>
+        $('body').prepend($btnContainer);
+
+        // "Show All" callback
+        $showAllBtn.on('click', function() {
+            $('.code_shower').each(function() {
+                var header = $(this);
+                var codecell = header.next();
+                // Only toggle if it's hidden
+                if (codecell.is(':hidden')) {
+                    // Update styles and text as necessary
+                    var wrapper = codecell.parent();
+                    wrapper.css('display', 'flex');
+                    wrapper.css('flex-direction', 'column');
+                    header.css('font-family', 'var(--jp-content-font-family)');
+                    header.text("Hide Code Cell");
+                    header.css("border-radius", "2px 2px 0px 0px");
+                    codecell.slideDown(0);
+                }
+            });
+        });
+
+        // "Hide All" callback
+        $hideAllBtn.on('click', function() {
+            $('.code_shower').each(function() {
+                var header = $(this);
+                var codecell = header.next();
+                // Only toggle if it's visible
+                if (codecell.is(':visible')) {
+                    var wrapper = codecell.parent();
+                    wrapper.css('display', 'flex');
+                    wrapper.css('flex-direction', 'column');
+                    header.css('font-family', 'var(--jp-content-font-family)');
+                    header.text("Show Code Cell");
+                    header.css("border-radius", "2px 2px 2px 2px");
+                    codecell.slideUp(0);
+                }
+            });
+        });
+    }
+
+    $('.code_shower').on('click', function() {
+        var header = $(this);
+        var codecell = $(this).next();
+
+        // Ensure the codecell's parent is a flex column wrapper
+        var wrapper = codecell.parent();
+        wrapper.css('display', 'flex');
+        wrapper.css('flex-direction', 'column');
+
+        // Ensure header uses the content font family
+        header.css('font-family', 'var(--jp-content-font-family)');
+
+        codecell.slideToggle(0, function() {
+            if (codecell.is(':hidden')) {
+                header.text("Show Code Cell");
+                header.css("border-radius", "2px 2px 2px 2px");
+            } else {
+                header.text("Hide Code Cell");
+                header.css("border-radius", "2px 2px 0px 0px");
+            }
+        });
+    });
+    $('.hidden_default').next().hide();
+});
+
+</script>
+
+<style>
+div.input {
+    flex-direction: column !important;
+}
+
+div.input_area {
+    border-radius: 0px 0px 2px 2px;
+}
+
+div.code_shower {
+    background: lightgray;
+    padding: 5px 10px;
+    cursor: pointer;
+    border-radius: 2px 2px 0px 0px;
+    font-family: var(--jp-content-font-family);
+}
+
+/* Container and button styles for Show/Hide All controls */
+.html-toggle-all-buttons {
+    display: flex;
+    justify-content: end;
+    gap: 8px;
+}
+
+.html-toggle-all-buttons .html-toggle-btn {
+    background: lightgray;
+    padding: 5px 10px;
+    cursor: pointer;
+    border-radius: 2px;
+    border: none;
+    font-family: var(--jp-content-font-family);
+}
+
+</style>
+{% endblock html_head %}
+
+{% block input %}
+{% if 'code_shown' in cell['metadata'].get('tags', []) %}
+<div class="code_shower">Hide Code</div>
+{% else %}
+<div class="code_shower hidden_default">Show Code</div>
+{% endif %}
+
+{{ super() }}
+{% endblock input %}
+
+{% block output_prompt %}
+{% endblock output_prompt %}
+
+{% block in_prompt %}
+{% endblock in_prompt %}