mito-ai 0.1.37__py3-none-any.whl → 0.1.39__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mito-ai might be problematic. Click here for more details.
- mito_ai/__init__.py +17 -1
- mito_ai/_version.py +1 -1
- mito_ai/app_builder/handlers.py +43 -38
- mito_ai/app_builder/models.py +1 -1
- mito_ai/completions/handlers.py +1 -1
- mito_ai/completions/prompt_builders/agent_system_message.py +18 -45
- mito_ai/completions/prompt_builders/chat_name_prompt.py +6 -6
- mito_ai/log/handlers.py +10 -3
- mito_ai/log/urls.py +3 -3
- mito_ai/openai_client.py +1 -1
- mito_ai/streamlit_conversion/agent_utils.py +116 -0
- mito_ai/streamlit_conversion/prompts/prompt_constants.py +59 -0
- mito_ai/streamlit_conversion/prompts/prompt_utils.py +10 -0
- mito_ai/streamlit_conversion/prompts/streamlit_app_creation_prompt.py +45 -0
- mito_ai/streamlit_conversion/prompts/streamlit_error_correction_prompt.py +28 -0
- mito_ai/streamlit_conversion/prompts/streamlit_finish_todo_prompt.py +44 -0
- mito_ai/streamlit_conversion/streamlit_agent_handler.py +90 -44
- mito_ai/streamlit_conversion/streamlit_system_prompt.py +30 -17
- mito_ai/streamlit_conversion/streamlit_utils.py +48 -8
- mito_ai/streamlit_conversion/validate_streamlit_app.py +116 -0
- mito_ai/streamlit_preview/__init__.py +7 -0
- mito_ai/streamlit_preview/handlers.py +164 -0
- mito_ai/streamlit_preview/manager.py +159 -0
- mito_ai/streamlit_preview/urls.py +22 -0
- mito_ai/tests/streamlit_conversion/test_streamlit_agent_handler.py +166 -78
- mito_ai/tests/streamlit_conversion/test_streamlit_utils.py +4 -5
- mito_ai/tests/streamlit_conversion/test_validate_streamlit_app.py +119 -0
- mito_ai/tests/streamlit_preview/test_streamlit_preview_manager.py +302 -0
- mito_ai/tests/utils/test_anthropic_utils.py +2 -2
- mito_ai/utils/anthropic_utils.py +4 -4
- mito_ai/utils/open_ai_utils.py +0 -4
- mito_ai/utils/telemetry_utils.py +28 -1
- {mito_ai-0.1.37.data → mito_ai-0.1.39.data}/data/share/jupyter/labextensions/mito_ai/build_log.json +1 -1
- {mito_ai-0.1.37.data → mito_ai-0.1.39.data}/data/share/jupyter/labextensions/mito_ai/package.json +2 -2
- {mito_ai-0.1.37.data → mito_ai-0.1.39.data}/data/share/jupyter/labextensions/mito_ai/schemas/mito_ai/package.json.orig +1 -1
- {mito_ai-0.1.37.data → mito_ai-0.1.39.data}/data/share/jupyter/labextensions/mito_ai/schemas/mito_ai/toolbar-buttons.json +6 -1
- mito_ai-0.1.37.data/data/share/jupyter/labextensions/mito_ai/static/lib_index_js.831f63b48760c7119b9b.js → mito_ai-0.1.39.data/data/share/jupyter/labextensions/mito_ai/static/lib_index_js.16b532b655cd2906e04a.js +799 -116
- mito_ai-0.1.39.data/data/share/jupyter/labextensions/mito_ai/static/lib_index_js.16b532b655cd2906e04a.js.map +1 -0
- mito_ai-0.1.37.data/data/share/jupyter/labextensions/mito_ai/static/remoteEntry.93ecc9bc0edba61535cc.js → mito_ai-0.1.39.data/data/share/jupyter/labextensions/mito_ai/static/remoteEntry.606207904e6aaa42b1bf.js +5 -5
- mito_ai-0.1.37.data/data/share/jupyter/labextensions/mito_ai/static/remoteEntry.93ecc9bc0edba61535cc.js.map → mito_ai-0.1.39.data/data/share/jupyter/labextensions/mito_ai/static/remoteEntry.606207904e6aaa42b1bf.js.map +1 -1
- {mito_ai-0.1.37.dist-info → mito_ai-0.1.39.dist-info}/METADATA +4 -1
- {mito_ai-0.1.37.dist-info → mito_ai-0.1.39.dist-info}/RECORD +53 -42
- mito_ai/streamlit_conversion/validate_and_run_streamlit_code.py +0 -207
- mito_ai/tests/streamlit_conversion/test_validate_and_run_streamlit_code.py +0 -418
- mito_ai-0.1.37.data/data/share/jupyter/labextensions/mito_ai/static/lib_index_js.831f63b48760c7119b9b.js.map +0 -1
- {mito_ai-0.1.37.data → mito_ai-0.1.39.data}/data/etc/jupyter/jupyter_server_config.d/mito_ai.json +0 -0
- {mito_ai-0.1.37.data → mito_ai-0.1.39.data}/data/share/jupyter/labextensions/mito_ai/static/style.js +0 -0
- {mito_ai-0.1.37.data → mito_ai-0.1.39.data}/data/share/jupyter/labextensions/mito_ai/static/style_index_js.5876024bb17dbd6a3ee6.js +0 -0
- {mito_ai-0.1.37.data → mito_ai-0.1.39.data}/data/share/jupyter/labextensions/mito_ai/static/style_index_js.5876024bb17dbd6a3ee6.js.map +0 -0
- {mito_ai-0.1.37.data → mito_ai-0.1.39.data}/data/share/jupyter/labextensions/mito_ai/static/vendors-node_modules_semver_index_js.9795f79265ddb416864b.js +0 -0
- {mito_ai-0.1.37.data → mito_ai-0.1.39.data}/data/share/jupyter/labextensions/mito_ai/static/vendors-node_modules_semver_index_js.9795f79265ddb416864b.js.map +0 -0
- {mito_ai-0.1.37.data → mito_ai-0.1.39.data}/data/share/jupyter/labextensions/mito_ai/static/vendors-node_modules_vscode-diff_dist_index_js.ea55f1f9346638aafbcf.js +0 -0
- {mito_ai-0.1.37.data → mito_ai-0.1.39.data}/data/share/jupyter/labextensions/mito_ai/static/vendors-node_modules_vscode-diff_dist_index_js.ea55f1f9346638aafbcf.js.map +0 -0
- {mito_ai-0.1.37.dist-info → mito_ai-0.1.39.dist-info}/WHEEL +0 -0
- {mito_ai-0.1.37.dist-info → mito_ai-0.1.39.dist-info}/entry_points.txt +0 -0
- {mito_ai-0.1.37.dist-info → mito_ai-0.1.39.dist-info}/licenses/LICENSE +0 -0
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# Copyright (c) Saga Inc.
|
|
2
|
+
# Distributed under the terms of the GNU Affero General Public License v3.0 License.
|
|
3
|
+
|
|
4
|
+
from mito_ai.streamlit_conversion.prompts.prompt_constants import MITO_TODO_PLACEHOLDER
|
|
5
|
+
|
|
6
|
+
def get_streamlit_app_creation_prompt(notebook: dict) -> str:
|
|
7
|
+
"""
|
|
8
|
+
This prompt is used to create a streamlit app from a notebook.
|
|
9
|
+
"""
|
|
10
|
+
return f"""Convert the following Jupyter notebook into a Streamlit application.
|
|
11
|
+
|
|
12
|
+
GOAL: Create a complete, runnable Streamlit app that accurately represents the notebook. It must completely convert the notebook.
|
|
13
|
+
|
|
14
|
+
TODO PLACEHOLDER RULES:
|
|
15
|
+
If you decide to leave any TODOs, you must mark them with {MITO_TODO_PLACEHOLDER}. You should use {MITO_TODO_PLACEHOLDER} instead of comments like the following:
|
|
16
|
+
- # ... (include all mappings from the notebook)
|
|
17
|
+
- # ... (include all violation codes from the notebook)
|
|
18
|
+
- # Fill in the rest of the code here
|
|
19
|
+
- # TODO: Add more code here
|
|
20
|
+
- # TODO: Add the visualization code here
|
|
21
|
+
|
|
22
|
+
For each TODO, use this exact format:
|
|
23
|
+
{MITO_TODO_PLACEHOLDER}: <specific description of what needs to be added>
|
|
24
|
+
|
|
25
|
+
IMPORTANT:
|
|
26
|
+
- The app must still be RUNNABLE even with placeholders
|
|
27
|
+
- Include enough sample data to show the structure
|
|
28
|
+
- Do NOT use placeholders for small/medium content - include it directly
|
|
29
|
+
- Do NOT use placeholders for file paths, imports, or core logic
|
|
30
|
+
- Only use placeholders when absolutely necessary. Add all of the content directly as much as possible.
|
|
31
|
+
|
|
32
|
+
<Example>
|
|
33
|
+
If the notebook has a list of dictionaries with 50 entries, you would write:
|
|
34
|
+
|
|
35
|
+
data = [
|
|
36
|
+
{{'id': 1, 'name': 'Item A', 'category': 'Type 1', 'value': 100}},
|
|
37
|
+
{{'id': 2, 'name': 'Item B', 'category': 'Type 2', 'value': 200}},
|
|
38
|
+
{MITO_TODO_PLACEHOLDER}: Add remaining entries from the data list
|
|
39
|
+
]
|
|
40
|
+
</Example>
|
|
41
|
+
|
|
42
|
+
Notebook to convert:
|
|
43
|
+
|
|
44
|
+
{notebook}
|
|
45
|
+
"""
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Copyright (c) Saga Inc.
|
|
2
|
+
# Distributed under the terms of the GNU Affero General Public License v3.0 License.
|
|
3
|
+
|
|
4
|
+
from mito_ai.streamlit_conversion.prompts.prompt_constants import unified_diff_instrucrions
|
|
5
|
+
from mito_ai.streamlit_conversion.prompts.prompt_utils import add_line_numbers_to_code
|
|
6
|
+
|
|
7
|
+
def get_streamlit_error_correction_prompt(error: str, streamlit_app_code: str) -> str:
|
|
8
|
+
|
|
9
|
+
existing_streamlit_app_code_with_line_numbers = add_line_numbers_to_code(streamlit_app_code)
|
|
10
|
+
|
|
11
|
+
return f"""You've created a Streamlit app, but it has an error in it when you try to run it.
|
|
12
|
+
|
|
13
|
+
Your job is to fix the error now. Only fix the specific error that you are instructed to fix now. Do not fix other error that that you anticipate. You will be asked to fix other errors later.
|
|
14
|
+
|
|
15
|
+
{unified_diff_instrucrions}
|
|
16
|
+
|
|
17
|
+
===============================================
|
|
18
|
+
|
|
19
|
+
EXISTING STREAMLIT APP:
|
|
20
|
+
{existing_streamlit_app_code_with_line_numbers}
|
|
21
|
+
|
|
22
|
+
===============================================
|
|
23
|
+
|
|
24
|
+
Please create a unified diff that corrects this error. Please keep your fix concise:
|
|
25
|
+
{error}
|
|
26
|
+
|
|
27
|
+
"""
|
|
28
|
+
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Copyright (c) Saga Inc.
|
|
2
|
+
# Distributed under the terms of the GNU Affero General Public License v3.0 License.
|
|
3
|
+
|
|
4
|
+
from mito_ai.streamlit_conversion.prompts.prompt_constants import MITO_TODO_PLACEHOLDER, unified_diff_instrucrions
|
|
5
|
+
from mito_ai.streamlit_conversion.prompts.prompt_utils import add_line_numbers_to_code
|
|
6
|
+
|
|
7
|
+
def get_finish_todo_prompt(notebook: dict, existing_streamlit_app_code: str, todo_placeholder: str) -> str:
|
|
8
|
+
|
|
9
|
+
existing_streamlit_app_code_with_line_numbers = add_line_numbers_to_code(existing_streamlit_app_code)
|
|
10
|
+
|
|
11
|
+
return f"""You've already created the first draft of a Streamlit app representation of a Jupyter notebook, but you left yourself some TODOs marked as `{MITO_TODO_PLACEHOLDER}`.
|
|
12
|
+
|
|
13
|
+
**CRITICAL COMPLETION REQUIREMENT:**
|
|
14
|
+
You have ONE and ONLY ONE opportunity to complete this TODO. If you do not finish the entire task completely, the application will be broken and unusable. This is your final chance to get it right.
|
|
15
|
+
|
|
16
|
+
**COMPLETION RULES:**
|
|
17
|
+
1. **NEVER leave partial work** - If the TODO asks for a list with 100 items, provide ALL 100 items
|
|
18
|
+
2. **NEVER use placeholders** - This is your only opportunity to fulfill this TODO, so do not leave yourself another TODO.
|
|
19
|
+
3. **NEVER assume "good enough"** - Complete the task to 100% satisfaction
|
|
20
|
+
4. **If the task seems large, that's exactly why it needs to be done now** - This is your only chance
|
|
21
|
+
|
|
22
|
+
**HOW TO DETERMINE IF TASK IS COMPLETE:**
|
|
23
|
+
- If building a list/dictionary: Include ALL items that should be in the final data structure
|
|
24
|
+
- If creating functions: Implement ALL required functionality
|
|
25
|
+
- If converting a visualization: Copy over ALL of the visualization code from the notebook, including all styling and formatting.
|
|
26
|
+
|
|
27
|
+
{unified_diff_instrucrions}
|
|
28
|
+
|
|
29
|
+
===============================================
|
|
30
|
+
|
|
31
|
+
Input Notebook that you are converting into the Streamlit app:
|
|
32
|
+
{notebook}
|
|
33
|
+
|
|
34
|
+
===============================================
|
|
35
|
+
|
|
36
|
+
EXISTING STREAMLIT APP:
|
|
37
|
+
{existing_streamlit_app_code_with_line_numbers}
|
|
38
|
+
|
|
39
|
+
===============================================
|
|
40
|
+
|
|
41
|
+
Please make the changes for this TODO. Only focus on this one TODO right now. You will be asked to fix others later:
|
|
42
|
+
{todo_placeholder}
|
|
43
|
+
|
|
44
|
+
"""
|
|
@@ -2,31 +2,26 @@
|
|
|
2
2
|
# Distributed under the terms of the GNU Affero General Public License v3.0 License.
|
|
3
3
|
|
|
4
4
|
import logging
|
|
5
|
+
import os
|
|
5
6
|
from anthropic.types import MessageParam
|
|
6
|
-
from typing import List, Tuple, cast
|
|
7
|
+
from typing import List, Optional, Tuple, cast, Union
|
|
7
8
|
|
|
8
9
|
from mito_ai.logger import get_logger
|
|
10
|
+
from mito_ai.streamlit_conversion.agent_utils import apply_patch_to_text, extract_todo_placeholders, fix_diff_headers
|
|
11
|
+
from mito_ai.streamlit_conversion.prompts.streamlit_app_creation_prompt import get_streamlit_app_creation_prompt
|
|
12
|
+
from mito_ai.streamlit_conversion.prompts.streamlit_error_correction_prompt import get_streamlit_error_correction_prompt
|
|
13
|
+
from mito_ai.streamlit_conversion.prompts.streamlit_finish_todo_prompt import get_finish_todo_prompt
|
|
9
14
|
from mito_ai.streamlit_conversion.streamlit_system_prompt import streamlit_system_prompt
|
|
10
|
-
from mito_ai.streamlit_conversion.
|
|
11
|
-
from mito_ai.streamlit_conversion.streamlit_utils import extract_code_blocks, create_app_file, parse_jupyter_notebook_to_extract_required_content
|
|
15
|
+
from mito_ai.streamlit_conversion.validate_streamlit_app import validate_app
|
|
16
|
+
from mito_ai.streamlit_conversion.streamlit_utils import extract_code_blocks, create_app_file, extract_unified_diff_blocks, parse_jupyter_notebook_to_extract_required_content
|
|
12
17
|
from mito_ai.utils.anthropic_utils import stream_anthropic_completion_from_mito_server
|
|
13
18
|
from mito_ai.completions.models import MessageType
|
|
19
|
+
from mito_ai.utils.telemetry_utils import log_streamlit_app_creation_error, log_streamlit_app_creation_retry, log_streamlit_app_creation_success
|
|
20
|
+
from mito_ai.streamlit_conversion.streamlit_utils import clean_directory_check
|
|
14
21
|
|
|
15
22
|
STREAMLIT_AI_MODEL = "claude-3-5-haiku-latest"
|
|
16
23
|
|
|
17
24
|
class StreamlitCodeGeneration:
|
|
18
|
-
def __init__(self, notebook: dict) -> None:
|
|
19
|
-
|
|
20
|
-
self.messages: List[MessageParam] = [
|
|
21
|
-
cast(MessageParam, {
|
|
22
|
-
"role": "user",
|
|
23
|
-
"content": [{
|
|
24
|
-
"type": "text",
|
|
25
|
-
"text": f"Here is my jupyter notebook content that I want to convert into a Streamlit dashboard - {notebook}"
|
|
26
|
-
}]
|
|
27
|
-
})
|
|
28
|
-
]
|
|
29
|
-
|
|
30
25
|
@property
|
|
31
26
|
def log(self) -> logging.Logger:
|
|
32
27
|
"""Use Mito AI logger."""
|
|
@@ -54,59 +49,110 @@ class StreamlitCodeGeneration:
|
|
|
54
49
|
accumulated_response += stream_chunk
|
|
55
50
|
return accumulated_response
|
|
56
51
|
|
|
57
|
-
def
|
|
58
|
-
"""
|
|
59
|
-
|
|
52
|
+
async def generate_streamlit_code(self, notebook: dict) -> str:
|
|
53
|
+
"""Send a query to the agent, get its response and parse the code"""
|
|
54
|
+
|
|
55
|
+
messages: List[MessageParam] = [
|
|
60
56
|
cast(MessageParam, {
|
|
61
|
-
"role": "
|
|
57
|
+
"role": "user",
|
|
62
58
|
"content": [{
|
|
63
59
|
"type": "text",
|
|
64
|
-
"text":
|
|
60
|
+
"text": get_streamlit_app_creation_prompt(notebook)
|
|
65
61
|
}]
|
|
66
62
|
})
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
"""Send a query to the agent, get its response and parse the code"""
|
|
71
|
-
agent_response = await self.get_response_from_agent(self.messages)
|
|
63
|
+
]
|
|
64
|
+
|
|
65
|
+
agent_response = await self.get_response_from_agent(messages)
|
|
72
66
|
|
|
73
67
|
converted_code = extract_code_blocks(agent_response)
|
|
74
|
-
|
|
68
|
+
|
|
69
|
+
# Extract the TODOs from the agent's response
|
|
70
|
+
todo_placeholders = extract_todo_placeholders(agent_response)
|
|
71
|
+
|
|
72
|
+
for todo_placeholder in todo_placeholders:
|
|
73
|
+
print(f"Processing AI TODO: {todo_placeholder}")
|
|
74
|
+
todo_prompt = get_finish_todo_prompt(notebook, converted_code, todo_placeholder)
|
|
75
|
+
todo_messages: List[MessageParam] = [
|
|
76
|
+
cast(MessageParam, {
|
|
77
|
+
"role": "user",
|
|
78
|
+
"content": [{
|
|
79
|
+
"type": "text",
|
|
80
|
+
"text": todo_prompt
|
|
81
|
+
}]
|
|
82
|
+
})
|
|
83
|
+
]
|
|
84
|
+
todo_response = await self.get_response_from_agent(todo_messages)
|
|
85
|
+
|
|
86
|
+
# Apply the diff to the streamlit app
|
|
87
|
+
exctracted_diff = extract_unified_diff_blocks(todo_response)
|
|
88
|
+
fixed_diff = fix_diff_headers(exctracted_diff)
|
|
89
|
+
converted_code = apply_patch_to_text(converted_code, fixed_diff)
|
|
90
|
+
|
|
75
91
|
return converted_code
|
|
76
92
|
|
|
77
93
|
|
|
78
|
-
async def correct_error_in_generation(self, error: str) -> str:
|
|
94
|
+
async def correct_error_in_generation(self, error: str, streamlit_app_code: str) -> str:
|
|
79
95
|
"""If errors are present, send it back to the agent to get corrections in code"""
|
|
80
|
-
|
|
96
|
+
messages: List[MessageParam] = [
|
|
81
97
|
cast(MessageParam, {
|
|
82
98
|
"role": "user",
|
|
83
99
|
"content": [{
|
|
84
100
|
"type": "text",
|
|
85
|
-
"text":
|
|
101
|
+
"text": get_streamlit_error_correction_prompt(error, streamlit_app_code)
|
|
86
102
|
}]
|
|
87
103
|
})
|
|
88
|
-
|
|
89
|
-
agent_response = await self.get_response_from_agent(
|
|
90
|
-
|
|
91
|
-
|
|
104
|
+
]
|
|
105
|
+
agent_response = await self.get_response_from_agent(messages)
|
|
106
|
+
|
|
107
|
+
# Apply the diff to the streamlit app
|
|
108
|
+
exctracted_diff = extract_unified_diff_blocks(agent_response)
|
|
109
|
+
|
|
110
|
+
print(f"\n\nExtracted diff: {exctracted_diff}")
|
|
111
|
+
fixed_diff = fix_diff_headers(exctracted_diff)
|
|
112
|
+
streamlit_app_code = apply_patch_to_text(streamlit_app_code, fixed_diff)
|
|
113
|
+
|
|
114
|
+
print("\n\nUpdated app code: ", streamlit_app_code)
|
|
92
115
|
|
|
93
|
-
return
|
|
116
|
+
return streamlit_app_code
|
|
94
117
|
|
|
95
118
|
|
|
96
|
-
async def streamlit_handler(notebook_path: str
|
|
119
|
+
async def streamlit_handler(notebook_path: str) -> Tuple[bool, Optional[str], str]:
|
|
97
120
|
"""Handler function for streamlit code generation and validation"""
|
|
121
|
+
|
|
122
|
+
clean_directory_check(notebook_path)
|
|
123
|
+
|
|
98
124
|
notebook_code = parse_jupyter_notebook_to_extract_required_content(notebook_path)
|
|
99
|
-
streamlit_code_generator = StreamlitCodeGeneration(
|
|
100
|
-
streamlit_code = await streamlit_code_generator.generate_streamlit_code()
|
|
101
|
-
|
|
125
|
+
streamlit_code_generator = StreamlitCodeGeneration()
|
|
126
|
+
streamlit_code = await streamlit_code_generator.generate_streamlit_code(notebook_code)
|
|
127
|
+
|
|
128
|
+
has_validation_error, errors = validate_app(streamlit_code, notebook_path)
|
|
102
129
|
tries = 0
|
|
103
130
|
while has_validation_error and tries < 5:
|
|
104
|
-
|
|
105
|
-
|
|
131
|
+
for error in errors:
|
|
132
|
+
streamlit_code = await streamlit_code_generator.correct_error_in_generation(error, streamlit_code)
|
|
133
|
+
|
|
134
|
+
has_validation_error, errors = validate_app(streamlit_code, notebook_path)
|
|
135
|
+
|
|
136
|
+
if has_validation_error:
|
|
137
|
+
# TODO: We can't easily get the key type here, so for the beta release
|
|
138
|
+
# we are just defaulting to the mito server key since that is by far the most common.
|
|
139
|
+
log_streamlit_app_creation_retry('mito_server_key', MessageType.STREAMLIT_CONVERSION, error)
|
|
106
140
|
tries+=1
|
|
107
141
|
|
|
108
142
|
if has_validation_error:
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
143
|
+
log_streamlit_app_creation_error('mito_server_key', MessageType.STREAMLIT_CONVERSION, error)
|
|
144
|
+
return False, '', "Error generating streamlit code by agent"
|
|
145
|
+
|
|
146
|
+
# Convert to absolute path for directory calculation
|
|
147
|
+
absolute_notebook_path = notebook_path
|
|
148
|
+
if not (notebook_path.startswith('/') or (len(notebook_path) > 1 and notebook_path[1] == ':')):
|
|
149
|
+
absolute_notebook_path = os.path.join(os.getcwd(), notebook_path)
|
|
150
|
+
|
|
151
|
+
app_directory = os.path.dirname(absolute_notebook_path)
|
|
152
|
+
success_flag, app_path, message = create_app_file(app_directory, streamlit_code)
|
|
153
|
+
|
|
154
|
+
if not success_flag:
|
|
155
|
+
log_streamlit_app_creation_error('mito_server_key', MessageType.STREAMLIT_CONVERSION, message)
|
|
156
|
+
|
|
157
|
+
log_streamlit_app_creation_success('mito_server_key', MessageType.STREAMLIT_CONVERSION)
|
|
158
|
+
return success_flag, app_path, message
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Copyright (c) Saga Inc.
|
|
2
2
|
# Distributed under the terms of the GNU Affero General Public License v3.0 License.
|
|
3
3
|
|
|
4
|
-
streamlit_system_prompt = """You are a
|
|
4
|
+
streamlit_system_prompt = """You are a code conversion specialist who converts Jupyter notebooks into Streamlit applications with ABSOLUTE FIDELITY.
|
|
5
5
|
|
|
6
6
|
ROLE AND EXPERTISE:
|
|
7
7
|
- Expert in Python, Jupyter notebooks, Streamlit, and data visualization
|
|
@@ -17,26 +17,39 @@ TASK REQUIREMENTS:
|
|
|
17
17
|
STREAMLIT IMPLEMENTATION GUIDELINES:
|
|
18
18
|
- Use appropriate Streamlit components (st.title, st.header, st.subheader, st.markdown, etc.)
|
|
19
19
|
- Display all visualizations using st.pyplot(), st.plotly_chart(), or st.altair_chart() as appropriate
|
|
20
|
+
- Do not convert database connections into Streamlit's secret.toml format. If the user inlined their database credentials, are importing from an environment variable, or reading from a connections file, assume that same approach will work in the streamlit app.
|
|
20
21
|
- Show dataframes and tables using st.dataframe() or st.table()
|
|
21
22
|
- Include all text explanations and insights from markdown cells
|
|
22
23
|
- Add interactive elements where beneficial (filters, selectors, etc.)
|
|
23
24
|
- Ensure professional styling and layout suitable for executives
|
|
24
25
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
26
|
+
CRITICAL REQUIREMENTS:
|
|
27
|
+
1. **PRESERVE ALL CODE EXACTLY**: Every line of code, every data structure, every import must be included in full
|
|
28
|
+
2. **NO PLACEHOLDERS**: Never use comments like "# Add more data here" or "# Fill in the rest"
|
|
29
|
+
3. **NO SIMPLIFICATION**: Do not replace actual data with sample data or hardcoded examples
|
|
30
|
+
4. **COMPLETE DATA STRUCTURES**: If a notebook has a 1000-line dictionary, include all 1000 lines
|
|
31
|
+
5. **PRESERVE DATA LOADING**: If the notebook reads from files, the Streamlit app must read from the same files
|
|
32
|
+
6. **NO IMPROVIZAITION**: Do not provide your own interpretations of the analysis. Just convert the existing analysis into a streamlit app.
|
|
33
|
+
|
|
34
|
+
STYLE GUIDELINES:
|
|
35
|
+
- Create a professional, executive-friendly dashboard
|
|
36
|
+
- If there are variables in the notebook that the streamlit app viewer would likely want to configure, then use the appropriate streamlit component to allow them to do so. For examples, if the notebook has a variable called "start_date" and "end_date", then use the st.date_input component to allow the user to select the start and end dates.
|
|
37
|
+
- Do not use emojis unless they are in the notebook already
|
|
38
|
+
- Do not modify the graphs or analysis. If the notebook has a graph, use the same graph in the streamlit app.
|
|
39
|
+
- Always include the following code at the top of the file so the user does not use the wrong deploy button
|
|
40
|
+
```python
|
|
41
|
+
st.markdown(\"\"\"
|
|
42
|
+
<style>
|
|
43
|
+
#MainMenu {visibility: hidden;}
|
|
44
|
+
.stAppDeployButton {display:none;}
|
|
45
|
+
footer {visibility: hidden;}
|
|
46
|
+
.stMainBlockContainer {padding: 2rem 1rem 2rem 1rem;}
|
|
47
|
+
</style>
|
|
48
|
+
\"\"\", unsafe_allow_html=True)
|
|
49
|
+
```
|
|
31
50
|
|
|
32
51
|
OUTPUT FORMAT:
|
|
33
|
-
-
|
|
34
|
-
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
- Don't give extra explanations, just give the python code
|
|
38
|
-
- Do NOT add emojis
|
|
39
|
-
- Do NOT modify the graphs or analysis
|
|
40
|
-
- Do NOT provide your own interpretations for the analysis
|
|
41
|
-
|
|
42
|
-
Remember: The goal is to transform technical analysis into a polished, interactive/visually appealing dashboard that executives can easily understand and navigate."""
|
|
52
|
+
- Output the complete, runnable app.py file.
|
|
53
|
+
- Do not output any extra text, just give the python code.
|
|
54
|
+
|
|
55
|
+
"""
|
|
@@ -3,7 +3,9 @@
|
|
|
3
3
|
|
|
4
4
|
import re
|
|
5
5
|
import json
|
|
6
|
-
|
|
6
|
+
import os
|
|
7
|
+
from typing import Dict, Optional, Tuple, Any
|
|
8
|
+
from pathlib import Path
|
|
7
9
|
|
|
8
10
|
def extract_code_blocks(message_content: str) -> str:
|
|
9
11
|
"""
|
|
@@ -18,7 +20,6 @@ def extract_code_blocks(message_content: str) -> str:
|
|
|
18
20
|
if "```python" not in message_content:
|
|
19
21
|
return message_content
|
|
20
22
|
|
|
21
|
-
# return message_content.split('```python\n')[1].split('\n```')[0]
|
|
22
23
|
# Use regex to find all Python code blocks
|
|
23
24
|
pattern = r'```python\n(.*?)```'
|
|
24
25
|
matches = re.findall(pattern, message_content, re.DOTALL)
|
|
@@ -26,8 +27,19 @@ def extract_code_blocks(message_content: str) -> str:
|
|
|
26
27
|
# Concatenate with single newlines
|
|
27
28
|
return '\n'.join(matches)
|
|
28
29
|
|
|
30
|
+
def extract_unified_diff_blocks(message_content: str) -> str:
|
|
31
|
+
"""
|
|
32
|
+
Extract all unified_diff blocks from Claude's response.
|
|
33
|
+
"""
|
|
34
|
+
if "```unified_diff" not in message_content:
|
|
35
|
+
return message_content
|
|
36
|
+
|
|
37
|
+
pattern = r'```unified_diff\n(.*?)```'
|
|
38
|
+
matches = re.findall(pattern, message_content, re.DOTALL)
|
|
39
|
+
return '\n'.join(matches)
|
|
40
|
+
|
|
29
41
|
|
|
30
|
-
def create_app_file(
|
|
42
|
+
def create_app_file(app_directory: str, code: str) -> Tuple[bool, str, str]:
|
|
31
43
|
"""
|
|
32
44
|
Create app.py file and write code to it with error handling
|
|
33
45
|
|
|
@@ -40,13 +52,14 @@ def create_app_file(file_path: str, code: str) -> Tuple[bool, str]:
|
|
|
40
52
|
|
|
41
53
|
"""
|
|
42
54
|
try:
|
|
43
|
-
|
|
55
|
+
app_path = os.path.join(app_directory, "app.py")
|
|
56
|
+
with open(app_path, 'w') as f:
|
|
44
57
|
f.write(code)
|
|
45
|
-
return True, f"Successfully created {
|
|
58
|
+
return True, app_path, f"Successfully created {app_directory}"
|
|
46
59
|
except IOError as e:
|
|
47
|
-
return False, f"Error creating file: {str(e)}"
|
|
60
|
+
return False, '', f"Error creating file: {str(e)}"
|
|
48
61
|
except Exception as e:
|
|
49
|
-
return False, f"Unexpected error: {str(e)}"
|
|
62
|
+
return False, '', f"Unexpected error: {str(e)}"
|
|
50
63
|
|
|
51
64
|
|
|
52
65
|
def parse_jupyter_notebook_to_extract_required_content(notebook_path: str) -> Dict[str, Any]:
|
|
@@ -54,7 +67,7 @@ def parse_jupyter_notebook_to_extract_required_content(notebook_path: str) -> Di
|
|
|
54
67
|
Read a Jupyter notebook and filter cells to keep only cell_type and source fields.
|
|
55
68
|
|
|
56
69
|
Args:
|
|
57
|
-
notebook_path (str):
|
|
70
|
+
notebook_path (str): Path to the .ipynb file (can be relative or absolute)
|
|
58
71
|
|
|
59
72
|
Returns:
|
|
60
73
|
dict: Filtered notebook dictionary with only cell_type and source in cells
|
|
@@ -64,6 +77,11 @@ def parse_jupyter_notebook_to_extract_required_content(notebook_path: str) -> Di
|
|
|
64
77
|
json.JSONDecodeError: If the file is not valid JSON
|
|
65
78
|
KeyError: If the notebook doesn't have the expected structure
|
|
66
79
|
"""
|
|
80
|
+
# Convert to absolute path if it's not already absolute
|
|
81
|
+
# Handle both Unix-style absolute paths (starting with /) and Windows-style absolute paths
|
|
82
|
+
if not (notebook_path.startswith('/') or (len(notebook_path) > 1 and notebook_path[1] == ':')):
|
|
83
|
+
notebook_path = os.path.join(os.getcwd(), notebook_path)
|
|
84
|
+
|
|
67
85
|
try:
|
|
68
86
|
# Read the notebook file
|
|
69
87
|
with open(notebook_path, 'r', encoding='utf-8') as f:
|
|
@@ -94,3 +112,25 @@ def parse_jupyter_notebook_to_extract_required_content(notebook_path: str) -> Di
|
|
|
94
112
|
raise json.JSONDecodeError(f"Invalid JSON in notebook file: {str(e)}", e.doc if hasattr(e, 'doc') else '', e.pos if hasattr(e, 'pos') else 0)
|
|
95
113
|
except Exception as e:
|
|
96
114
|
raise Exception(f"Error processing notebook: {str(e)}")
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def resolve_notebook_path(notebook_path:str) -> str:
|
|
118
|
+
# Convert to absolute path if it's not already absolute
|
|
119
|
+
# Handle both Unix-style absolute paths (starting with /) and Windows-style absolute paths
|
|
120
|
+
if not (notebook_path.startswith('/') or (len(notebook_path) > 1 and notebook_path[1] == ':')):
|
|
121
|
+
notebook_path = os.path.join(os.getcwd(), notebook_path)
|
|
122
|
+
return notebook_path
|
|
123
|
+
|
|
124
|
+
def clean_directory_check(notebook_path: str) -> None:
|
|
125
|
+
notebook_path = resolve_notebook_path(notebook_path)
|
|
126
|
+
# pathlib handles the cross OS path conversion automatically
|
|
127
|
+
path = Path(notebook_path).resolve()
|
|
128
|
+
dir_path = path.parent
|
|
129
|
+
|
|
130
|
+
if not dir_path.exists():
|
|
131
|
+
raise ValueError(f"Directory does not exist: {dir_path}")
|
|
132
|
+
|
|
133
|
+
file_count = len([f for f in dir_path.iterdir() if f.is_file()])
|
|
134
|
+
if file_count > 10:
|
|
135
|
+
raise ValueError(
|
|
136
|
+
f"Too many files in directory: 10 allowed but {file_count} present. Create a new directory and retry")
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# Copyright (c) Saga Inc.
|
|
2
|
+
# Distributed under the terms of the GNU Affero General Public License v3.0 License.
|
|
3
|
+
|
|
4
|
+
import sys
|
|
5
|
+
import os
|
|
6
|
+
import time
|
|
7
|
+
import requests
|
|
8
|
+
import tempfile
|
|
9
|
+
import shutil
|
|
10
|
+
import traceback
|
|
11
|
+
import ast
|
|
12
|
+
import importlib.util
|
|
13
|
+
import warnings
|
|
14
|
+
from typing import List, Tuple, Optional, Dict, Any, Generator
|
|
15
|
+
from streamlit.testing.v1 import AppTest
|
|
16
|
+
from contextlib import contextmanager
|
|
17
|
+
from mito_ai.streamlit_conversion.streamlit_utils import resolve_notebook_path
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
# warnings.filterwarnings("ignore", message=r".*missing ScriptRunContext.*")
|
|
21
|
+
# warnings.filterwarnings("ignore", category=UserWarning)
|
|
22
|
+
|
|
23
|
+
warnings.filterwarnings("ignore", message=".*bare mode.*")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class StreamlitValidator:
|
|
27
|
+
def __init__(self, port: int = 8501) -> None:
|
|
28
|
+
self.temp_dir: Optional[str] = None
|
|
29
|
+
|
|
30
|
+
def get_syntax_error(self, app_code: str) -> Optional[str]:
|
|
31
|
+
"""Check if the Python code has valid syntax"""
|
|
32
|
+
try:
|
|
33
|
+
ast.parse(app_code)
|
|
34
|
+
return None
|
|
35
|
+
except SyntaxError as e:
|
|
36
|
+
error_msg = ''.join(traceback.format_exception(type(e), e, e.__traceback__))
|
|
37
|
+
return error_msg
|
|
38
|
+
|
|
39
|
+
def get_runtime_errors(self, app_code: str, app_path: str) -> Optional[List[Dict[str, Any]]]:
|
|
40
|
+
"""Start the Streamlit app in a subprocess"""
|
|
41
|
+
|
|
42
|
+
directory = os.path.dirname(app_path)
|
|
43
|
+
|
|
44
|
+
@contextmanager
|
|
45
|
+
def change_working_directory(path: str) -> Generator[None, Any, None]:
|
|
46
|
+
"""
|
|
47
|
+
Context manager to temporarily change working directory
|
|
48
|
+
so that relative paths are still valid when we run the app
|
|
49
|
+
"""
|
|
50
|
+
if path == '':
|
|
51
|
+
yield
|
|
52
|
+
|
|
53
|
+
original_cwd = os.getcwd()
|
|
54
|
+
try:
|
|
55
|
+
os.chdir(path)
|
|
56
|
+
yield
|
|
57
|
+
finally:
|
|
58
|
+
os.chdir(original_cwd)
|
|
59
|
+
|
|
60
|
+
with change_working_directory(directory):
|
|
61
|
+
app_test = AppTest.from_string(app_code, default_timeout=30)
|
|
62
|
+
app_test.run()
|
|
63
|
+
|
|
64
|
+
# Check for exceptions
|
|
65
|
+
if app_test.exception:
|
|
66
|
+
errors = [{'type': 'exception', 'details': exc.value, 'message': exc.message, 'stack_trace': exc.stack_trace} for exc in app_test.exception]
|
|
67
|
+
return errors
|
|
68
|
+
|
|
69
|
+
# Check for error messages
|
|
70
|
+
if app_test.error:
|
|
71
|
+
errors = [{'type': 'error', 'details': err.value} for err in app_test.error]
|
|
72
|
+
return errors
|
|
73
|
+
|
|
74
|
+
return None
|
|
75
|
+
|
|
76
|
+
def cleanup(self) -> None:
|
|
77
|
+
"""Clean up the temporary files"""
|
|
78
|
+
if self.temp_dir and os.path.exists(self.temp_dir):
|
|
79
|
+
shutil.rmtree(self.temp_dir)
|
|
80
|
+
self.temp_dir = None
|
|
81
|
+
|
|
82
|
+
def _validate_app(self, app_code: str, app_path: str) -> List[Dict[str, Any]]:
|
|
83
|
+
"""Complete validation pipeline"""
|
|
84
|
+
errors: List[Dict[str, Any]] = []
|
|
85
|
+
|
|
86
|
+
try:
|
|
87
|
+
# Step 1: Check syntax
|
|
88
|
+
syntax_error = self.get_syntax_error(app_code)
|
|
89
|
+
if syntax_error:
|
|
90
|
+
errors.append({'type': 'syntax', 'details': syntax_error})
|
|
91
|
+
|
|
92
|
+
runtime_errors = self.get_runtime_errors(app_code, app_path)
|
|
93
|
+
|
|
94
|
+
print('Found Runtime Errors', runtime_errors)
|
|
95
|
+
|
|
96
|
+
if runtime_errors:
|
|
97
|
+
errors.extend(runtime_errors)
|
|
98
|
+
|
|
99
|
+
except Exception as e:
|
|
100
|
+
errors.append({'type': 'validation', 'details': str(e)})
|
|
101
|
+
|
|
102
|
+
finally:
|
|
103
|
+
self.cleanup()
|
|
104
|
+
|
|
105
|
+
return errors
|
|
106
|
+
|
|
107
|
+
def validate_app(app_code: str, notebook_path: str) -> Tuple[bool, List[str]]:
|
|
108
|
+
"""Convenience function to validate Streamlit code"""
|
|
109
|
+
notebook_path = resolve_notebook_path(notebook_path)
|
|
110
|
+
|
|
111
|
+
validator = StreamlitValidator()
|
|
112
|
+
errors = validator._validate_app(app_code, notebook_path)
|
|
113
|
+
|
|
114
|
+
has_validation_error = len(errors) > 0
|
|
115
|
+
stringified_errors = [str(error) for error in errors]
|
|
116
|
+
return has_validation_error, stringified_errors
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Copyright (c) Saga Inc.
|
|
2
|
+
# Distributed under the terms of the GNU Affero General Public License v3.0 License.
|
|
3
|
+
|
|
4
|
+
from .manager import get_preview_manager
|
|
5
|
+
from .handlers import StreamlitPreviewHandler
|
|
6
|
+
|
|
7
|
+
__all__ = ['get_preview_manager', 'StreamlitPreviewHandler']
|