aidialog 0.0.2__tar.gz → 0.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {aidialog-0.0.2/aidialog.egg-info → aidialog-0.0.4}/PKG-INFO +3 -3
- {aidialog-0.0.2 → aidialog-0.0.4}/README.md +1 -1
- aidialog-0.0.4/aidialog/__init__.py +7 -0
- {aidialog-0.0.2 → aidialog-0.0.4}/aidialog/_modidx.py +53 -3
- {aidialog-0.0.2 → aidialog-0.0.4}/aidialog/dialog.py +183 -59
- {aidialog-0.0.2 → aidialog-0.0.4}/aidialog/dlgskill.py +5 -5
- {aidialog-0.0.2 → aidialog-0.0.4}/aidialog/ipynb.py +17 -17
- aidialog-0.0.4/aidialog/msg_parts.py +340 -0
- {aidialog-0.0.2 → aidialog-0.0.4/aidialog.egg-info}/PKG-INFO +3 -3
- {aidialog-0.0.2 → aidialog-0.0.4}/aidialog.egg-info/SOURCES.txt +1 -0
- {aidialog-0.0.2 → aidialog-0.0.4}/aidialog.egg-info/requires.txt +1 -1
- {aidialog-0.0.2 → aidialog-0.0.4}/pyproject.toml +1 -1
- aidialog-0.0.2/aidialog/__init__.py +0 -1
- {aidialog-0.0.2 → aidialog-0.0.4}/LICENSE +0 -0
- {aidialog-0.0.2 → aidialog-0.0.4}/MANIFEST.in +0 -0
- {aidialog-0.0.2 → aidialog-0.0.4}/aidialog.egg-info/dependency_links.txt +0 -0
- {aidialog-0.0.2 → aidialog-0.0.4}/aidialog.egg-info/entry_points.txt +0 -0
- {aidialog-0.0.2 → aidialog-0.0.4}/aidialog.egg-info/top_level.txt +0 -0
- {aidialog-0.0.2 → aidialog-0.0.4}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: aidialog
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.4
|
|
4
4
|
Summary: Read, search, edit, and persist AI dialogs stored as Jupyter notebooks
|
|
5
5
|
Author-email: Jeremy Howard <github@jhoward.fastmail.fm>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3 :: Only
|
|
|
12
12
|
Requires-Python: >=3.11
|
|
13
13
|
Description-Content-Type: text/markdown
|
|
14
14
|
License-File: LICENSE
|
|
15
|
-
Requires-Dist: fastcore>=2.1.
|
|
15
|
+
Requires-Dist: fastcore>=2.1.13
|
|
16
16
|
Provides-Extra: dev
|
|
17
17
|
Requires-Dist: exhash; extra == "dev"
|
|
18
18
|
Requires-Dist: nbformat; extra == "dev"
|
|
@@ -51,7 +51,7 @@ The [`Dialog`](https://AnswerDotAI.github.io/aidialog/dialog.html#dialog) is the
|
|
|
51
51
|
| ipynb file | storage, pragmatically lossless | [`read_ipynb`](https://AnswerDotAI.github.io/aidialog/ipynb.html#read_ipynb) | [`write_ipynb`](https://AnswerDotAI.github.io/aidialog/ipynb.html#write_ipynb) |
|
|
52
52
|
| Claude Code session | storage | `sess2dlg` | `dlg2sess` |
|
|
53
53
|
| Codex thread | storage (write-only so far) | | `dlg2thread` |
|
|
54
|
-
| fastllm chat (`Msg
|
|
54
|
+
| fastllm chat ([`Msg`](https://AnswerDotAI.github.io/aidialog/msg_parts.html#msg)/[`Part`](https://AnswerDotAI.github.io/aidialog/msg_parts.html#part)) | transmission, normalizing | `chat2dlg` | `dlg2chat` |
|
|
55
55
|
| fastllm hist (live call input) | transmission, one-way | | `dlg2hist` |
|
|
56
56
|
| a prompt’s reply | self-similar | `reply2dlg` | `dlg2reply` |
|
|
57
57
|
| XML views | display, one-way | | [`view_dlg`](https://AnswerDotAI.github.io/aidialog/dlgskill.html#view_dlg), [`msg2xml`](https://AnswerDotAI.github.io/aidialog/dlgskill.html#msg2xml) |
|
|
@@ -31,7 +31,7 @@ The [`Dialog`](https://AnswerDotAI.github.io/aidialog/dialog.html#dialog) is the
|
|
|
31
31
|
| ipynb file | storage, pragmatically lossless | [`read_ipynb`](https://AnswerDotAI.github.io/aidialog/ipynb.html#read_ipynb) | [`write_ipynb`](https://AnswerDotAI.github.io/aidialog/ipynb.html#write_ipynb) |
|
|
32
32
|
| Claude Code session | storage | `sess2dlg` | `dlg2sess` |
|
|
33
33
|
| Codex thread | storage (write-only so far) | | `dlg2thread` |
|
|
34
|
-
| fastllm chat (`Msg
|
|
34
|
+
| fastllm chat ([`Msg`](https://AnswerDotAI.github.io/aidialog/msg_parts.html#msg)/[`Part`](https://AnswerDotAI.github.io/aidialog/msg_parts.html#part)) | transmission, normalizing | `chat2dlg` | `dlg2chat` |
|
|
35
35
|
| fastllm hist (live call input) | transmission, one-way | | `dlg2hist` |
|
|
36
36
|
| a prompt’s reply | self-similar | `reply2dlg` | `dlg2reply` |
|
|
37
37
|
| XML views | display, one-way | | [`view_dlg`](https://AnswerDotAI.github.io/aidialog/dlgskill.html#view_dlg), [`msg2xml`](https://AnswerDotAI.github.io/aidialog/dlgskill.html#msg2xml) |
|
|
@@ -27,6 +27,7 @@ d = { 'settings': { 'branch': 'main',
|
|
|
27
27
|
'aidialog.dialog.Message._neighbor': ('dialog.html#message._neighbor', 'aidialog/dialog.py'),
|
|
28
28
|
'aidialog.dialog.Message._repr_markdown_': ('dialog.html#message._repr_markdown_', 'aidialog/dialog.py'),
|
|
29
29
|
'aidialog.dialog.Message.add_output': ('dialog.html#message.add_output', 'aidialog/dialog.py'),
|
|
30
|
+
'aidialog.dialog.Message.ai_output': ('dialog.html#message.ai_output', 'aidialog/dialog.py'),
|
|
30
31
|
'aidialog.dialog.Message.ai_res': ('dialog.html#message.ai_res', 'aidialog/dialog.py'),
|
|
31
32
|
'aidialog.dialog.Message.clear_inp_cache': ('dialog.html#message.clear_inp_cache', 'aidialog/dialog.py'),
|
|
32
33
|
'aidialog.dialog.Message.clear_out_cache': ('dialog.html#message.clear_out_cache', 'aidialog/dialog.py'),
|
|
@@ -47,7 +48,9 @@ d = { 'settings': { 'branch': 'main',
|
|
|
47
48
|
'aidialog.dialog.Message.previous': ('dialog.html#message.previous', 'aidialog/dialog.py'),
|
|
48
49
|
'aidialog.dialog.Message.rel_msg': ('dialog.html#message.rel_msg', 'aidialog/dialog.py'),
|
|
49
50
|
'aidialog.dialog.Message.remove_from_dlg': ('dialog.html#message.remove_from_dlg', 'aidialog/dialog.py'),
|
|
51
|
+
'aidialog.dialog.Message.render_out': ('dialog.html#message.render_out', 'aidialog/dialog.py'),
|
|
50
52
|
'aidialog.dialog.Message.summ': ('dialog.html#message.summ', 'aidialog/dialog.py'),
|
|
53
|
+
'aidialog.dialog.Message.todict': ('dialog.html#message.todict', 'aidialog/dialog.py'),
|
|
51
54
|
'aidialog.dialog.Message.update': ('dialog.html#message.update', 'aidialog/dialog.py'),
|
|
52
55
|
'aidialog.dialog.Message.validate': ('dialog.html#message.validate', 'aidialog/dialog.py'),
|
|
53
56
|
'aidialog.dialog.MsgRow': ('dialog.html#msgrow', 'aidialog/dialog.py'),
|
|
@@ -58,14 +61,21 @@ d = { 'settings': { 'branch': 'main',
|
|
|
58
61
|
'aidialog.dialog.Msgs': ('dialog.html#msgs', 'aidialog/dialog.py'),
|
|
59
62
|
'aidialog.dialog.Msgs.__repr__': ('dialog.html#msgs.__repr__', 'aidialog/dialog.py'),
|
|
60
63
|
'aidialog.dialog.Msgs.show': ('dialog.html#msgs.show', 'aidialog/dialog.py'),
|
|
64
|
+
'aidialog.dialog._code_span': ('dialog.html#_code_span', 'aidialog/dialog.py'),
|
|
65
|
+
'aidialog.dialog._fmt_param': ('dialog.html#_fmt_param', 'aidialog/dialog.py'),
|
|
61
66
|
'aidialog.dialog._get_exported': ('dialog.html#_get_exported', 'aidialog/dialog.py'),
|
|
67
|
+
'aidialog.dialog._get_mexp': ('dialog.html#_get_mexp', 'aidialog/dialog.py'),
|
|
62
68
|
'aidialog.dialog._latex_parts': ('dialog.html#_latex_parts', 'aidialog/dialog.py'),
|
|
63
|
-
'aidialog.dialog.
|
|
69
|
+
'aidialog.dialog._prev_line': ('dialog.html#_prev_line', 'aidialog/dialog.py'),
|
|
70
|
+
'aidialog.dialog._set_mexp': ('dialog.html#_set_mexp', 'aidialog/dialog.py'),
|
|
64
71
|
'aidialog.dialog.add_id_hash': ('dialog.html#add_id_hash', 'aidialog/dialog.py'),
|
|
72
|
+
'aidialog.dialog.ai_fmt': ('dialog.html#ai_fmt', 'aidialog/dialog.py'),
|
|
65
73
|
'aidialog.dialog.code_output': ('dialog.html#code_output', 'aidialog/dialog.py'),
|
|
66
74
|
'aidialog.dialog.copy_export': ('dialog.html#copy_export', 'aidialog/dialog.py'),
|
|
67
75
|
'aidialog.dialog.displayobj': ('dialog.html#displayobj', 'aidialog/dialog.py'),
|
|
76
|
+
'aidialog.dialog.dlg2md': ('dialog.html#dlg2md', 'aidialog/dialog.py'),
|
|
68
77
|
'aidialog.dialog.dlg2py': ('dialog.html#dlg2py', 'aidialog/dialog.py'),
|
|
78
|
+
'aidialog.dialog.fmt_tools': ('dialog.html#fmt_tools', 'aidialog/dialog.py'),
|
|
69
79
|
'aidialog.dialog.get_msg': ('dialog.html#get_msg', 'aidialog/dialog.py'),
|
|
70
80
|
'aidialog.dialog.get_output_mds': ('dialog.html#get_output_mds', 'aidialog/dialog.py'),
|
|
71
81
|
'aidialog.dialog.header_info': ('dialog.html#header_info', 'aidialog/dialog.py'),
|
|
@@ -80,13 +90,17 @@ d = { 'settings': { 'branch': 'main',
|
|
|
80
90
|
'aidialog.dialog.mk_jmsg': ('dialog.html#mk_jmsg', 'aidialog/dialog.py'),
|
|
81
91
|
'aidialog.dialog.mk_output': ('dialog.html#mk_output', 'aidialog/dialog.py'),
|
|
82
92
|
'aidialog.dialog.mk_stream': ('dialog.html#mk_stream', 'aidialog/dialog.py'),
|
|
93
|
+
'aidialog.dialog.msg2md': ('dialog.html#msg2md', 'aidialog/dialog.py'),
|
|
83
94
|
'aidialog.dialog.normalize_text_latex': ('dialog.html#normalize_text_latex', 'aidialog/dialog.py'),
|
|
84
95
|
'aidialog.dialog.output_from_msg': ('dialog.html#output_from_msg', 'aidialog/dialog.py'),
|
|
85
96
|
'aidialog.dialog.prompt_output': ('dialog.html#prompt_output', 'aidialog/dialog.py'),
|
|
86
97
|
'aidialog.dialog.render_output_ai': ('dialog.html#render_output_ai', 'aidialog/dialog.py'),
|
|
87
98
|
'aidialog.dialog.render_outputs_ai': ('dialog.html#render_outputs_ai', 'aidialog/dialog.py'),
|
|
88
99
|
'aidialog.dialog.ruuid4': ('dialog.html#ruuid4', 'aidialog/dialog.py'),
|
|
89
|
-
'aidialog.dialog.section_msgs': ('dialog.html#section_msgs', 'aidialog/dialog.py')
|
|
100
|
+
'aidialog.dialog.section_msgs': ('dialog.html#section_msgs', 'aidialog/dialog.py'),
|
|
101
|
+
'aidialog.dialog.tool_md': ('dialog.html#tool_md', 'aidialog/dialog.py'),
|
|
102
|
+
'aidialog.dialog.try_eval': ('dialog.html#try_eval', 'aidialog/dialog.py'),
|
|
103
|
+
'aidialog.dialog.usage_md': ('dialog.html#usage_md', 'aidialog/dialog.py')},
|
|
90
104
|
'aidialog.dlgskill': { 'aidialog.dlgskill.Dialog.copy_msgs': ('dlgskill.html#dialog.copy_msgs', 'aidialog/dlgskill.py'),
|
|
91
105
|
'aidialog.dlgskill.Dialog.cut_msgs': ('dlgskill.html#dialog.cut_msgs', 'aidialog/dlgskill.py'),
|
|
92
106
|
'aidialog.dlgskill.Dialog.find_msgs': ('dlgskill.html#dialog.find_msgs', 'aidialog/dlgskill.py'),
|
|
@@ -150,4 +164,40 @@ d = { 'settings': { 'branch': 'main',
|
|
|
150
164
|
'aidialog.ipynb.read_ipynb': ('ipynb.html#read_ipynb', 'aidialog/ipynb.py'),
|
|
151
165
|
'aidialog.ipynb.reads_ipynb': ('ipynb.html#reads_ipynb', 'aidialog/ipynb.py'),
|
|
152
166
|
'aidialog.ipynb.split_cell_src': ('ipynb.html#split_cell_src', 'aidialog/ipynb.py'),
|
|
153
|
-
'aidialog.ipynb.write_ipynb': ('ipynb.html#write_ipynb', 'aidialog/ipynb.py')}
|
|
167
|
+
'aidialog.ipynb.write_ipynb': ('ipynb.html#write_ipynb', 'aidialog/ipynb.py')},
|
|
168
|
+
'aidialog.msg_parts': { 'aidialog.msg_parts.FullResponse': ('msg_parts.html#fullresponse', 'aidialog/msg_parts.py'),
|
|
169
|
+
'aidialog.msg_parts.MediaUrl': ('msg_parts.html#mediaurl', 'aidialog/msg_parts.py'),
|
|
170
|
+
'aidialog.msg_parts.MediaUrl.__init__': ('msg_parts.html#mediaurl.__init__', 'aidialog/msg_parts.py'),
|
|
171
|
+
'aidialog.msg_parts.Msg': ('msg_parts.html#msg', 'aidialog/msg_parts.py'),
|
|
172
|
+
'aidialog.msg_parts.Msg._repr_markdown_': ( 'msg_parts.html#msg._repr_markdown_',
|
|
173
|
+
'aidialog/msg_parts.py'),
|
|
174
|
+
'aidialog.msg_parts.Msg.text': ('msg_parts.html#msg.text', 'aidialog/msg_parts.py'),
|
|
175
|
+
'aidialog.msg_parts.Part': ('msg_parts.html#part', 'aidialog/msg_parts.py'),
|
|
176
|
+
'aidialog.msg_parts.Part._repr_markdown_': ( 'msg_parts.html#part._repr_markdown_',
|
|
177
|
+
'aidialog/msg_parts.py'),
|
|
178
|
+
'aidialog.msg_parts.StopResponse': ('msg_parts.html#stopresponse', 'aidialog/msg_parts.py'),
|
|
179
|
+
'aidialog.msg_parts.ToolCall': ('msg_parts.html#toolcall', 'aidialog/msg_parts.py'),
|
|
180
|
+
'aidialog.msg_parts.ToolCall._repr_markdown_': ( 'msg_parts.html#toolcall._repr_markdown_',
|
|
181
|
+
'aidialog/msg_parts.py'),
|
|
182
|
+
'aidialog.msg_parts._extract_tool_parts': ( 'msg_parts.html#_extract_tool_parts',
|
|
183
|
+
'aidialog/msg_parts.py'),
|
|
184
|
+
'aidialog.msg_parts._fetch_url_partial': ('msg_parts.html#_fetch_url_partial', 'aidialog/msg_parts.py'),
|
|
185
|
+
'aidialog.msg_parts._mk_result_fence': ('msg_parts.html#_mk_result_fence', 'aidialog/msg_parts.py'),
|
|
186
|
+
'aidialog.msg_parts._split_fence_msgs': ('msg_parts.html#_split_fence_msgs', 'aidialog/msg_parts.py'),
|
|
187
|
+
'aidialog.msg_parts._split_msg_on_fences': ( 'msg_parts.html#_split_msg_on_fences',
|
|
188
|
+
'aidialog/msg_parts.py'),
|
|
189
|
+
'aidialog.msg_parts._trunc_content': ('msg_parts.html#_trunc_content', 'aidialog/msg_parts.py'),
|
|
190
|
+
'aidialog.msg_parts._trunc_str': ('msg_parts.html#_trunc_str', 'aidialog/msg_parts.py'),
|
|
191
|
+
'aidialog.msg_parts._trunc_strs': ('msg_parts.html#_trunc_strs', 'aidialog/msg_parts.py'),
|
|
192
|
+
'aidialog.msg_parts.conv_tools': ('msg_parts.html#conv_tools', 'aidialog/msg_parts.py'),
|
|
193
|
+
'aidialog.msg_parts.data_url': ('msg_parts.html#data_url', 'aidialog/msg_parts.py'),
|
|
194
|
+
'aidialog.msg_parts.display_list': ('msg_parts.html#display_list', 'aidialog/msg_parts.py'),
|
|
195
|
+
'aidialog.msg_parts.fmt2hist': ('msg_parts.html#fmt2hist', 'aidialog/msg_parts.py'),
|
|
196
|
+
'aidialog.msg_parts.hist2fmt': ('msg_parts.html#hist2fmt', 'aidialog/msg_parts.py'),
|
|
197
|
+
'aidialog.msg_parts.mk_tool_res_msg': ('msg_parts.html#mk_tool_res_msg', 'aidialog/msg_parts.py'),
|
|
198
|
+
'aidialog.msg_parts.mk_tr_details': ('msg_parts.html#mk_tr_details', 'aidialog/msg_parts.py'),
|
|
199
|
+
'aidialog.msg_parts.parse_tools': ('msg_parts.html#parse_tools', 'aidialog/msg_parts.py'),
|
|
200
|
+
'aidialog.msg_parts.part_txt': ('msg_parts.html#part_txt', 'aidialog/msg_parts.py'),
|
|
201
|
+
'aidialog.msg_parts.strip_tools': ('msg_parts.html#strip_tools', 'aidialog/msg_parts.py'),
|
|
202
|
+
'aidialog.msg_parts.sys_text': ('msg_parts.html#sys_text', 'aidialog/msg_parts.py'),
|
|
203
|
+
'aidialog.msg_parts.url_mime': ('msg_parts.html#url_mime', 'aidialog/msg_parts.py')}}}
|
|
@@ -2,27 +2,29 @@
|
|
|
2
2
|
|
|
3
3
|
Docs: https://AnswerDotAI.github.io/aidialog/dialog.html.md"""
|
|
4
4
|
|
|
5
|
-
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/
|
|
5
|
+
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/01_dialog.ipynb.
|
|
6
6
|
|
|
7
7
|
# %% auto #0
|
|
8
8
|
__all__ = ['smsg_types', 'scode', 'snote', 'sprompt', 'sraw', 'AI_RENDERERS', 'tiny_png', 'add_id_hash', 'Msgs', 'Dialog',
|
|
9
9
|
'mk_output', 'mk_displayobj', 'displayobj', 'mk_code_output', 'code_output', 'prompt_output', 'Message',
|
|
10
10
|
'MsgRow', 'MsgRows', 'get_msg', 'header_info', 'section_msgs', 'get_output_mds', 'join_out',
|
|
11
|
-
'normalize_text_latex', 'render_output_ai', 'render_outputs_ai', '
|
|
12
|
-
'
|
|
13
|
-
'ruuid4', 'Attachment']
|
|
11
|
+
'normalize_text_latex', 'render_output_ai', 'render_outputs_ai', 'ai_fmt', 'try_eval', 'mk_jmsg',
|
|
12
|
+
'mk_stream', 'mk_error', 'mk_dispdata', 'mk_execresult', 'output_from_msg', 'dlg2py', 'copy_export',
|
|
13
|
+
'merge_metas', 'merge_parts', 'ruuid4', 'Attachment', 'tool_md', 'usage_md', 'fmt_tools', 'msg2md', 'dlg2md']
|
|
14
14
|
|
|
15
|
-
# %% ../nbs/
|
|
16
|
-
import base64, copy, random, weakref
|
|
15
|
+
# %% ../nbs/01_dialog.ipynb #5571d07a
|
|
16
|
+
import base64, copy, random, re, weakref
|
|
17
|
+
from ast import literal_eval
|
|
17
18
|
from json import loads, dumps
|
|
18
19
|
from fastcore.utils import *
|
|
19
20
|
from fastcore.ansi import strip_ansi
|
|
20
|
-
from fastcore.nbio import mk_cell,preferred_msg_out,concat_streams
|
|
21
|
+
from fastcore.nbio import mk_cell,preferred_msg_out,concat_streams,render_md
|
|
22
|
+
from .msg_parts import strip_tools as _strip_tools
|
|
21
23
|
|
|
22
|
-
# %% ../nbs/
|
|
24
|
+
# %% ../nbs/01_dialog.ipynb #9d79a4b9
|
|
23
25
|
smsg_types = scode,snote,sprompt,sraw = "code","note","prompt","raw"
|
|
24
26
|
|
|
25
|
-
# %% ../nbs/
|
|
27
|
+
# %% ../nbs/01_dialog.ipynb #61883528
|
|
26
28
|
def add_id_hash(cls, attr):
|
|
27
29
|
'Adds `__eq__` and `__hash__` to a cls based on a str or int `attr` id'
|
|
28
30
|
@patch
|
|
@@ -34,14 +36,14 @@ def add_id_hash(cls, attr):
|
|
|
34
36
|
@patch
|
|
35
37
|
def __hash__(self:cls): return hash(getattr(self, attr))
|
|
36
38
|
|
|
37
|
-
# %% ../nbs/
|
|
39
|
+
# %% ../nbs/01_dialog.ipynb #352b878b
|
|
38
40
|
class Msgs(L):
|
|
39
|
-
"A list of `Message`s with a preview
|
|
41
|
+
"A list of `Message`s with a `preview` repr"
|
|
40
42
|
def show(self,
|
|
41
43
|
maxlen:int=120, # Maximum characters per preview line
|
|
42
44
|
rows:int=None, # Max messages to show (None: all)
|
|
43
45
|
):
|
|
44
|
-
"One `preview` line
|
|
46
|
+
"One `preview` per message: a row line, plus a reply line for prompts"
|
|
45
47
|
xs = self if rows is None else self[:rows]
|
|
46
48
|
res = '\n'.join(m.preview(maxlen) for m in xs)
|
|
47
49
|
if rows is not None and len(self)>rows: res += f'\n…({len(self)-rows} more)'
|
|
@@ -49,7 +51,7 @@ class Msgs(L):
|
|
|
49
51
|
|
|
50
52
|
def __repr__(self): return str(self.show(rows=20))
|
|
51
53
|
|
|
52
|
-
# %% ../nbs/
|
|
54
|
+
# %% ../nbs/01_dialog.ipynb #bc320318
|
|
53
55
|
class Dialog(BasicRepr):
|
|
54
56
|
"A named, ordered list of messages"
|
|
55
57
|
def __init__(self,
|
|
@@ -84,7 +86,7 @@ class Dialog(BasicRepr):
|
|
|
84
86
|
|
|
85
87
|
add_id_hash(Dialog, 'name')
|
|
86
88
|
|
|
87
|
-
# %% ../nbs/
|
|
89
|
+
# %% ../nbs/01_dialog.ipynb #579ba71b
|
|
88
90
|
def mk_output(typ, d, meta=None, **kw): return dict(output_type=typ, metadata=meta or {}, data=d, **kw)
|
|
89
91
|
def mk_displayobj(d, meta=None): return mk_output('display_data', d, meta)
|
|
90
92
|
def displayobj(data='', subtype='plain', mimetype='text', meta=None): return mk_displayobj({f'{mimetype}/{subtype}': data}, meta)
|
|
@@ -95,7 +97,7 @@ def prompt_output(txt=''):
|
|
|
95
97
|
"Create prompt output list from AI response text"
|
|
96
98
|
return [displayobj(txt, 'markdown', meta={'is_ai_res': True})] if txt else []
|
|
97
99
|
|
|
98
|
-
# %% ../nbs/
|
|
100
|
+
# %% ../nbs/01_dialog.ipynb #b25cd968
|
|
99
101
|
class Message:
|
|
100
102
|
# Attribute name -> cell metadata key: fields promoted from cell metadata and serialized back (falsy omitted).
|
|
101
103
|
# `skipped` (hidden from AI context) and `pinned` (kept through any context eviction) use solveit's literal keys,
|
|
@@ -177,11 +179,11 @@ class Message:
|
|
|
177
179
|
def summ(self): return truncstr(self.content, 96) + (f" ⇒ {truncstr(str(self.output), 60)}" if self.output else '')
|
|
178
180
|
__repr__ = basic_repr('id,content,output,msg_type')
|
|
179
181
|
|
|
180
|
-
# %% ../nbs/
|
|
182
|
+
# %% ../nbs/01_dialog.ipynb #135858d9
|
|
181
183
|
add_id_hash(Message, 'id')
|
|
182
184
|
Dialog.msg_cls = Message
|
|
183
185
|
|
|
184
|
-
# %% ../nbs/
|
|
186
|
+
# %% ../nbs/01_dialog.ipynb #e3e3ab18
|
|
185
187
|
@patch
|
|
186
188
|
def _repr_markdown_(self:Message):
|
|
187
189
|
detls = '\n'.join(f'- {k}: {getattr(self, k)}' for k in self.flds)
|
|
@@ -193,20 +195,25 @@ def _repr_markdown_(self:Message):
|
|
|
193
195
|
|
|
194
196
|
</details>"""
|
|
195
197
|
|
|
196
|
-
# %% ../nbs/
|
|
198
|
+
# %% ../nbs/01_dialog.ipynb #145c8a42
|
|
199
|
+
def _prev_line(txt, maxlen, pre=''):
|
|
200
|
+
"One escaped preview line, `pre`+`txt` capped at `maxlen`, ending in a humanized `[size]` when `txt` was cut"
|
|
201
|
+
return truncstr((pre+txt).replace('\n', '\\n'), maxlen, suf=f'…[{humanize(len(txt))}]')
|
|
202
|
+
|
|
197
203
|
@patch
|
|
198
204
|
def preview(self:Message,
|
|
199
|
-
maxlen:int=120, # Maximum characters
|
|
205
|
+
maxlen:int=120, # Maximum characters per line
|
|
200
206
|
):
|
|
201
|
-
"Escaped
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
207
|
+
"Escaped summary rows: `id:t:content` (t: c=code n=note p=prompt r=raw), plus a `> ` line for a prompt's reply; a contentless tagged raw shows its `<kind>` instead"
|
|
208
|
+
txt = self.content or (f"<{self.meta['rec_kind']}>" if self.meta.get('rec_kind') else '')
|
|
209
|
+
res = _prev_line(txt, maxlen, f"{self.id}:{self.msg_type[0]}:")
|
|
210
|
+
if self.msg_type==sprompt and self.ai_res: return res + '\n' + _prev_line(self.ai_res, maxlen, '> ')
|
|
211
|
+
if self.output: res += f" ⇒ out({humanize(len(str(self.output)))})"
|
|
212
|
+
return res
|
|
206
213
|
|
|
207
|
-
# %% ../nbs/
|
|
214
|
+
# %% ../nbs/01_dialog.ipynb #a72644ce
|
|
208
215
|
class MsgRow:
|
|
209
|
-
"Snapshot of one message: `id`, `msg_type`, `content`, `out` (reply or output text), and `meta`; shown as its preview
|
|
216
|
+
"Snapshot of one message: `id`, `msg_type`, `content`, `out` (reply or output text), and `meta`; shown as its preview"
|
|
210
217
|
def __init__(self, m, maxlen=120):
|
|
211
218
|
self.id,self.msg_type,self.content = m.id,m.msg_type,m.content
|
|
212
219
|
self.meta = copy.deepcopy(m.meta)
|
|
@@ -214,11 +221,11 @@ class MsgRow:
|
|
|
214
221
|
self._pv = m.preview(maxlen)
|
|
215
222
|
def __repr__(self): return self._pv
|
|
216
223
|
|
|
217
|
-
class MsgRows(
|
|
218
|
-
"A list of `MsgRow`s,
|
|
224
|
+
class MsgRows(L):
|
|
225
|
+
"A list of `MsgRow`s, shown as their previews"
|
|
219
226
|
def __repr__(self): return '\n'.join(repr(o) for o in self)
|
|
220
227
|
|
|
221
|
-
# %% ../nbs/
|
|
228
|
+
# %% ../nbs/01_dialog.ipynb #e4fb159d
|
|
222
229
|
@patch
|
|
223
230
|
def update(self:Message, **kwargs):
|
|
224
231
|
"Update message attributes with provided keyword arguments"
|
|
@@ -226,7 +233,7 @@ def update(self:Message, **kwargs):
|
|
|
226
233
|
if v is not UNSET: setattr(self, k, v)
|
|
227
234
|
return self
|
|
228
235
|
|
|
229
|
-
# %% ../nbs/
|
|
236
|
+
# %% ../nbs/01_dialog.ipynb #a0465cdb
|
|
230
237
|
@patch
|
|
231
238
|
def mk_message(self:Dialog,
|
|
232
239
|
content:str, # Message text
|
|
@@ -250,14 +257,14 @@ def mk_message(self:Dialog,
|
|
|
250
257
|
self.messages.insert(idx, msg)
|
|
251
258
|
return msg
|
|
252
259
|
|
|
253
|
-
# %% ../nbs/
|
|
260
|
+
# %% ../nbs/01_dialog.ipynb #d499654b
|
|
254
261
|
@patch
|
|
255
262
|
def mk_messages(self:Dialog, msgs, after=None, before=None):
|
|
256
263
|
"Make new messages and insert them sequentially into the notebook after/before a specific message."
|
|
257
264
|
def _kws(m): return {o: getattr(m,o) for o in m.flds if o!='id'}
|
|
258
265
|
return [after:=self.mk_message(m.content,output=m.output,attachments=m.attachments,after=after, before=before, **_kws(m)) for m in msgs]
|
|
259
266
|
|
|
260
|
-
# %% ../nbs/
|
|
267
|
+
# %% ../nbs/01_dialog.ipynb #e668fd4f
|
|
261
268
|
@patch
|
|
262
269
|
def insert_after(self:Message, msgs):
|
|
263
270
|
curr = self.dlg.messages
|
|
@@ -265,7 +272,7 @@ def insert_after(self:Message, msgs):
|
|
|
265
272
|
curr[idx+1:idx+1] = msgs
|
|
266
273
|
for m in msgs: m.dlg = self.dlg
|
|
267
274
|
|
|
268
|
-
# %% ../nbs/
|
|
275
|
+
# %% ../nbs/01_dialog.ipynb #0c98e2c1
|
|
269
276
|
@patch
|
|
270
277
|
def _neighbor(self:Message, fwd=True, pred=None):
|
|
271
278
|
"Find next (fwd=True) or previous (fwd=False) message, optionally filtered by pred."
|
|
@@ -281,7 +288,7 @@ def previous(self:Message): return self._neighbor(fwd=False)
|
|
|
281
288
|
@patch
|
|
282
289
|
def rel_msg(self:Message, is_up:bool): return self._neighbor(fwd=not is_up)
|
|
283
290
|
|
|
284
|
-
# %% ../nbs/
|
|
291
|
+
# %% ../nbs/01_dialog.ipynb #991f63bc
|
|
285
292
|
@patch
|
|
286
293
|
def find_msg(self:Dialog, content): return first(m for m in self.messages if m.content==content)
|
|
287
294
|
|
|
@@ -289,7 +296,7 @@ def get_msg(id_, dlg):
|
|
|
289
296
|
if not dlg or not id_: return
|
|
290
297
|
return first(o for o in dlg.messages if o.id==id_)
|
|
291
298
|
|
|
292
|
-
# %% ../nbs/
|
|
299
|
+
# %% ../nbs/01_dialog.ipynb #2b6197c3
|
|
293
300
|
@patch
|
|
294
301
|
def remove_msgs(self:Dialog, msgs):
|
|
295
302
|
"Remove messages from dialog, return removed list"
|
|
@@ -302,7 +309,7 @@ def remove_from_dlg(self:Message):
|
|
|
302
309
|
"Remove this message from its dialog"
|
|
303
310
|
return self.dlg.remove_msgs([self])
|
|
304
311
|
|
|
305
|
-
# %% ../nbs/
|
|
312
|
+
# %% ../nbs/01_dialog.ipynb #03ed3d82
|
|
306
313
|
_re_hdr = re.compile(r' {0,3}(?P<level>#{1,6})\s+(?P<text>.*)')
|
|
307
314
|
|
|
308
315
|
def header_info(content):
|
|
@@ -326,7 +333,7 @@ def section_msgs(ms, head):
|
|
|
326
333
|
res.append(m)
|
|
327
334
|
return Msgs(res)
|
|
328
335
|
|
|
329
|
-
# %% ../nbs/
|
|
336
|
+
# %% ../nbs/01_dialog.ipynb #e074c630
|
|
330
337
|
@patch(as_prop=True)
|
|
331
338
|
def has_error(self:Message):
|
|
332
339
|
if self.msg_type != scode: return False
|
|
@@ -338,7 +345,7 @@ def get_output_mds(output):
|
|
|
338
345
|
return [o['data']['text/markdown'] for o in output
|
|
339
346
|
if o.get('output_type') in ('display_data', 'execute_result') and 'text/markdown' in o.get('data', {})]
|
|
340
347
|
|
|
341
|
-
# %% ../nbs/
|
|
348
|
+
# %% ../nbs/01_dialog.ipynb #26bfd638
|
|
342
349
|
def join_out(d):
|
|
343
350
|
"Join Jupyter's list-of-lines output data into one string"
|
|
344
351
|
return ''.join(d) if isinstance(d, list) else d
|
|
@@ -366,7 +373,7 @@ AI_RENDERERS = { # chkstyle: ignore-node
|
|
|
366
373
|
'application/javascript': lambda d: f'<script>{join_out(d)}</script>',
|
|
367
374
|
}
|
|
368
375
|
|
|
369
|
-
# %% ../nbs/
|
|
376
|
+
# %% ../nbs/01_dialog.ipynb #fe8fc0f9
|
|
370
377
|
def render_output_ai(out, renderers=None, dollars=False):
|
|
371
378
|
"Plain-text rendering of one Jupyter output, as the AI sees it; `renderers` overrides/extends the per-mime table and `dollars` picks `$`-spelled math"
|
|
372
379
|
r = AI_RENDERERS | ({'text/latex': lambda d: normalize_text_latex(join_out(d), dollars=True)} if dollars else {}) | (renderers or {})
|
|
@@ -380,7 +387,42 @@ def render_outputs_ai(outputs, renderers=None, dollars=False):
|
|
|
380
387
|
return ''
|
|
381
388
|
return '\n'.join(render_output_ai(o, renderers=renderers, dollars=dollars) for o in concat_streams(outputs))
|
|
382
389
|
|
|
383
|
-
# %% ../nbs/
|
|
390
|
+
# %% ../nbs/01_dialog.ipynb #027d1774
|
|
391
|
+
def ai_fmt(out, strip_tools=False):
|
|
392
|
+
"Format AI response as it will be attached to the chat history"
|
|
393
|
+
res = _strip_tools(out) if strip_tools else out
|
|
394
|
+
res = res.strip().removesuffix("\n*[Response interrupted]*").strip()
|
|
395
|
+
return res or '<output result="pending" reason="incomplete"/>'
|
|
396
|
+
|
|
397
|
+
# %% ../nbs/01_dialog.ipynb #d42b33a4
|
|
398
|
+
_prims = str, int, float, complex, bool, tuple, list, dict, set, frozenset
|
|
399
|
+
|
|
400
|
+
def try_eval(s, typ:str|None=None):
|
|
401
|
+
"Like `literal_eval`, but wraps in dynamic class named `typ` if succeeds, and returns `s` if fails"
|
|
402
|
+
try:
|
|
403
|
+
res = literal_eval(s)
|
|
404
|
+
if typ and isinstance(res, _prims): res = type(typ, (type(res),), {})(res)
|
|
405
|
+
return res
|
|
406
|
+
except: return s
|
|
407
|
+
|
|
408
|
+
# %% ../nbs/01_dialog.ipynb #37824d36
|
|
409
|
+
Message.ai_renderers = {}
|
|
410
|
+
|
|
411
|
+
@patch
|
|
412
|
+
def render_out(self:Message):
|
|
413
|
+
"Compute and cache the for-AI rendering of this message's output"
|
|
414
|
+
if self._ai_rend is not None: return
|
|
415
|
+
if self.msg_type in (sprompt, scode):
|
|
416
|
+
air = render_outputs_ai(self.output or [], renderers=self.ai_renderers)
|
|
417
|
+
self._ai_rend = ai_fmt(air) if self.msg_type == sprompt else str(try_eval(air))
|
|
418
|
+
else: self._ai_rend = ''
|
|
419
|
+
|
|
420
|
+
@patch(as_prop=True)
|
|
421
|
+
def ai_output(self:Message):
|
|
422
|
+
self.render_out()
|
|
423
|
+
return self._ai_rend
|
|
424
|
+
|
|
425
|
+
# %% ../nbs/01_dialog.ipynb #c09bb8be
|
|
384
426
|
def mk_jmsg(typ, metadata=None, **kwargs):
|
|
385
427
|
"A minimal Jupyter iopub message dict of type `typ`, with `kwargs` as its content"
|
|
386
428
|
kwargs['metadata'] = metadata or {}
|
|
@@ -404,7 +446,7 @@ def output_from_msg(jmsg):
|
|
|
404
446
|
return res
|
|
405
447
|
raise ValueError(f'Unrecognized output msg type: {mt!r}')
|
|
406
448
|
|
|
407
|
-
# %% ../nbs/
|
|
449
|
+
# %% ../nbs/01_dialog.ipynb #eb52bbdb
|
|
408
450
|
@patch
|
|
409
451
|
def clear_output(self:Message, wait=False):
|
|
410
452
|
"Clear outputs now, or on the next `add_output` if `wait`"
|
|
@@ -438,28 +480,31 @@ def add_output(self:Message,
|
|
|
438
480
|
return
|
|
439
481
|
self.output.append(out)
|
|
440
482
|
|
|
441
|
-
# %% ../nbs/
|
|
483
|
+
# %% ../nbs/01_dialog.ipynb #08680316
|
|
442
484
|
@patch(as_prop=True)
|
|
443
485
|
def directives(self:Message):
|
|
444
486
|
"nbdev directives from content comments and the meta `nbdev` dict, comments winning"
|
|
445
487
|
return mk_cell(self.content, metadata=self.meta).directives
|
|
446
488
|
|
|
447
489
|
def _get_exported(self): return 'export' in self.directives or 'exports' in self.directives
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
if
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
490
|
+
Message.exported = property(_get_exported,
|
|
491
|
+
doc="Whether this message carries an nbdev export directive, in content or meta (read-only: edit content, or assign `meta_exported`)")
|
|
492
|
+
|
|
493
|
+
def _get_mexp(self): return any(k in self.meta.get('nbdev',{}) for k in ('export','exports'))
|
|
494
|
+
def _set_mexp(self, v):
|
|
495
|
+
if v: self.meta.setdefault('nbdev',{})['export'] = 'true'
|
|
496
|
+
else:
|
|
497
|
+
nbd = self.meta.get('nbdev',{})
|
|
498
|
+
for k in ('export','exports'): nbd.pop(k, None)
|
|
499
|
+
if not nbd: self.meta.pop('nbdev', None)
|
|
500
|
+
Message.meta_exported = property(_get_mexp, _set_mexp,
|
|
501
|
+
doc="The meta `nbdev` export entry alone - the host-owned switch; assigning writes or clears only meta, never content")
|
|
457
502
|
|
|
458
503
|
def dlg2py(dlg):
|
|
459
504
|
"The exported code of `dlg`, as a python source string"
|
|
460
505
|
return '\n\n'.join(m.content for m in dlg.messages if m.msg_type==scode and m.exported)
|
|
461
506
|
|
|
462
|
-
# %% ../nbs/
|
|
507
|
+
# %% ../nbs/01_dialog.ipynb #0db3221b
|
|
463
508
|
_re_exp = re.compile(r'#\|\s*exports?[^\n]*\n')
|
|
464
509
|
|
|
465
510
|
def copy_export(s, src):
|
|
@@ -467,7 +512,7 @@ def copy_export(s, src):
|
|
|
467
512
|
m = _re_exp.match(src)
|
|
468
513
|
return m.group(0)+s if m and not s.startswith('#|') else s
|
|
469
514
|
|
|
470
|
-
# %% ../nbs/
|
|
515
|
+
# %% ../nbs/01_dialog.ipynb #860657a4
|
|
471
516
|
@patch
|
|
472
517
|
def merge_content(self:Message,
|
|
473
518
|
mtype:str, # The merge target's msg_type
|
|
@@ -479,7 +524,7 @@ def merge_content(self:Message,
|
|
|
479
524
|
if self.msg_type==sraw: return f'```plaintext\n{self.content}\n```'
|
|
480
525
|
return self.content
|
|
481
526
|
|
|
482
|
-
# %% ../nbs/
|
|
527
|
+
# %% ../nbs/01_dialog.ipynb #c201c787
|
|
483
528
|
def merge_metas(metas):
|
|
484
529
|
"First-wins merge of cell metadata dicts, merging the `nbdev` directives dict one level deeper"
|
|
485
530
|
res,nbd = {},{}
|
|
@@ -502,7 +547,7 @@ def merge_parts(msgs, srcs=None):
|
|
|
502
547
|
else: meta.pop('nbdev', None)
|
|
503
548
|
return '\n\n'.join(c.source for c in cells), meta
|
|
504
549
|
|
|
505
|
-
# %% ../nbs/
|
|
550
|
+
# %% ../nbs/01_dialog.ipynb #c872b4ee
|
|
506
551
|
_out_types = {'stream','display_data','execute_result','error'}
|
|
507
552
|
|
|
508
553
|
@patch
|
|
@@ -521,7 +566,7 @@ def validate(self:Message):
|
|
|
521
566
|
except TypeError as e: raise ValueError(f"{self.id}: meta not JSON-serializable: {e}") from None
|
|
522
567
|
return self
|
|
523
568
|
|
|
524
|
-
# %% ../nbs/
|
|
569
|
+
# %% ../nbs/01_dialog.ipynb #d07ccb3d
|
|
525
570
|
@patch
|
|
526
571
|
def validate(self:Dialog):
|
|
527
572
|
"Validate every message plus dialog-scope rules (unique ids, serializable `meta`); returns self if fine"
|
|
@@ -532,7 +577,7 @@ def validate(self:Dialog):
|
|
|
532
577
|
except TypeError as e: raise ValueError(f"dialog meta not JSON-serializable: {e}") from None
|
|
533
578
|
return self
|
|
534
579
|
|
|
535
|
-
# %% ../nbs/
|
|
580
|
+
# %% ../nbs/01_dialog.ipynb #1c03c386
|
|
536
581
|
def ruuid4():
|
|
537
582
|
"Generate a deterministic UUID4-like string using rtoken_hex"
|
|
538
583
|
parts = [rtoken_hex(4), rtoken_hex(2), rtoken_hex(2), rtoken_hex(2), rtoken_hex(6)]
|
|
@@ -551,9 +596,88 @@ def mk_attachment(self:Message, data:bytes, content_type:str, id:str=''):
|
|
|
551
596
|
att = Attachment(data=data, content_type=content_type, id=id)
|
|
552
597
|
return self.attachments.append(att)
|
|
553
598
|
|
|
554
|
-
# %% ../nbs/
|
|
599
|
+
# %% ../nbs/01_dialog.ipynb #18689653
|
|
555
600
|
tiny_png = base64.b64decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==')
|
|
556
601
|
|
|
557
|
-
# %% ../nbs/
|
|
602
|
+
# %% ../nbs/01_dialog.ipynb #036da886
|
|
558
603
|
@patch
|
|
559
604
|
def todict(self:Dialog): return {k:v for k,v in asdict(self).items() if k[0]!='_'}
|
|
605
|
+
|
|
606
|
+
# %% ../nbs/01_dialog.ipynb #63c06529
|
|
607
|
+
@patch
|
|
608
|
+
def todict(self:Message):
|
|
609
|
+
"Public stored fields plus the derived `exported` flag, with `output` as the AI-facing form"
|
|
610
|
+
res = AttrDict({k:v for k,v in asdict(self).items() if k[0]!='_' and k!='attachments'})
|
|
611
|
+
res['output'] = self.ai_output
|
|
612
|
+
res['exported'] = self.exported
|
|
613
|
+
return res
|
|
614
|
+
|
|
615
|
+
# %% ../nbs/01_dialog.ipynb #ca70a93e
|
|
616
|
+
from .msg_parts import tool_info as _tool_info, usage_info as _usage_info
|
|
617
|
+
|
|
618
|
+
def _fmt_param(v, mx=40):
|
|
619
|
+
"Compact display form of one tool arg or result value"
|
|
620
|
+
s = v if isinstance(v, str) else dumps(v, ensure_ascii=False)
|
|
621
|
+
s = s.replace('\n', '\\n')
|
|
622
|
+
if len(s) > mx: s = s[:mx] + '…'
|
|
623
|
+
return dumps(s, ensure_ascii=False)
|
|
624
|
+
|
|
625
|
+
def _code_span(txt):
|
|
626
|
+
"Wrap `txt` in a backtick span longer than any backtick run inside it"
|
|
627
|
+
ticks = '`'*(max(map(len, re.findall('`+', txt)), default=0)+1)
|
|
628
|
+
pad = ' ' if '`' in txt else ''
|
|
629
|
+
return f"{ticks}{pad}{txt}{pad}{ticks}"
|
|
630
|
+
|
|
631
|
+
_code_args = {'py':'code', 'python':'code', 'bash':'cmd'}
|
|
632
|
+
|
|
633
|
+
def tool_md(d):
|
|
634
|
+
"Display markdown for one parsed `{.tool}` block: a folded details div labeled `func(params)→result`, code tools shown as code"
|
|
635
|
+
params = ', '.join(f"{k}={_fmt_param(v)}" for k,v in (d.get('args') or {}).items())
|
|
636
|
+
res = d.get('result')
|
|
637
|
+
tail = f"→{_fmt_param(res)}" if res not in (None, '') else ''
|
|
638
|
+
label = _code_span(f"{d.get('name')}({params}){tail}")
|
|
639
|
+
code = (d.get('args') or {}).get(_code_args.get(d.get('name'), ''))
|
|
640
|
+
if code is not None:
|
|
641
|
+
body = f"Code:\n{fenced(str(code), d.get('name'))}"
|
|
642
|
+
if str(res or '').strip(): body += f"\n\nOutput:\n\n{fenced(str(res))}"
|
|
643
|
+
else: body = fenced(dumps(d, indent=2, ensure_ascii=False), 'json')
|
|
644
|
+
return fenced(f"## {label}\n\n{body}", ' {.details .tool-usage-details}', ch=':')
|
|
645
|
+
|
|
646
|
+
def usage_md(d):
|
|
647
|
+
"Display markdown for one parsed `{.usage}` block: a folded details div labeled with cost or tokens"
|
|
648
|
+
summ = d.get('summary') or (f"${d['cost']:.4f}" if d.get('cost') else f"{d.get('total_tokens', 0):,} tokens")
|
|
649
|
+
det = d.get('detail') or ' | '.join(f"{k}={v:,}" if isinstance(v, int) else f"{k}={v}"
|
|
650
|
+
for k,v in d.items() if v and k != 'model')
|
|
651
|
+
body = f"## {summ}\n\n{_code_span(det)}"
|
|
652
|
+
return fenced(body, ' {.details .token-usage-details}', ch=':')
|
|
653
|
+
|
|
654
|
+
def fmt_tools(s):
|
|
655
|
+
"Rewrite fenced JSON tool/usage wire blocks in `s` into folded `::: details` display markdown"
|
|
656
|
+
out, pos = [], 0
|
|
657
|
+
for info,body,start,end in fenced_blocks(s):
|
|
658
|
+
if info not in (_tool_info, _usage_info): continue
|
|
659
|
+
try: d = loads(body)
|
|
660
|
+
except Exception: continue
|
|
661
|
+
out.append(s[pos:start])
|
|
662
|
+
out.append((tool_md(d) if info == _tool_info else usage_md(d)) + '\n')
|
|
663
|
+
pos = end
|
|
664
|
+
out.append(s[pos:])
|
|
665
|
+
return ''.join(out)
|
|
666
|
+
|
|
667
|
+
# %% ../nbs/01_dialog.ipynb #103d7fb7
|
|
668
|
+
def msg2md(m):
|
|
669
|
+
"One message as Markdown: notes verbatim, code in a ```python fence with outputs in a `::: output` div, prompts and replies in `::: prompt`/`::: reply` divs; None for empty code"
|
|
670
|
+
if m.msg_type == snote: return m.content
|
|
671
|
+
if m.msg_type == sraw: return fenced(m.content)
|
|
672
|
+
if m.msg_type == sprompt:
|
|
673
|
+
parts = [fenced(m.content, ' prompt', ch=':')]
|
|
674
|
+
if (ai := (m.ai_res or '')).strip(): parts.append(fenced(fmt_tools(ai), ' reply', ch=':'))
|
|
675
|
+
return '\n\n'.join(parts)
|
|
676
|
+
if not m.content.strip(): return None
|
|
677
|
+
parts = [fenced(m.content, 'python')]
|
|
678
|
+
if (outs := render_md(m.output or [])): parts.append(fenced(outs, ' output', ch=':'))
|
|
679
|
+
return '\n\n'.join(parts)
|
|
680
|
+
|
|
681
|
+
def dlg2md(d):
|
|
682
|
+
"The dialog as one Markdown document"
|
|
683
|
+
return '\n\n'.join(filter(None, map(msg2md, d))) + '\n'
|
|
@@ -16,8 +16,8 @@ Dropping a level is correct exactly when the question is about the representatio
|
|
|
16
16
|
|
|
17
17
|
The function/method two-shapes contract is `fastcore.editskill`'s, learned once: the function is a transaction addressed by `dlg=` (an ipynb path, or None meaning the current dialog file: `set_dlg`); the method is a session on a held `Dialog` or `Message`, saved by an explicit `dlg.save()` (`msg_str_replace(id, ..., dlg=p)` ⟷ `m.str_replace(...)`). Wrapping any message list in an ephemeral `Dialog(msgs)` makes the whole session surface available on it.
|
|
18
18
|
|
|
19
|
-
- `summary_dlg(dlg)` / `d.summary()`: one preview
|
|
20
|
-
- `find_msgs(pattern, dlg, ...)`: search by regex, type, error state, heading, ids, or a `pred` (`symdef_finder`/`symref_finder`/`ast_finder` build structural ones); `context` defaults to 1 (the neighbouring message usually explains the match). Returns `MsgRow` snapshots (`id`, `msg_type`, `content`, `out`, `meta`), shown as
|
|
19
|
+
- `summary_dlg(dlg)` / `d.summary()`: one `preview` per message, `id:t:content` (t: c=code n=note p=prompt r=raw), with a prompt's reply on a following `> ` line; a line cut short ends in a humanized `[size]`.
|
|
20
|
+
- `find_msgs(pattern, dlg, ...)`: search by regex, type, error state, heading, ids, or a `pred` (`symdef_finder`/`symref_finder`/`ast_finder` build structural ones); `context` defaults to 1 (the neighbouring message usually explains the match). Returns `MsgRow` snapshots (`id`, `msg_type`, `content`, `out`, `meta`), shown as previews. `d.find_msgs(...)` returns live `Message` objects in a `Msgs` list instead, edited directly rather than re-addressed. Every live message carries a `dlg` backref to its owning `Dialog`, so dialog-level operations are always in reach from a message in hand (e.g. `m.dlg.save()` after mutating `m.output` directly).
|
|
21
21
|
- `view_dlg(dlg)` / `d.view()` / `view_msg(id)` / `m.view()` / `view_msgs(*ids)` / `msg2xml(m)` / `m.to_xml()`: full views in the shared `item2xml` grammar (a prompt's reply is its `<out>` section); `incl_out=True` on the line views appends the message's output the same way.
|
|
22
22
|
- Structure: `add_msg`, `del_msgs`, `move_msgs`, `split_msg`, `merge_msgs`, `copy_msgs`/`cut_msgs`/`paste_msgs`, `create_dlg`, with session twins `d.move_msgs`, `m.split`, `d.merge_msgs`, `d.copy_msgs`/`d.cut_msgs`/`d.paste_msgs` (session adds go through `d.mk_message`, deletes through `d.remove_msgs`); the `%%add_msg` magic takes its body verbatim: its line is `%%add_msg [dlg] [msg_type] [before=|after=<id>]`, where a bare path token is the dlg and a bare type name the msg_type, and keyword spellings win over bare tokens.
|
|
23
23
|
- Text edits: `msg_str_replace`, `msg_strs_replace`, `msg_insert_line`, `msg_replace_lines`, `msg_del_lines`, `msg_ast_replace` (all with `re_filter`/line-range powers; `out=True` edits a prompt's reply or a code message's outputs literal), with the same names as `Message` methods for in-memory editing; `lnhashview_msg`/`msg_exhash` (and `m.lnhashview()`/`m.exhash()`) for hash-verified line edits (`lnhashview_msg` is `view_msg(..., lnhashs=True)`; only the exhash pair needs the `exhash` package).
|
|
@@ -102,14 +102,14 @@ def msg(self:Dialog,
|
|
|
102
102
|
def summary(self:Dialog,
|
|
103
103
|
maxlen:int=120, # Maximum characters per line
|
|
104
104
|
):
|
|
105
|
-
"One `preview`
|
|
105
|
+
"One `preview` per message: `id:t:content` (t: c=code n=note p=prompt r=raw), plus a `> ` line for a prompt's reply"
|
|
106
106
|
return self.messages.show(maxlen)
|
|
107
107
|
|
|
108
108
|
def summary_dlg(
|
|
109
109
|
dlg=None, # An ipynb path; the current dialog file if None
|
|
110
110
|
maxlen:int=120, # Maximum characters per line
|
|
111
111
|
):
|
|
112
|
-
"One `preview`
|
|
112
|
+
"One `preview` per message of the dialog file"
|
|
113
113
|
return _to_dlg(dlg).summary(maxlen)
|
|
114
114
|
|
|
115
115
|
# %% ../nbs/03_dlgskill.ipynb #9d26ea0f
|
|
@@ -256,7 +256,7 @@ def find_msgs(
|
|
|
256
256
|
re_pattern:str='', # Regex over content (a prompt's reply included), DOTALL+MULTILINE; an invalid regex matches literally
|
|
257
257
|
dlg=None, # An ipynb path; the current dialog file if None
|
|
258
258
|
**kwargs,
|
|
259
|
-
)->MsgRows: # Snapshot rows (`id`, `msg_type`, `content`, `out`, `meta`), shown as
|
|
259
|
+
)->MsgRows: # Snapshot rows (`id`, `msg_type`, `content`, `out`, `meta`), shown as previews
|
|
260
260
|
"Find messages in the dialog file matching all the given criteria; for live results, call `Dialog.find_msgs`"
|
|
261
261
|
return MsgRows(MsgRow(m) for m in _to_dlg(dlg).find_msgs(re_pattern, **kwargs))
|
|
262
262
|
|
|
@@ -2,13 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
Docs: https://AnswerDotAI.github.io/aidialog/ipynb.html.md"""
|
|
4
4
|
|
|
5
|
-
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/
|
|
5
|
+
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/02_ipynb.ipynb.
|
|
6
6
|
|
|
7
7
|
# %% auto #0
|
|
8
8
|
__all__ = ['reply_sep', 'att2dict', 'split_cell_src', 'get_ipynb', 'write_ipynb', 'ipynb_cells', 'dict2att', 'reads_ipynb',
|
|
9
9
|
'read_ipynb']
|
|
10
10
|
|
|
11
|
-
# %% ../nbs/
|
|
11
|
+
# %% ../nbs/02_ipynb.ipynb #d3d0463b
|
|
12
12
|
from fastcore.utils import *
|
|
13
13
|
from fastcore.xtras import atomic_save
|
|
14
14
|
from fastcore.nbio import mk_cell,new_nb,dict2nb,read_nb,nb2str,repair_cell,repair_nb
|
|
@@ -16,10 +16,10 @@ import json
|
|
|
16
16
|
from base64 import b64encode,b64decode
|
|
17
17
|
from .dialog import *
|
|
18
18
|
|
|
19
|
-
# %% ../nbs/
|
|
19
|
+
# %% ../nbs/02_ipynb.ipynb #aaaeb85a
|
|
20
20
|
def att2dict(att): return {att.content_type: b64encode(att.data).decode('ascii') if isinstance(att.data, bytes) else att.data}
|
|
21
21
|
|
|
22
|
-
# %% ../nbs/
|
|
22
|
+
# %% ../nbs/02_ipynb.ipynb #3fde3c72
|
|
23
23
|
reply_sep = "\n\n##### 🤖Reply🤖<!-- SOLVEIT_SEPARATOR_7f3a9b2c -->\n\n"
|
|
24
24
|
|
|
25
25
|
def split_cell_src(cell):
|
|
@@ -27,14 +27,14 @@ def split_cell_src(cell):
|
|
|
27
27
|
content, *reply = cell.get('source', '').split(reply_sep)
|
|
28
28
|
return content, (reply[0] if reply else None)
|
|
29
29
|
|
|
30
|
-
# %% ../nbs/
|
|
30
|
+
# %% ../nbs/02_ipynb.ipynb #58354f4a
|
|
31
31
|
_out_meta_skip = {'__type'}
|
|
32
32
|
|
|
33
33
|
def _clean_out_meta(o):
|
|
34
34
|
if m := o.get('metadata'): o['metadata'] = {k:v for k,v in m.items() if k not in _out_meta_skip}
|
|
35
35
|
return o
|
|
36
36
|
|
|
37
|
-
# %% ../nbs/
|
|
37
|
+
# %% ../nbs/02_ipynb.ipynb #20f31fdc
|
|
38
38
|
@patch
|
|
39
39
|
def cell_meta(self:Message):
|
|
40
40
|
"Metadata dict to write: `meta` plus demoted `meta_attrs` fields, falsy values omitted"
|
|
@@ -44,7 +44,7 @@ def cell_meta(self:Message):
|
|
|
44
44
|
else: meta.pop(k, None)
|
|
45
45
|
return meta
|
|
46
46
|
|
|
47
|
-
# %% ../nbs/
|
|
47
|
+
# %% ../nbs/02_ipynb.ipynb #0dde746d
|
|
48
48
|
@patch
|
|
49
49
|
def to_cell(self:Message, version=2):
|
|
50
50
|
"Convert message to a notebook cell"
|
|
@@ -65,7 +65,7 @@ def to_cell(self:Message, version=2):
|
|
|
65
65
|
if repairs := repair_cell(cell): print('NB repair:', '; '.join(repairs))
|
|
66
66
|
return cell
|
|
67
67
|
|
|
68
|
-
# %% ../nbs/
|
|
68
|
+
# %% ../nbs/02_ipynb.ipynb #10a73cfc
|
|
69
69
|
def get_ipynb(dlg:Dialog, version=2, msgs=None):
|
|
70
70
|
"Notebook object for `dlg`; `msgs` defaults to all its messages"
|
|
71
71
|
cells = [m.to_cell(version=version) for m in (dlg.messages if msgs is None else msgs)]
|
|
@@ -74,19 +74,19 @@ def get_ipynb(dlg:Dialog, version=2, msgs=None):
|
|
|
74
74
|
return nb
|
|
75
75
|
|
|
76
76
|
|
|
77
|
-
# %% ../nbs/
|
|
77
|
+
# %% ../nbs/02_ipynb.ipynb #ea5fa1dd
|
|
78
78
|
def write_ipynb(dlg:Dialog, fname=None, version=2, msgs=None, **kwargs):
|
|
79
79
|
"Write `dlg` as a notebook, or return the JSON string if `fname` is None; `kwargs` (e.g. `uid`/`gid`) pass to `atomic_save`"
|
|
80
80
|
res = nb2str(get_ipynb(dlg, version=version, msgs=msgs))
|
|
81
81
|
if not fname: return res
|
|
82
82
|
with atomic_save(Path(fname).expanduser(), mode='w', encoding='utf-8', **kwargs) as f: f.write(res)
|
|
83
83
|
|
|
84
|
-
# %% ../nbs/
|
|
84
|
+
# %% ../nbs/02_ipynb.ipynb #5527e596
|
|
85
85
|
@patch
|
|
86
86
|
def write(self:Dialog, base_path, version=2, msgs=None, **kwargs):
|
|
87
87
|
write_ipynb(self, Path(base_path).expanduser()/f'{self.name}.ipynb', version=version, msgs=msgs, **kwargs)
|
|
88
88
|
|
|
89
|
-
# %% ../nbs/
|
|
89
|
+
# %% ../nbs/02_ipynb.ipynb #0e16e93a
|
|
90
90
|
def ipynb_cells(path, nm, prefix=None, suffix=None):
|
|
91
91
|
tmpl = Path(path).expanduser()/f'{nm}.ipynb'
|
|
92
92
|
if not tmpl.exists(): return []
|
|
@@ -95,7 +95,7 @@ def ipynb_cells(path, nm, prefix=None, suffix=None):
|
|
|
95
95
|
if repairs := repair_nb(nb): print('NB repair:', '; '.join(repairs))
|
|
96
96
|
return listify(prefix) + nb.cells + listify(suffix)
|
|
97
97
|
|
|
98
|
-
# %% ../nbs/
|
|
98
|
+
# %% ../nbs/02_ipynb.ipynb #ba0bb61e
|
|
99
99
|
def dict2att(att_id, att_data):
|
|
100
100
|
"Convert attachment dict to Attachment object"
|
|
101
101
|
content_type, data = first(att_data.items())
|
|
@@ -106,7 +106,7 @@ def _output_from_cell(cell):
|
|
|
106
106
|
if cell.cell_type!='code': return ''
|
|
107
107
|
return getattr(cell, 'outputs', [])
|
|
108
108
|
|
|
109
|
-
# %% ../nbs/
|
|
109
|
+
# %% ../nbs/02_ipynb.ipynb #1b43e94f
|
|
110
110
|
@patch
|
|
111
111
|
def cell2msg(self:Dialog, cell):
|
|
112
112
|
"Convert single notebook cell to message object"
|
|
@@ -121,13 +121,13 @@ def cell2msg(self:Dialog, cell):
|
|
|
121
121
|
id = getattr(cell, 'id', None) or rtoken_hex(4)
|
|
122
122
|
return self.msg_cls(content, id=id, output=output, msg_type=msg_type, dlg=self, attachments=atts, meta=meta, **kwargs)
|
|
123
123
|
|
|
124
|
-
# %% ../nbs/
|
|
124
|
+
# %% ../nbs/02_ipynb.ipynb #aef05275
|
|
125
125
|
@patch
|
|
126
126
|
def from_cells(self:Dialog, cells):
|
|
127
127
|
self.messages = Msgs(cells).map(self.cell2msg)
|
|
128
128
|
return self
|
|
129
129
|
|
|
130
|
-
# %% ../nbs/
|
|
130
|
+
# %% ../nbs/02_ipynb.ipynb #6bdaa293
|
|
131
131
|
def reads_ipynb(txt, cls=Dialog, name='dialog'):
|
|
132
132
|
"Read a dialog from notebook JSON string `txt`, constructing via `cls`"
|
|
133
133
|
nb = json.loads(txt)
|
|
@@ -135,7 +135,7 @@ def reads_ipynb(txt, cls=Dialog, name='dialog'):
|
|
|
135
135
|
nb = dict2nb(nb)
|
|
136
136
|
return cls(name=name, meta=dict(nb.get('metadata', {}))).from_cells(nb.cells)
|
|
137
137
|
|
|
138
|
-
# %% ../nbs/
|
|
138
|
+
# %% ../nbs/02_ipynb.ipynb #7e8912e8
|
|
139
139
|
def read_ipynb(fname, cls=Dialog, name=None):
|
|
140
140
|
"Read a dialog from notebook file `fname` (`.ipynb` added if missing), constructing via `cls`; `name` defaults to the file stem"
|
|
141
141
|
f = Path(fname).expanduser()
|
|
@@ -146,7 +146,7 @@ def read_ipynb(fname, cls=Dialog, name=None):
|
|
|
146
146
|
res.path_ = f
|
|
147
147
|
return res
|
|
148
148
|
|
|
149
|
-
# %% ../nbs/
|
|
149
|
+
# %% ../nbs/02_ipynb.ipynb #87786976
|
|
150
150
|
@patch
|
|
151
151
|
def save(self:Dialog, fname=None):
|
|
152
152
|
"Write back to `fname`, or to the `path_` stamped by `read_ipynb`"
|
|
@@ -0,0 +1,340 @@
|
|
|
1
|
+
"""The canonical LLM message model - `Msg`, `Part`, `ToolCall` - and its formatted text form
|
|
2
|
+
|
|
3
|
+
Docs: https://AnswerDotAI.github.io/aidialog/msg_parts.html.md"""
|
|
4
|
+
|
|
5
|
+
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_msg_parts.ipynb.
|
|
6
|
+
|
|
7
|
+
# %% auto #0
|
|
8
|
+
__all__ = ['PartType', 'tool_info', 'usage_info', 'think_start', 'think_end', 're_think', 'Part', 'Msg', 'ToolCall',
|
|
9
|
+
'display_list', 'mk_tool_res_msg', 'sys_text', 'part_txt', 'data_url', 'url_mime', 'MediaUrl', 'parse_tools',
|
|
10
|
+
'strip_tools', 'conv_tools', 'fmt2hist', 'StopResponse', 'FullResponse', 'mk_tr_details', 'hist2fmt']
|
|
11
|
+
|
|
12
|
+
# %% ../nbs/00_msg_parts.ipynb #a616b4f5
|
|
13
|
+
import json
|
|
14
|
+
from json import dumps
|
|
15
|
+
from dataclasses import dataclass, field
|
|
16
|
+
from fastcore.utils import *
|
|
17
|
+
from fastcore.xtras import detect_mime
|
|
18
|
+
|
|
19
|
+
# %% ../nbs/00_msg_parts.ipynb #2212704a
|
|
20
|
+
@dataclass
|
|
21
|
+
class Part:
|
|
22
|
+
"A normalized content part."
|
|
23
|
+
type: str
|
|
24
|
+
text: str = None
|
|
25
|
+
data: dict = None
|
|
26
|
+
|
|
27
|
+
# %% ../nbs/00_msg_parts.ipynb #5342db55
|
|
28
|
+
PartType = str_enum('PartType', 'text', 'thinking', 'refusal', 'tool_use', 'server_tool_result', 'tool_result',
|
|
29
|
+
'input_image', 'input_audio', 'input_video', 'input_file')
|
|
30
|
+
|
|
31
|
+
# %% ../nbs/00_msg_parts.ipynb #026bfff6
|
|
32
|
+
def _trunc_strs(o, n=200):
|
|
33
|
+
"Truncate str or dict"
|
|
34
|
+
if not o: return o
|
|
35
|
+
if isinstance(o,str) and len(o)>n: return o[:100]+'...'
|
|
36
|
+
if isinstance(o,dict): return {k: (v[:100]+'...' if isinstance(v,str) and len(v)>n else v) for k,v in o.items()}
|
|
37
|
+
return o
|
|
38
|
+
|
|
39
|
+
@patch
|
|
40
|
+
def _repr_markdown_(self: Part):
|
|
41
|
+
body = _trunc_strs(self.text) if self.text else ''
|
|
42
|
+
data = _trunc_strs(self.data)
|
|
43
|
+
return f"""**Part** (`{self.type}`)
|
|
44
|
+
|
|
45
|
+
{body}
|
|
46
|
+
|
|
47
|
+
<details markdown='1'>
|
|
48
|
+
|
|
49
|
+
- data: `{data}`
|
|
50
|
+
|
|
51
|
+
</details>"""
|
|
52
|
+
|
|
53
|
+
# %% ../nbs/00_msg_parts.ipynb #ae349c18
|
|
54
|
+
@dataclass
|
|
55
|
+
class Msg:
|
|
56
|
+
"A normalized message."
|
|
57
|
+
role: str
|
|
58
|
+
content: List[Part]
|
|
59
|
+
|
|
60
|
+
@property
|
|
61
|
+
def text(self): return ''.join(p.text or '' for p in self.content if p.type == PartType.text)
|
|
62
|
+
|
|
63
|
+
def _repr_markdown_(self):
|
|
64
|
+
return f"""**Msg**
|
|
65
|
+
|
|
66
|
+
- role: `{self.role}`
|
|
67
|
+
|
|
68
|
+
<contents>
|
|
69
|
+
|
|
70
|
+
{'\n\n'.join(p._repr_markdown_() for p in self.content)}
|
|
71
|
+
|
|
72
|
+
</contents>"""
|
|
73
|
+
|
|
74
|
+
# %% ../nbs/00_msg_parts.ipynb #3cbac5be
|
|
75
|
+
@dataclass
|
|
76
|
+
class ToolCall:
|
|
77
|
+
"Normalized tool call."
|
|
78
|
+
id: str
|
|
79
|
+
name: str
|
|
80
|
+
arguments: dict = field(default_factory=dict)
|
|
81
|
+
server: bool = False
|
|
82
|
+
extra: dict = field(default_factory=dict)
|
|
83
|
+
|
|
84
|
+
# %% ../nbs/00_msg_parts.ipynb #195b4d89
|
|
85
|
+
@patch
|
|
86
|
+
def _repr_markdown_(self: ToolCall):
|
|
87
|
+
extra = _trunc_strs(self.extra)
|
|
88
|
+
return f"""🔧 **{self.name}**(`{self.arguments}`)
|
|
89
|
+
|
|
90
|
+
<details markdown='1'>
|
|
91
|
+
|
|
92
|
+
- id: `{self.id}`
|
|
93
|
+
- server: `{self.server}`
|
|
94
|
+
- extra: `{extra}`
|
|
95
|
+
|
|
96
|
+
</details>"""
|
|
97
|
+
|
|
98
|
+
def display_list(l):
|
|
99
|
+
from IPython.display import Markdown, display
|
|
100
|
+
display(Markdown('\n\n'.join(o._repr_markdown_() for o in l)))
|
|
101
|
+
|
|
102
|
+
# %% ../nbs/00_msg_parts.ipynb #7c88fce1
|
|
103
|
+
def mk_tool_res_msg(tool_calls:list[ToolCall], results:list[str|list]):
|
|
104
|
+
'A util to prepare parallel tool call with str or media list results'
|
|
105
|
+
parts = []
|
|
106
|
+
for tc,res in zip(tool_calls, results):
|
|
107
|
+
data = dict(id=tc.id, name=tc.name, arguments=tc.arguments, server=tc.server)
|
|
108
|
+
parts.append(Part(type=PartType.tool_result, text=res, data=data))
|
|
109
|
+
return Msg(role="tool", content=parts)
|
|
110
|
+
|
|
111
|
+
# %% ../nbs/00_msg_parts.ipynb #07b612b3
|
|
112
|
+
def sys_text(system):
|
|
113
|
+
"Extract text from system (str or Part)."
|
|
114
|
+
if system is None: return None
|
|
115
|
+
return system if isinstance(system, str) else system.text
|
|
116
|
+
|
|
117
|
+
def part_txt(p): return p.text if isinstance(p,Part) else p
|
|
118
|
+
|
|
119
|
+
# %% ../nbs/00_msg_parts.ipynb #f30c7962
|
|
120
|
+
@flexicache(time_policy(24*3600))
|
|
121
|
+
def _fetch_url_partial(url, nbytes=512):
|
|
122
|
+
"Fetch remote media bytes, optionally only first `nbytes`."
|
|
123
|
+
import httpx # deliberately lazy: keeps aidialog's deps to fastcore alone (may re-base on fastcore.net later)
|
|
124
|
+
try:
|
|
125
|
+
with httpx.stream('GET', url, headers={'Range': f'bytes=0-{nbytes-1}'}, follow_redirects=True) as r:
|
|
126
|
+
if r.status_code not in (200, 206): return
|
|
127
|
+
return r.read()
|
|
128
|
+
except (httpx.HTTPError, httpx.InvalidURL): return
|
|
129
|
+
|
|
130
|
+
# %% ../nbs/00_msg_parts.ipynb #b6f61273
|
|
131
|
+
_ext_mime = {
|
|
132
|
+
'.jpg':'image/jpeg', '.jpeg':'image/jpeg', '.png':'image/png', '.gif':'image/gif', '.webp':'image/webp',
|
|
133
|
+
'.pdf':'application/pdf',
|
|
134
|
+
'.mp3':'audio/mpeg', '.wav':'audio/wav', '.ogg':'audio/ogg', '.flac':'audio/flac', '.m4a':'audio/mp4',
|
|
135
|
+
'.mp4':'video/mp4', '.mov':'video/quicktime', '.webm':'video/webm',
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
def data_url(url):
|
|
139
|
+
"Parse data:mime;base64,data URL into (mime, b64_data), or None."
|
|
140
|
+
if not isinstance(url, str) or not url.startswith('data:') or ',' not in url: return None
|
|
141
|
+
header, body = url.split(',', 1)
|
|
142
|
+
if ';base64' not in header or not body: return None
|
|
143
|
+
return header[5:].split(';',1)[0].strip() or 'application/octet-stream', body
|
|
144
|
+
|
|
145
|
+
def url_mime(url, default='application/octet-stream'):
|
|
146
|
+
"Guess mime from URL extension, and optional bytes fallback."
|
|
147
|
+
if "youtube.com" in url or "youtu.be" in url: return "video/mp4"
|
|
148
|
+
ext = '.' + url.rsplit('.', 1)[-1].split('?')[0].lower() if '.' in url.split('?')[0].split('/')[-1] else ''
|
|
149
|
+
if (mime:=_ext_mime.get(ext)) is None: return detect_mime(_fetch_url_partial(url))
|
|
150
|
+
return ifnone(mime, default)
|
|
151
|
+
|
|
152
|
+
# %% ../nbs/00_msg_parts.ipynb #189772f5
|
|
153
|
+
class MediaUrl(BasicRepr):
|
|
154
|
+
"Direct URL media reference"
|
|
155
|
+
def __init__(self, url, mime=None): self.url, self.mime = url, ifnone(mime, url_mime(url))
|
|
156
|
+
|
|
157
|
+
# %% ../nbs/00_msg_parts.ipynb #ee9c825e
|
|
158
|
+
tool_info = 'json {.tool}' # fence info string of a tool block: {id, name, args, result} (+server; `error` reserved)
|
|
159
|
+
usage_info = 'json {.usage}' # fence info string of a usage block: UsageStats fields
|
|
160
|
+
|
|
161
|
+
def parse_tools(s):
|
|
162
|
+
"Split `s` into `(text, data)` segments: `data` is a parsed `{.tool}` block dict, `None` for the final segment"
|
|
163
|
+
res, pos = [], 0
|
|
164
|
+
for info,body,start,end in fenced_blocks(s):
|
|
165
|
+
if info != tool_info: continue
|
|
166
|
+
try: d = json.loads(body)
|
|
167
|
+
except Exception: continue
|
|
168
|
+
res.append((s[pos:start], d))
|
|
169
|
+
pos = end
|
|
170
|
+
return res + [(s[pos:], None)]
|
|
171
|
+
|
|
172
|
+
def strip_tools(s, tools=True, usage=True):
|
|
173
|
+
"Remove `{.tool}` (and `{.usage}`) blocks from `s`"
|
|
174
|
+
out, pos = [], 0
|
|
175
|
+
for info,body,start,end in fenced_blocks(s):
|
|
176
|
+
if not (tools and info == tool_info) and not (usage and info == usage_info): continue
|
|
177
|
+
out.append(s[pos:start])
|
|
178
|
+
pos = end
|
|
179
|
+
out.append(s[pos:])
|
|
180
|
+
return ''.join(out)
|
|
181
|
+
|
|
182
|
+
think_start,think_end = '<!--think_start-->','<!--think_end-->'
|
|
183
|
+
re_think = re.compile(rf'{re.escape(think_start)}.*?{re.escape(think_end)}\n?', re.DOTALL)
|
|
184
|
+
|
|
185
|
+
# Frozen legacy envelope recognition, used only by `conv_tools`. These are the
|
|
186
|
+
# exact patterns fastllm shipped for the released `<details markdown='1'>`
|
|
187
|
+
# envelopes plus the never-released `::: {.details}` spelling.
|
|
188
|
+
_lg_tool_tag = "<details class='tool-usage-details' markdown='1'>"
|
|
189
|
+
_lg_token_tag = "<details class='token-usage-details' markdown='1'>"
|
|
190
|
+
_lg_tool_attrs, _lg_token_attrs = "{.details .tool-usage-details}", "{.details .token-usage-details}"
|
|
191
|
+
_lg_tools = re.compile(
|
|
192
|
+
fr"^(?:{_lg_tool_tag}\n*(?:<summary>(?P<summ1>.*?)</summary>\n*)?\n*```json\n+(?P<json1>.*?)\n+```\n+</details>"
|
|
193
|
+
fr"|(?P<fence>:{{3,}}) {re.escape(_lg_tool_attrs)}\n+(?:## (?P<summ2>.*?)\n+)?```json\n+(?P<json2>.*?)\n+```\n+(?P=fence)$)",
|
|
194
|
+
flags=re.DOTALL|re.MULTILINE)
|
|
195
|
+
_lg_token = re.compile(
|
|
196
|
+
fr"^(?:{re.escape(_lg_token_tag)}\n*<summary>(?P<tsumm1>.*?)</summary>\n*\n*`(?P<trepr1>.*?)`\n*\n*</details>"
|
|
197
|
+
fr"|(?P<tfence>:{{3,}}) {re.escape(_lg_token_attrs)}\n+## (?P<tsumm2>.*?)\n+`(?P<trepr2>.*?)`\n+(?P=tfence)$)\n?",
|
|
198
|
+
flags=re.DOTALL|re.MULTILINE)
|
|
199
|
+
|
|
200
|
+
def conv_tools(s):
|
|
201
|
+
"Convert legacy tool/usage envelopes in `s` (both historical spellings) to the fenced JSON wire format. Idempotent."
|
|
202
|
+
def _tool(m):
|
|
203
|
+
tj = m['json1'] if m['json1'] is not None else m['json2']
|
|
204
|
+
try: d = json.loads(tj.strip())
|
|
205
|
+
except Exception: return m[0]
|
|
206
|
+
call = d.get('call') or {}
|
|
207
|
+
res = dict(id=d.get('id'), name=call.get('function'), args=call.get('arguments') or {}, result=d.get('result'))
|
|
208
|
+
if d.get('server'): res['server'] = True
|
|
209
|
+
return fenced(dumps(res, indent=2, ensure_ascii=False), tool_info)
|
|
210
|
+
def _tok(m):
|
|
211
|
+
summ = m['tsumm1'] if m['tsumm1'] is not None else m['tsumm2']
|
|
212
|
+
det = m['trepr1'] if m['trepr1'] is not None else m['trepr2']
|
|
213
|
+
return fenced(dumps(dict(summary=summ, detail=det), ensure_ascii=False), usage_info)
|
|
214
|
+
return _lg_token.sub(_tok, _lg_tools.sub(_tool, s))
|
|
215
|
+
|
|
216
|
+
# %% ../nbs/00_msg_parts.ipynb #7ae2959f
|
|
217
|
+
_fence_back = '`````'
|
|
218
|
+
_result_re = re.compile(f'\n{_fence_back}result\n(.*?)\n{_fence_back}\n', re.DOTALL)
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
# %% ../nbs/00_msg_parts.ipynb #6d60c135
|
|
222
|
+
def _mk_result_fence(output): return f"\n{_fence_back}result\n{output}\n{_fence_back}\n"
|
|
223
|
+
|
|
224
|
+
def _split_msg_on_fences(msg):
|
|
225
|
+
"Split an assistant Msg on result fences, return list of Msgs"
|
|
226
|
+
if msg.role != 'assistant': return [msg]
|
|
227
|
+
if not _result_re.search(msg.text): return [msg]
|
|
228
|
+
res, asst_parts, tool_parts = [], [], []
|
|
229
|
+
for msg_part in msg.content:
|
|
230
|
+
if msg_part.type == PartType.thinking: asst_parts.append(msg_part)
|
|
231
|
+
elif msg_part.type == PartType.tool_use: tool_parts.append(msg_part)
|
|
232
|
+
elif parts := _result_re.split(msg_part.text or ''):
|
|
233
|
+
for i,p in enumerate(parts):
|
|
234
|
+
if not p: continue
|
|
235
|
+
if i % 2 == 0: res.append(Msg(role='assistant', content=asst_parts+[Part(type=PartType.text, text=p.strip())]))
|
|
236
|
+
else: res.append(Msg(role='user', content=[Part(type=PartType.text, text=_mk_result_fence(p))]))
|
|
237
|
+
if tool_parts: res.append(Msg(role='assistant', content=tool_parts))
|
|
238
|
+
return res
|
|
239
|
+
|
|
240
|
+
def _split_fence_msgs(msgs):
|
|
241
|
+
"Split all assistant msgs on result fences for wire protocol"
|
|
242
|
+
res = []
|
|
243
|
+
for m in msgs: res.extend(_split_msg_on_fences(m))
|
|
244
|
+
return res
|
|
245
|
+
|
|
246
|
+
# %% ../nbs/00_msg_parts.ipynb #ade55a7b
|
|
247
|
+
def _extract_tool_parts(d:dict):
|
|
248
|
+
"Build (tool_use_part, tool_result_part) from a parsed `{.tool}` block"
|
|
249
|
+
# Skip server tool calls in deserialization (round trip issues with Gemini/Anthropic)
|
|
250
|
+
if not d or d.get('server') or d.get('id') is None: return None
|
|
251
|
+
tu = Part(type=PartType.tool_use, text=None, data=dict(id=d['id'], name=d['name'], arguments=d.get('args') or {}))
|
|
252
|
+
tr = Part(type=PartType.tool_result, text=str(d.get('result')), data={'id': d['id'], 'name': d['name']})
|
|
253
|
+
return tu, tr
|
|
254
|
+
|
|
255
|
+
# %% ../nbs/00_msg_parts.ipynb #916f8df0
|
|
256
|
+
def fmt2hist(outp:str)->list[Msg]:
|
|
257
|
+
"Transform a formatted output string into fastllm canonical Msgs"
|
|
258
|
+
if usage_info in outp: outp = strip_tools(outp, tools=False)
|
|
259
|
+
if think_start in outp: outp = re_think.sub('', outp)
|
|
260
|
+
if tool_info not in outp:
|
|
261
|
+
msg = Msg(role='assistant', content=[Part(type=PartType.text, text=outp.strip() or '.')])
|
|
262
|
+
return _split_msg_on_fences(msg)
|
|
263
|
+
hist, asst_parts, tool_parts = [], [], []
|
|
264
|
+
def flush():
|
|
265
|
+
if tool_parts:
|
|
266
|
+
hist.append(Msg(role='assistant', content=asst_parts.copy()))
|
|
267
|
+
hist.append(Msg(role='tool', content=tool_parts.copy()))
|
|
268
|
+
asst_parts.clear()
|
|
269
|
+
tool_parts.clear()
|
|
270
|
+
for txt,d in parse_tools(outp.strip()):
|
|
271
|
+
if txt and txt.strip():
|
|
272
|
+
if tool_parts: flush()
|
|
273
|
+
asst_parts.append(Part(type=PartType.text, text=txt.strip() or '.'))
|
|
274
|
+
if d and (tp := _extract_tool_parts(d)):
|
|
275
|
+
asst_parts.append(tp[0])
|
|
276
|
+
tool_parts.append(tp[1])
|
|
277
|
+
flush()
|
|
278
|
+
if asst_parts: hist.append(Msg(role='assistant', content=asst_parts))
|
|
279
|
+
if not hist: hist.append(Msg(role='assistant', content=[Part(type=PartType.text, text='.')]))
|
|
280
|
+
result = []
|
|
281
|
+
for msg in hist:
|
|
282
|
+
if msg.role == 'assistant': result.extend(_split_msg_on_fences(msg))
|
|
283
|
+
else: result.append(msg)
|
|
284
|
+
if result[-1].role == 'tool': result.append(Msg(role='assistant', content=[Part(type=PartType.text, text='.')]))
|
|
285
|
+
return result
|
|
286
|
+
|
|
287
|
+
# %% ../nbs/00_msg_parts.ipynb #86265805
|
|
288
|
+
class StopResponse(str): pass
|
|
289
|
+
class FullResponse(str): pass
|
|
290
|
+
|
|
291
|
+
# %% ../nbs/00_msg_parts.ipynb #4f105e4d
|
|
292
|
+
def _trunc_str(s, mx=2000, skip=10, replace="TRUNCATED"):
|
|
293
|
+
"Truncate `s` to `mx` chars max, adding `replace` if truncated; `mx=None` disables truncation"
|
|
294
|
+
if mx is None or isinstance_str(s, ('FullResponse','Safe','PrettyString')): return s
|
|
295
|
+
if not isinstance(s, str): s = str(s)
|
|
296
|
+
s = type(s)(s.rstrip())
|
|
297
|
+
if len(s)>2 and s[0]=='𝍁' and s[-1]=='𝍁':
|
|
298
|
+
s = s[1:-1]
|
|
299
|
+
if replace: return s
|
|
300
|
+
if mx is None or len(s)<=mx: return s
|
|
301
|
+
s = s[skip:mx-skip]
|
|
302
|
+
ss = s.split(' ')
|
|
303
|
+
if len(ss[-1])>150: ss[-1] = ss[-1][:5]
|
|
304
|
+
s = ' '.join(ss)
|
|
305
|
+
if skip: s = f"…{s}"
|
|
306
|
+
s = f"{s}…"
|
|
307
|
+
if replace: s = f"<{replace}>{s}</{replace}>"
|
|
308
|
+
return s
|
|
309
|
+
|
|
310
|
+
# %% ../nbs/00_msg_parts.ipynb #3f2a30e7
|
|
311
|
+
def _trunc_content(content, mx):
|
|
312
|
+
"Truncate tool result content, respecting '_full' flag"
|
|
313
|
+
if isinstance(content, dict) and '_full' in content and len(content)==1: return content['_full']
|
|
314
|
+
return _trunc_str(content, mx=mx)
|
|
315
|
+
|
|
316
|
+
# %% ../nbs/00_msg_parts.ipynb #f08c8cbd
|
|
317
|
+
def mk_tr_details(tr, mx=2000):
|
|
318
|
+
"Create the `{.tool}` wire block for a tool call; `mx=None` disables truncation"
|
|
319
|
+
args = {k:_trunc_str(v, mx=None if mx is None else mx*5) if isinstance(v, str) else v for k,v in tr.data['arguments'].items()}
|
|
320
|
+
res = dict(id=tr.data['id'], name=tr.data['name'], args=args, result=_trunc_content(tr.text, mx=mx))
|
|
321
|
+
if tr.data.get('server'): res['server'] = True
|
|
322
|
+
return "\n\n" + fenced(dumps(res, indent=2, ensure_ascii=False), tool_info) + "\n\n"
|
|
323
|
+
|
|
324
|
+
# %% ../nbs/00_msg_parts.ipynb #e5acd16a
|
|
325
|
+
def hist2fmt(msgs:list[Msg], mx=2000)->str:
|
|
326
|
+
"Render assistant/tool `msgs` as one formatted output string, the inverse of `fmt2hist`"
|
|
327
|
+
tus, out = {}, []
|
|
328
|
+
for m in msgs:
|
|
329
|
+
if m.role == 'assistant':
|
|
330
|
+
for p in m.content:
|
|
331
|
+
if p.type == PartType.text and p.text: out.append(p.text.strip())
|
|
332
|
+
elif p.type == PartType.tool_use: tus[p.data['id']] = p
|
|
333
|
+
elif m.role == 'tool':
|
|
334
|
+
for p in m.content:
|
|
335
|
+
if p.type != PartType.tool_result: continue
|
|
336
|
+
tu = tus.get(p.data['id'])
|
|
337
|
+
d = dict(p.data, arguments=tu.data.get('arguments', {}) if tu else {})
|
|
338
|
+
out.append(mk_tr_details(Part(type=PartType.tool_result, text=p.text, data=d), mx=mx).strip())
|
|
339
|
+
else: raise ValueError(f"hist2fmt renders assistant and tool messages only, got {m.role!r}")
|
|
340
|
+
return '\n\n'.join(o for o in out if o)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: aidialog
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.4
|
|
4
4
|
Summary: Read, search, edit, and persist AI dialogs stored as Jupyter notebooks
|
|
5
5
|
Author-email: Jeremy Howard <github@jhoward.fastmail.fm>
|
|
6
6
|
License: Apache-2.0
|
|
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3 :: Only
|
|
|
12
12
|
Requires-Python: >=3.11
|
|
13
13
|
Description-Content-Type: text/markdown
|
|
14
14
|
License-File: LICENSE
|
|
15
|
-
Requires-Dist: fastcore>=2.1.
|
|
15
|
+
Requires-Dist: fastcore>=2.1.13
|
|
16
16
|
Provides-Extra: dev
|
|
17
17
|
Requires-Dist: exhash; extra == "dev"
|
|
18
18
|
Requires-Dist: nbformat; extra == "dev"
|
|
@@ -51,7 +51,7 @@ The [`Dialog`](https://AnswerDotAI.github.io/aidialog/dialog.html#dialog) is the
|
|
|
51
51
|
| ipynb file | storage, pragmatically lossless | [`read_ipynb`](https://AnswerDotAI.github.io/aidialog/ipynb.html#read_ipynb) | [`write_ipynb`](https://AnswerDotAI.github.io/aidialog/ipynb.html#write_ipynb) |
|
|
52
52
|
| Claude Code session | storage | `sess2dlg` | `dlg2sess` |
|
|
53
53
|
| Codex thread | storage (write-only so far) | | `dlg2thread` |
|
|
54
|
-
| fastllm chat (`Msg
|
|
54
|
+
| fastllm chat ([`Msg`](https://AnswerDotAI.github.io/aidialog/msg_parts.html#msg)/[`Part`](https://AnswerDotAI.github.io/aidialog/msg_parts.html#part)) | transmission, normalizing | `chat2dlg` | `dlg2chat` |
|
|
55
55
|
| fastllm hist (live call input) | transmission, one-way | | `dlg2hist` |
|
|
56
56
|
| a prompt’s reply | self-similar | `reply2dlg` | `dlg2reply` |
|
|
57
57
|
| XML views | display, one-way | | [`view_dlg`](https://AnswerDotAI.github.io/aidialog/dlgskill.html#view_dlg), [`msg2xml`](https://AnswerDotAI.github.io/aidialog/dlgskill.html#msg2xml) |
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.0.2"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|