aidialog 0.0.2__tar.gz → 0.0.3__tar.gz

{aidialog-0.0.2/aidialog.egg-info → aidialog-0.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aidialog
-Version: 0.0.2
+Version: 0.0.3
 Summary: Read, search, edit, and persist AI dialogs stored as Jupyter notebooks
 Author-email: Jeremy Howard <github@jhoward.fastmail.fm>
 License: Apache-2.0
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: fastcore>=2.1.9
+Requires-Dist: fastcore>=2.1.12
 Provides-Extra: dev
 Requires-Dist: exhash; extra == "dev"
 Requires-Dist: nbformat; extra == "dev"
@@ -51,7 +51,7 @@ The [`Dialog`](https://AnswerDotAI.github.io/aidialog/dialog.html#dialog) is the
 | ipynb file | storage, pragmatically lossless | [`read_ipynb`](https://AnswerDotAI.github.io/aidialog/ipynb.html#read_ipynb) | [`write_ipynb`](https://AnswerDotAI.github.io/aidialog/ipynb.html#write_ipynb) |
 | Claude Code session | storage | `sess2dlg` | `dlg2sess` |
 | Codex thread | storage (write-only so far) |  | `dlg2thread` |
-| fastllm chat (`Msg`/`Part`) | transmission, normalizing | `chat2dlg` | `dlg2chat` |
+| fastllm chat ([`Msg`](https://AnswerDotAI.github.io/aidialog/msg_parts.html#msg)/[`Part`](https://AnswerDotAI.github.io/aidialog/msg_parts.html#part)) | transmission, normalizing | `chat2dlg` | `dlg2chat` |
 | fastllm hist (live call input) | transmission, one-way |  | `dlg2hist` |
 | a prompt’s reply | self-similar | `reply2dlg` | `dlg2reply` |
 | XML views | display, one-way |  | [`view_dlg`](https://AnswerDotAI.github.io/aidialog/dlgskill.html#view_dlg), [`msg2xml`](https://AnswerDotAI.github.io/aidialog/dlgskill.html#msg2xml) |

{aidialog-0.0.2 → aidialog-0.0.3}/README.md

@@ -31,7 +31,7 @@ The [`Dialog`](https://AnswerDotAI.github.io/aidialog/dialog.html#dialog) is the
 | ipynb file | storage, pragmatically lossless | [`read_ipynb`](https://AnswerDotAI.github.io/aidialog/ipynb.html#read_ipynb) | [`write_ipynb`](https://AnswerDotAI.github.io/aidialog/ipynb.html#write_ipynb) |
 | Claude Code session | storage | `sess2dlg` | `dlg2sess` |
 | Codex thread | storage (write-only so far) |  | `dlg2thread` |
-| fastllm chat (`Msg`/`Part`) | transmission, normalizing | `chat2dlg` | `dlg2chat` |
+| fastllm chat ([`Msg`](https://AnswerDotAI.github.io/aidialog/msg_parts.html#msg)/[`Part`](https://AnswerDotAI.github.io/aidialog/msg_parts.html#part)) | transmission, normalizing | `chat2dlg` | `dlg2chat` |
 | fastllm hist (live call input) | transmission, one-way |  | `dlg2hist` |
 | a prompt’s reply | self-similar | `reply2dlg` | `dlg2reply` |
 | XML views | display, one-way |  | [`view_dlg`](https://AnswerDotAI.github.io/aidialog/dlgskill.html#view_dlg), [`msg2xml`](https://AnswerDotAI.github.io/aidialog/dlgskill.html#msg2xml) |

aidialog-0.0.3/aidialog/__init__.py

@@ -0,0 +1,7 @@
+"""Read, search, edit, and persist AI dialogs stored as Jupyter notebooks
+
+Modules:
+
+- `aidialog.dlgskill`: Read, search, and edit dialogs and notebooks through the aidialog `Dialog`/`Message` model"""
+
+__version__ = "0.0.3"

{aidialog-0.0.2 → aidialog-0.0.3}/aidialog/_modidx.py

@@ -27,6 +27,7 @@ d = { 'settings': { 'branch': 'main',
                                  'aidialog.dialog.Message._neighbor': ('dialog.html#message._neighbor', 'aidialog/dialog.py'),
                                  'aidialog.dialog.Message._repr_markdown_': ('dialog.html#message._repr_markdown_', 'aidialog/dialog.py'),
                                  'aidialog.dialog.Message.add_output': ('dialog.html#message.add_output', 'aidialog/dialog.py'),
+                                 'aidialog.dialog.Message.ai_output': ('dialog.html#message.ai_output', 'aidialog/dialog.py'),
                                  'aidialog.dialog.Message.ai_res': ('dialog.html#message.ai_res', 'aidialog/dialog.py'),
                                  'aidialog.dialog.Message.clear_inp_cache': ('dialog.html#message.clear_inp_cache', 'aidialog/dialog.py'),
                                  'aidialog.dialog.Message.clear_out_cache': ('dialog.html#message.clear_out_cache', 'aidialog/dialog.py'),
@@ -47,7 +48,9 @@ d = { 'settings': { 'branch': 'main',
                                  'aidialog.dialog.Message.previous': ('dialog.html#message.previous', 'aidialog/dialog.py'),
                                  'aidialog.dialog.Message.rel_msg': ('dialog.html#message.rel_msg', 'aidialog/dialog.py'),
                                  'aidialog.dialog.Message.remove_from_dlg': ('dialog.html#message.remove_from_dlg', 'aidialog/dialog.py'),
+                                 'aidialog.dialog.Message.render_out': ('dialog.html#message.render_out', 'aidialog/dialog.py'),
                                  'aidialog.dialog.Message.summ': ('dialog.html#message.summ', 'aidialog/dialog.py'),
+                                 'aidialog.dialog.Message.todict': ('dialog.html#message.todict', 'aidialog/dialog.py'),
                                  'aidialog.dialog.Message.update': ('dialog.html#message.update', 'aidialog/dialog.py'),
                                  'aidialog.dialog.Message.validate': ('dialog.html#message.validate', 'aidialog/dialog.py'),
                                  'aidialog.dialog.MsgRow': ('dialog.html#msgrow', 'aidialog/dialog.py'),
@@ -58,14 +61,19 @@ d = { 'settings': { 'branch': 'main',
                                  'aidialog.dialog.Msgs': ('dialog.html#msgs', 'aidialog/dialog.py'),
                                  'aidialog.dialog.Msgs.__repr__': ('dialog.html#msgs.__repr__', 'aidialog/dialog.py'),
                                  'aidialog.dialog.Msgs.show': ('dialog.html#msgs.show', 'aidialog/dialog.py'),
+                                 'aidialog.dialog._code_span': ('dialog.html#_code_span', 'aidialog/dialog.py'),
+                                 'aidialog.dialog._fmt_param': ('dialog.html#_fmt_param', 'aidialog/dialog.py'),
                                  'aidialog.dialog._get_exported': ('dialog.html#_get_exported', 'aidialog/dialog.py'),
                                  'aidialog.dialog._latex_parts': ('dialog.html#_latex_parts', 'aidialog/dialog.py'),
                                  'aidialog.dialog._set_exported': ('dialog.html#_set_exported', 'aidialog/dialog.py'),
                                  'aidialog.dialog.add_id_hash': ('dialog.html#add_id_hash', 'aidialog/dialog.py'),
+                                 'aidialog.dialog.ai_fmt': ('dialog.html#ai_fmt', 'aidialog/dialog.py'),
                                  'aidialog.dialog.code_output': ('dialog.html#code_output', 'aidialog/dialog.py'),
                                  'aidialog.dialog.copy_export': ('dialog.html#copy_export', 'aidialog/dialog.py'),
                                  'aidialog.dialog.displayobj': ('dialog.html#displayobj', 'aidialog/dialog.py'),
+                                 'aidialog.dialog.dlg2md': ('dialog.html#dlg2md', 'aidialog/dialog.py'),
                                  'aidialog.dialog.dlg2py': ('dialog.html#dlg2py', 'aidialog/dialog.py'),
+                                 'aidialog.dialog.fmt_tools': ('dialog.html#fmt_tools', 'aidialog/dialog.py'),
                                  'aidialog.dialog.get_msg': ('dialog.html#get_msg', 'aidialog/dialog.py'),
                                  'aidialog.dialog.get_output_mds': ('dialog.html#get_output_mds', 'aidialog/dialog.py'),
                                  'aidialog.dialog.header_info': ('dialog.html#header_info', 'aidialog/dialog.py'),
@@ -80,13 +88,17 @@ d = { 'settings': { 'branch': 'main',
                                  'aidialog.dialog.mk_jmsg': ('dialog.html#mk_jmsg', 'aidialog/dialog.py'),
                                  'aidialog.dialog.mk_output': ('dialog.html#mk_output', 'aidialog/dialog.py'),
                                  'aidialog.dialog.mk_stream': ('dialog.html#mk_stream', 'aidialog/dialog.py'),
+                                 'aidialog.dialog.msg2md': ('dialog.html#msg2md', 'aidialog/dialog.py'),
                                  'aidialog.dialog.normalize_text_latex': ('dialog.html#normalize_text_latex', 'aidialog/dialog.py'),
                                  'aidialog.dialog.output_from_msg': ('dialog.html#output_from_msg', 'aidialog/dialog.py'),
                                  'aidialog.dialog.prompt_output': ('dialog.html#prompt_output', 'aidialog/dialog.py'),
                                  'aidialog.dialog.render_output_ai': ('dialog.html#render_output_ai', 'aidialog/dialog.py'),
                                  'aidialog.dialog.render_outputs_ai': ('dialog.html#render_outputs_ai', 'aidialog/dialog.py'),
                                  'aidialog.dialog.ruuid4': ('dialog.html#ruuid4', 'aidialog/dialog.py'),
-                                 'aidialog.dialog.section_msgs': ('dialog.html#section_msgs', 'aidialog/dialog.py')},
+                                 'aidialog.dialog.section_msgs': ('dialog.html#section_msgs', 'aidialog/dialog.py'),
+                                 'aidialog.dialog.tool_md': ('dialog.html#tool_md', 'aidialog/dialog.py'),
+                                 'aidialog.dialog.try_eval': ('dialog.html#try_eval', 'aidialog/dialog.py'),
+                                 'aidialog.dialog.usage_md': ('dialog.html#usage_md', 'aidialog/dialog.py')},
             'aidialog.dlgskill': { 'aidialog.dlgskill.Dialog.copy_msgs': ('dlgskill.html#dialog.copy_msgs', 'aidialog/dlgskill.py'),
                                    'aidialog.dlgskill.Dialog.cut_msgs': ('dlgskill.html#dialog.cut_msgs', 'aidialog/dlgskill.py'),
                                    'aidialog.dlgskill.Dialog.find_msgs': ('dlgskill.html#dialog.find_msgs', 'aidialog/dlgskill.py'),
@@ -150,4 +162,40 @@ d = { 'settings': { 'branch': 'main',
                                 'aidialog.ipynb.read_ipynb': ('ipynb.html#read_ipynb', 'aidialog/ipynb.py'),
                                 'aidialog.ipynb.reads_ipynb': ('ipynb.html#reads_ipynb', 'aidialog/ipynb.py'),
                                 'aidialog.ipynb.split_cell_src': ('ipynb.html#split_cell_src', 'aidialog/ipynb.py'),
-                                'aidialog.ipynb.write_ipynb': ('ipynb.html#write_ipynb', 'aidialog/ipynb.py')}}}
+                                'aidialog.ipynb.write_ipynb': ('ipynb.html#write_ipynb', 'aidialog/ipynb.py')},
+            'aidialog.msg_parts': { 'aidialog.msg_parts.FullResponse': ('msg_parts.html#fullresponse', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.MediaUrl': ('msg_parts.html#mediaurl', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.MediaUrl.__init__': ('msg_parts.html#mediaurl.__init__', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.Msg': ('msg_parts.html#msg', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.Msg._repr_markdown_': ( 'msg_parts.html#msg._repr_markdown_',
+                                                                                'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.Msg.text': ('msg_parts.html#msg.text', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.Part': ('msg_parts.html#part', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.Part._repr_markdown_': ( 'msg_parts.html#part._repr_markdown_',
+                                                                                 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.StopResponse': ('msg_parts.html#stopresponse', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.ToolCall': ('msg_parts.html#toolcall', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.ToolCall._repr_markdown_': ( 'msg_parts.html#toolcall._repr_markdown_',
+                                                                                     'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts._extract_tool_parts': ( 'msg_parts.html#_extract_tool_parts',
+                                                                                'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts._fetch_url_partial': ('msg_parts.html#_fetch_url_partial', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts._mk_result_fence': ('msg_parts.html#_mk_result_fence', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts._split_fence_msgs': ('msg_parts.html#_split_fence_msgs', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts._split_msg_on_fences': ( 'msg_parts.html#_split_msg_on_fences',
+                                                                                 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts._trunc_content': ('msg_parts.html#_trunc_content', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts._trunc_str': ('msg_parts.html#_trunc_str', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts._trunc_strs': ('msg_parts.html#_trunc_strs', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.conv_tools': ('msg_parts.html#conv_tools', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.data_url': ('msg_parts.html#data_url', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.display_list': ('msg_parts.html#display_list', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.fmt2hist': ('msg_parts.html#fmt2hist', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.hist2fmt': ('msg_parts.html#hist2fmt', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.mk_tool_res_msg': ('msg_parts.html#mk_tool_res_msg', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.mk_tr_details': ('msg_parts.html#mk_tr_details', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.parse_tools': ('msg_parts.html#parse_tools', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.part_txt': ('msg_parts.html#part_txt', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.strip_tools': ('msg_parts.html#strip_tools', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.sys_text': ('msg_parts.html#sys_text', 'aidialog/msg_parts.py'),
+                                    'aidialog.msg_parts.url_mime': ('msg_parts.html#url_mime', 'aidialog/msg_parts.py')}}}

{aidialog-0.0.2 → aidialog-0.0.3}/aidialog/dialog.py

@@ -2,27 +2,29 @@
 
 Docs: https://AnswerDotAI.github.io/aidialog/dialog.html.md"""
 
-# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_dialog.ipynb.
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/01_dialog.ipynb.
 
 # %% auto #0
 __all__ = ['smsg_types', 'scode', 'snote', 'sprompt', 'sraw', 'AI_RENDERERS', 'tiny_png', 'add_id_hash', 'Msgs', 'Dialog',
            'mk_output', 'mk_displayobj', 'displayobj', 'mk_code_output', 'code_output', 'prompt_output', 'Message',
            'MsgRow', 'MsgRows', 'get_msg', 'header_info', 'section_msgs', 'get_output_mds', 'join_out',
-           'normalize_text_latex', 'render_output_ai', 'render_outputs_ai', 'mk_jmsg', 'mk_stream', 'mk_error',
-           'mk_dispdata', 'mk_execresult', 'output_from_msg', 'dlg2py', 'copy_export', 'merge_metas', 'merge_parts',
-           'ruuid4', 'Attachment']
+           'normalize_text_latex', 'render_output_ai', 'render_outputs_ai', 'ai_fmt', 'try_eval', 'mk_jmsg',
+           'mk_stream', 'mk_error', 'mk_dispdata', 'mk_execresult', 'output_from_msg', 'dlg2py', 'copy_export',
+           'merge_metas', 'merge_parts', 'ruuid4', 'Attachment', 'tool_md', 'usage_md', 'fmt_tools', 'msg2md', 'dlg2md']
 
-# %% ../nbs/00_dialog.ipynb #5571d07a
-import base64, copy, random, weakref
+# %% ../nbs/01_dialog.ipynb #5571d07a
+import base64, copy, random, re, weakref
+from ast import literal_eval
 from json import loads, dumps
 from fastcore.utils import *
 from fastcore.ansi import strip_ansi
-from fastcore.nbio import mk_cell,preferred_msg_out,concat_streams
+from fastcore.nbio import mk_cell,preferred_msg_out,concat_streams,render_md
+from .msg_parts import strip_tools as _strip_tools
 
-# %% ../nbs/00_dialog.ipynb #9d79a4b9
+# %% ../nbs/01_dialog.ipynb #9d79a4b9
 smsg_types = scode,snote,sprompt,sraw = "code","note","prompt","raw"
 
-# %% ../nbs/00_dialog.ipynb #61883528
+# %% ../nbs/01_dialog.ipynb #61883528
 def add_id_hash(cls, attr):
     'Adds `__eq__` and `__hash__` to a cls based on a str or int `attr` id'
     @patch
@@ -34,7 +36,7 @@ def add_id_hash(cls, attr):
     @patch
     def __hash__(self:cls): return hash(getattr(self, attr))
 
-# %% ../nbs/00_dialog.ipynb #352b878b
+# %% ../nbs/01_dialog.ipynb #352b878b
 class Msgs(L):
     "A list of `Message`s with a preview-per-line repr"
     def show(self,
@@ -49,7 +51,7 @@ class Msgs(L):
 
     def __repr__(self): return str(self.show(rows=20))
 
-# %% ../nbs/00_dialog.ipynb #bc320318
+# %% ../nbs/01_dialog.ipynb #bc320318
 class Dialog(BasicRepr):
     "A named, ordered list of messages"
     def __init__(self,
@@ -84,7 +86,7 @@ class Dialog(BasicRepr):
 
 add_id_hash(Dialog, 'name')
 
-# %% ../nbs/00_dialog.ipynb #579ba71b
+# %% ../nbs/01_dialog.ipynb #579ba71b
 def mk_output(typ, d, meta=None, **kw): return dict(output_type=typ, metadata=meta or {}, data=d, **kw)
 def mk_displayobj(d, meta=None): return mk_output('display_data', d, meta)
 def displayobj(data='', subtype='plain', mimetype='text', meta=None): return mk_displayobj({f'{mimetype}/{subtype}': data}, meta)
@@ -95,7 +97,7 @@ def prompt_output(txt=''):
     "Create prompt output list from AI response text"
     return [displayobj(txt, 'markdown', meta={'is_ai_res': True})] if txt else []
 
-# %% ../nbs/00_dialog.ipynb #b25cd968
+# %% ../nbs/01_dialog.ipynb #b25cd968
 class Message:
     # Attribute name -> cell metadata key: fields promoted from cell metadata and serialized back (falsy omitted).
     # `skipped` (hidden from AI context) and `pinned` (kept through any context eviction) use solveit's literal keys,
@@ -177,11 +179,11 @@ class Message:
     def summ(self): return truncstr(self.content, 96) + (f" ⇒ {truncstr(str(self.output), 60)}" if self.output else '')
     __repr__ = basic_repr('id,content,output,msg_type')
 
-# %% ../nbs/00_dialog.ipynb #135858d9
+# %% ../nbs/01_dialog.ipynb #135858d9
 add_id_hash(Message, 'id')
 Dialog.msg_cls = Message
 
-# %% ../nbs/00_dialog.ipynb #e3e3ab18
+# %% ../nbs/01_dialog.ipynb #e3e3ab18
 @patch
 def _repr_markdown_(self:Message):
     detls = '\n'.join(f'- {k}: {getattr(self, k)}' for k in self.flds)
@@ -193,7 +195,7 @@ def _repr_markdown_(self:Message):
 
 </details>"""
 
-# %% ../nbs/00_dialog.ipynb #145c8a42
+# %% ../nbs/01_dialog.ipynb #145c8a42
 @patch
 def preview(self:Message,
     maxlen:int=120, # Maximum characters in the line
@@ -204,7 +206,7 @@ def preview(self:Message,
     elif self.output: s += f" ⇒ out({len(str(self.output))})"
     return truncstr(s.replace('\n', '\\n'), maxlen)
 
-# %% ../nbs/00_dialog.ipynb #a72644ce
+# %% ../nbs/01_dialog.ipynb #a72644ce
 class MsgRow:
     "Snapshot of one message: `id`, `msg_type`, `content`, `out` (reply or output text), and `meta`; shown as its preview line"
     def __init__(self, m, maxlen=120):
@@ -218,7 +220,7 @@ class MsgRows(list):
     "A list of `MsgRow`s, one preview line each"
     def __repr__(self): return '\n'.join(repr(o) for o in self)
 
-# %% ../nbs/00_dialog.ipynb #e4fb159d
+# %% ../nbs/01_dialog.ipynb #e4fb159d
 @patch
 def update(self:Message, **kwargs):
     "Update message attributes with provided keyword arguments"
@@ -226,7 +228,7 @@ def update(self:Message, **kwargs):
         if v is not UNSET: setattr(self, k, v)
     return self
 
-# %% ../nbs/00_dialog.ipynb #a0465cdb
+# %% ../nbs/01_dialog.ipynb #a0465cdb
 @patch
 def mk_message(self:Dialog,
     content:str, # Message text
@@ -250,14 +252,14 @@ def mk_message(self:Dialog,
     self.messages.insert(idx, msg)
     return msg
 
-# %% ../nbs/00_dialog.ipynb #d499654b
+# %% ../nbs/01_dialog.ipynb #d499654b
 @patch
 def mk_messages(self:Dialog, msgs, after=None, before=None):
     "Make new messages and insert them sequentially into the notebook after/before a specific message."
     def _kws(m): return {o: getattr(m,o) for o in m.flds if o!='id'}
     return [after:=self.mk_message(m.content,output=m.output,attachments=m.attachments,after=after, before=before, **_kws(m)) for m in msgs]
 
-# %% ../nbs/00_dialog.ipynb #e668fd4f
+# %% ../nbs/01_dialog.ipynb #e668fd4f
 @patch
 def insert_after(self:Message, msgs):
     curr = self.dlg.messages
@@ -265,7 +267,7 @@ def insert_after(self:Message, msgs):
     curr[idx+1:idx+1] = msgs
     for m in msgs: m.dlg = self.dlg
 
-# %% ../nbs/00_dialog.ipynb #0c98e2c1
+# %% ../nbs/01_dialog.ipynb #0c98e2c1
 @patch
 def _neighbor(self:Message, fwd=True, pred=None):
     "Find next (fwd=True) or previous (fwd=False) message, optionally filtered by pred."
@@ -281,7 +283,7 @@ def previous(self:Message): return self._neighbor(fwd=False)
 @patch
 def rel_msg(self:Message, is_up:bool): return self._neighbor(fwd=not is_up)
 
-# %% ../nbs/00_dialog.ipynb #991f63bc
+# %% ../nbs/01_dialog.ipynb #991f63bc
 @patch
 def find_msg(self:Dialog, content): return first(m for m in self.messages if m.content==content)
 
@@ -289,7 +291,7 @@ def get_msg(id_, dlg):
     if not dlg or not id_: return
     return first(o for o in dlg.messages if o.id==id_)
 
-# %% ../nbs/00_dialog.ipynb #2b6197c3
+# %% ../nbs/01_dialog.ipynb #2b6197c3
 @patch
 def remove_msgs(self:Dialog, msgs):
     "Remove messages from dialog, return removed list"
@@ -302,7 +304,7 @@ def remove_from_dlg(self:Message):
     "Remove this message from its dialog"
     return self.dlg.remove_msgs([self])
 
-# %% ../nbs/00_dialog.ipynb #03ed3d82
+# %% ../nbs/01_dialog.ipynb #03ed3d82
 _re_hdr = re.compile(r' {0,3}(?P<level>#{1,6})\s+(?P<text>.*)')
 
 def header_info(content):
@@ -326,7 +328,7 @@ def section_msgs(ms, head):
         res.append(m)
     return Msgs(res)
 
-# %% ../nbs/00_dialog.ipynb #e074c630
+# %% ../nbs/01_dialog.ipynb #e074c630
 @patch(as_prop=True)
 def has_error(self:Message):
     if self.msg_type != scode: return False
@@ -338,7 +340,7 @@ def get_output_mds(output):
     return [o['data']['text/markdown'] for o in output
         if o.get('output_type') in ('display_data', 'execute_result') and 'text/markdown' in o.get('data', {})]
 
-# %% ../nbs/00_dialog.ipynb #26bfd638
+# %% ../nbs/01_dialog.ipynb #26bfd638
 def join_out(d):
     "Join Jupyter's list-of-lines output data into one string"
     return ''.join(d) if isinstance(d, list) else d
@@ -366,7 +368,7 @@ AI_RENDERERS = {  # chkstyle: ignore-node
     'application/javascript': lambda d: f'<script>{join_out(d)}</script>',
 }
 
-# %% ../nbs/00_dialog.ipynb #fe8fc0f9
+# %% ../nbs/01_dialog.ipynb #fe8fc0f9
 def render_output_ai(out, renderers=None, dollars=False):
     "Plain-text rendering of one Jupyter output, as the AI sees it; `renderers` overrides/extends the per-mime table and `dollars` picks `$`-spelled math"
     r = AI_RENDERERS | ({'text/latex': lambda d: normalize_text_latex(join_out(d), dollars=True)} if dollars else {}) | (renderers or {})
@@ -380,7 +382,42 @@ def render_outputs_ai(outputs, renderers=None, dollars=False):
         return ''
     return '\n'.join(render_output_ai(o, renderers=renderers, dollars=dollars) for o in concat_streams(outputs))
 
-# %% ../nbs/00_dialog.ipynb #c09bb8be
+# %% ../nbs/01_dialog.ipynb #027d1774
+def ai_fmt(out, strip_tools=False):
+    "Format AI response as it will be attached to the chat history"
+    res = _strip_tools(out) if strip_tools else out
+    res = res.strip().removesuffix("\n*[Response interrupted]*").strip()
+    return res or '<output result="pending" reason="incomplete"/>'
+
+# %% ../nbs/01_dialog.ipynb #d42b33a4
+_prims = str, int, float, complex, bool, tuple, list, dict, set, frozenset
+
+def try_eval(s, typ:str|None=None):
+    "Like `literal_eval`, but wraps in dynamic class named `typ` if succeeds, and returns `s` if fails"
+    try:
+        res = literal_eval(s)
+        if typ and isinstance(res, _prims): res = type(typ, (type(res),), {})(res)
+        return res
+    except: return s
+
+# %% ../nbs/01_dialog.ipynb #37824d36
+Message.ai_renderers = {}
+
+@patch
+def render_out(self:Message):
+    "Compute and cache the for-AI rendering of this message's output"
+    if self._ai_rend is not None: return
+    if self.msg_type in (sprompt, scode):
+        air = render_outputs_ai(self.output or [], renderers=self.ai_renderers)
+        self._ai_rend = ai_fmt(air) if self.msg_type == sprompt else str(try_eval(air))
+    else: self._ai_rend = ''
+
+@patch(as_prop=True)
+def ai_output(self:Message):
+    self.render_out()
+    return self._ai_rend
+
+# %% ../nbs/01_dialog.ipynb #c09bb8be
 def mk_jmsg(typ, metadata=None, **kwargs):
     "A minimal Jupyter iopub message dict of type `typ`, with `kwargs` as its content"
     kwargs['metadata'] = metadata or {}
@@ -404,7 +441,7 @@ def output_from_msg(jmsg):
         return res
     raise ValueError(f'Unrecognized output msg type: {mt!r}')
 
-# %% ../nbs/00_dialog.ipynb #eb52bbdb
+# %% ../nbs/01_dialog.ipynb #eb52bbdb
 @patch
 def clear_output(self:Message, wait=False):
     "Clear outputs now, or on the next `add_output` if `wait`"
@@ -438,7 +475,7 @@ def add_output(self:Message,
                 return
     self.output.append(out)
 
-# %% ../nbs/00_dialog.ipynb #08680316
+# %% ../nbs/01_dialog.ipynb #08680316
 @patch(as_prop=True)
 def directives(self:Message):
     "nbdev directives from content comments and the meta `nbdev` dict, comments winning"
@@ -459,7 +496,7 @@ def dlg2py(dlg):
     "The exported code of `dlg`, as a python source string"
     return '\n\n'.join(m.content for m in dlg.messages if m.msg_type==scode and m.exported)
 
-# %% ../nbs/00_dialog.ipynb #0db3221b
+# %% ../nbs/01_dialog.ipynb #0db3221b
 _re_exp = re.compile(r'#\|\s*exports?[^\n]*\n')
 
 def copy_export(s, src):
@@ -467,7 +504,7 @@ def copy_export(s, src):
     m = _re_exp.match(src)
     return m.group(0)+s if m and not s.startswith('#|') else s
 
-# %% ../nbs/00_dialog.ipynb #860657a4
+# %% ../nbs/01_dialog.ipynb #860657a4
 @patch
 def merge_content(self:Message,
     mtype:str, # The merge target's msg_type
@@ -479,7 +516,7 @@ def merge_content(self:Message,
     if self.msg_type==sraw: return f'```plaintext\n{self.content}\n```'
     return self.content
 
-# %% ../nbs/00_dialog.ipynb #c201c787
+# %% ../nbs/01_dialog.ipynb #c201c787
 def merge_metas(metas):
     "First-wins merge of cell metadata dicts, merging the `nbdev` directives dict one level deeper"
     res,nbd = {},{}
@@ -502,7 +539,7 @@ def merge_parts(msgs, srcs=None):
     else: meta.pop('nbdev', None)
     return '\n\n'.join(c.source for c in cells), meta
 
-# %% ../nbs/00_dialog.ipynb #c872b4ee
+# %% ../nbs/01_dialog.ipynb #c872b4ee
 _out_types = {'stream','display_data','execute_result','error'}
 
 @patch
@@ -521,7 +558,7 @@ def validate(self:Message):
     except TypeError as e: raise ValueError(f"{self.id}: meta not JSON-serializable: {e}") from None
     return self
 
-# %% ../nbs/00_dialog.ipynb #d07ccb3d
+# %% ../nbs/01_dialog.ipynb #d07ccb3d
 @patch
 def validate(self:Dialog):
     "Validate every message plus dialog-scope rules (unique ids, serializable `meta`); returns self if fine"
@@ -532,7 +569,7 @@ def validate(self:Dialog):
     except TypeError as e: raise ValueError(f"dialog meta not JSON-serializable: {e}") from None
     return self
 
-# %% ../nbs/00_dialog.ipynb #1c03c386
+# %% ../nbs/01_dialog.ipynb #1c03c386
 def ruuid4():
     "Generate a deterministic UUID4-like string using rtoken_hex"
     parts = [rtoken_hex(4), rtoken_hex(2), rtoken_hex(2), rtoken_hex(2), rtoken_hex(6)]
@@ -551,9 +588,88 @@ def mk_attachment(self:Message, data:bytes, content_type:str, id:str=''):
     att = Attachment(data=data, content_type=content_type, id=id)
     return self.attachments.append(att)
 
-# %% ../nbs/00_dialog.ipynb #18689653
+# %% ../nbs/01_dialog.ipynb #18689653
 tiny_png = base64.b64decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==')
 
-# %% ../nbs/00_dialog.ipynb #036da886
+# %% ../nbs/01_dialog.ipynb #036da886
 @patch
 def todict(self:Dialog): return {k:v for k,v in asdict(self).items() if k[0]!='_'}
+
+# %% ../nbs/01_dialog.ipynb #63c06529
+@patch
+def todict(self:Message):
+    "Public stored fields plus the derived `exported` flag, with `output` as the AI-facing form"
+    res = AttrDict({k:v for k,v in asdict(self).items() if k[0]!='_' and k!='attachments'})
+    res['output'] = self.ai_output
+    res['exported'] = self.exported
+    return res
+
+# %% ../nbs/01_dialog.ipynb #ca70a93e
+from .msg_parts import tool_info as _tool_info, usage_info as _usage_info
+
+def _fmt_param(v, mx=40):
+    "Compact display form of one tool arg or result value"
+    s = v if isinstance(v, str) else dumps(v, ensure_ascii=False)
+    s = s.replace('\n', '\\n')
+    if len(s) > mx: s = s[:mx] + '…'
+    return dumps(s, ensure_ascii=False)
+
+def _code_span(txt):
+    "Wrap `txt` in a backtick span longer than any backtick run inside it"
+    ticks = '`'*(max(map(len, re.findall('`+', txt)), default=0)+1)
+    pad = ' ' if '`' in txt else ''
+    return f"{ticks}{pad}{txt}{pad}{ticks}"
+
+_code_args = {'py':'code', 'python':'code', 'bash':'cmd'}
+
+def tool_md(d):
+    "Display markdown for one parsed `{.tool}` block: a folded details div labeled `func(params)→result`, code tools shown as code"
+    params = ', '.join(f"{k}={_fmt_param(v)}" for k,v in (d.get('args') or {}).items())
+    res = d.get('result')
+    tail = f"→{_fmt_param(res)}" if res not in (None, '') else ''
+    label = _code_span(f"{d.get('name')}({params}){tail}")
+    code = (d.get('args') or {}).get(_code_args.get(d.get('name'), ''))
+    if code is not None:
+        body = f"Code:\n{fenced(str(code), d.get('name'))}"
+        if str(res or '').strip(): body += f"\n\nOutput:\n\n{fenced(str(res))}"
+    else: body = fenced(dumps(d, indent=2, ensure_ascii=False), 'json')
+    return fenced(f"## {label}\n\n{body}", ' {.details .tool-usage-details}', ch=':')
+
+def usage_md(d):
+    "Display markdown for one parsed `{.usage}` block: a folded details div labeled with cost or tokens"
+    summ = d.get('summary') or (f"${d['cost']:.4f}" if d.get('cost') else f"{d.get('total_tokens', 0):,} tokens")
+    det = d.get('detail') or ' | '.join(f"{k}={v:,}" if isinstance(v, int) else f"{k}={v}"
+                                        for k,v in d.items() if v and k != 'model')
+    body = f"## {summ}\n\n{_code_span(det)}"
+    return fenced(body, ' {.details .token-usage-details}', ch=':')
+
+def fmt_tools(s):
+    "Rewrite fenced JSON tool/usage wire blocks in `s` into folded `::: details` display markdown"
+    out, pos = [], 0
+    for info,body,start,end in fenced_blocks(s):
+        if info not in (_tool_info, _usage_info): continue
+        try: d = loads(body)
+        except Exception: continue
+        out.append(s[pos:start])
+        out.append((tool_md(d) if info == _tool_info else usage_md(d)) + '\n')
+        pos = end
+    out.append(s[pos:])
+    return ''.join(out)
+
+# %% ../nbs/01_dialog.ipynb #103d7fb7
+def msg2md(m):
+    "One message as Markdown: notes verbatim, code in a ```python fence with outputs in a `::: output` div, prompts and replies in `::: prompt`/`::: reply` divs; None for empty code"
+    if m.msg_type == snote: return m.content
+    if m.msg_type == sraw: return fenced(m.content)
+    if m.msg_type == sprompt:
+        parts = [fenced(m.content, ' prompt', ch=':')]
+        if (ai := (m.ai_res or '')).strip(): parts.append(fenced(fmt_tools(ai), ' reply', ch=':'))
+        return '\n\n'.join(parts)
+    if not m.content.strip(): return None
+    parts = [fenced(m.content, 'python')]
+    if (outs := render_md(m.output or [])): parts.append(fenced(outs, ' output', ch=':'))
+    return '\n\n'.join(parts)
+
+def dlg2md(d):
+    "The dialog as one Markdown document"
+    return '\n\n'.join(filter(None, map(msg2md, d))) + '\n'

{aidialog-0.0.2 → aidialog-0.0.3}/aidialog/ipynb.py

@@ -2,13 +2,13 @@
 
 Docs: https://AnswerDotAI.github.io/aidialog/ipynb.html.md"""
 
-# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/01_ipynb.ipynb.
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/02_ipynb.ipynb.
 
 # %% auto #0
 __all__ = ['reply_sep', 'att2dict', 'split_cell_src', 'get_ipynb', 'write_ipynb', 'ipynb_cells', 'dict2att', 'reads_ipynb',
            'read_ipynb']
 
-# %% ../nbs/01_ipynb.ipynb #d3d0463b
+# %% ../nbs/02_ipynb.ipynb #d3d0463b
 from fastcore.utils import *
 from fastcore.xtras import atomic_save
 from fastcore.nbio import mk_cell,new_nb,dict2nb,read_nb,nb2str,repair_cell,repair_nb
@@ -16,10 +16,10 @@ import json
 from base64 import b64encode,b64decode
 from .dialog import *
 
-# %% ../nbs/01_ipynb.ipynb #aaaeb85a
+# %% ../nbs/02_ipynb.ipynb #aaaeb85a
 def att2dict(att): return {att.content_type: b64encode(att.data).decode('ascii') if isinstance(att.data, bytes) else att.data}
 
-# %% ../nbs/01_ipynb.ipynb #3fde3c72
+# %% ../nbs/02_ipynb.ipynb #3fde3c72
 reply_sep = "\n\n##### 🤖Reply🤖<!-- SOLVEIT_SEPARATOR_7f3a9b2c -->\n\n"
 
 def split_cell_src(cell):
@@ -27,14 +27,14 @@ def split_cell_src(cell):
     content, *reply = cell.get('source', '').split(reply_sep)
     return content, (reply[0] if reply else None)
 
-# %% ../nbs/01_ipynb.ipynb #58354f4a
+# %% ../nbs/02_ipynb.ipynb #58354f4a
 _out_meta_skip = {'__type'}
 
 def _clean_out_meta(o):
     if m := o.get('metadata'): o['metadata'] = {k:v for k,v in m.items() if k not in _out_meta_skip}
     return o
 
-# %% ../nbs/01_ipynb.ipynb #20f31fdc
+# %% ../nbs/02_ipynb.ipynb #20f31fdc
 @patch
 def cell_meta(self:Message):
     "Metadata dict to write: `meta` plus demoted `meta_attrs` fields, falsy values omitted"
@@ -44,7 +44,7 @@ def cell_meta(self:Message):
         else: meta.pop(k, None)
     return meta
 
-# %% ../nbs/01_ipynb.ipynb #0dde746d
+# %% ../nbs/02_ipynb.ipynb #0dde746d
 @patch
 def to_cell(self:Message, version=2):
     "Convert message to a notebook cell"
@@ -65,7 +65,7 @@ def to_cell(self:Message, version=2):
     if repairs := repair_cell(cell): print('NB repair:', '; '.join(repairs))
     return cell
 
-# %% ../nbs/01_ipynb.ipynb #10a73cfc
+# %% ../nbs/02_ipynb.ipynb #10a73cfc
 def get_ipynb(dlg:Dialog, version=2, msgs=None):
     "Notebook object for `dlg`; `msgs` defaults to all its messages"
     cells = [m.to_cell(version=version) for m in (dlg.messages if msgs is None else msgs)]
@@ -74,19 +74,19 @@ def get_ipynb(dlg:Dialog, version=2, msgs=None):
     return nb
 
 
-# %% ../nbs/01_ipynb.ipynb #ea5fa1dd
+# %% ../nbs/02_ipynb.ipynb #ea5fa1dd
 def write_ipynb(dlg:Dialog, fname=None, version=2, msgs=None, **kwargs):
     "Write `dlg` as a notebook, or return the JSON string if `fname` is None; `kwargs` (e.g. `uid`/`gid`) pass to `atomic_save`"
     res = nb2str(get_ipynb(dlg, version=version, msgs=msgs))
     if not fname: return res
     with atomic_save(Path(fname).expanduser(), mode='w', encoding='utf-8', **kwargs) as f: f.write(res)
 
-# %% ../nbs/01_ipynb.ipynb #5527e596
+# %% ../nbs/02_ipynb.ipynb #5527e596
 @patch
 def write(self:Dialog, base_path, version=2, msgs=None, **kwargs):
     write_ipynb(self, Path(base_path).expanduser()/f'{self.name}.ipynb', version=version, msgs=msgs, **kwargs)
 
-# %% ../nbs/01_ipynb.ipynb #0e16e93a
+# %% ../nbs/02_ipynb.ipynb #0e16e93a
 def ipynb_cells(path, nm, prefix=None, suffix=None):
     tmpl = Path(path).expanduser()/f'{nm}.ipynb'
     if not tmpl.exists(): return []
@@ -95,7 +95,7 @@ def ipynb_cells(path, nm, prefix=None, suffix=None):
     if repairs := repair_nb(nb): print('NB repair:', '; '.join(repairs))
     return listify(prefix) + nb.cells + listify(suffix)
 
-# %% ../nbs/01_ipynb.ipynb #ba0bb61e
+# %% ../nbs/02_ipynb.ipynb #ba0bb61e
 def dict2att(att_id, att_data):
     "Convert attachment dict to Attachment object"
     content_type, data = first(att_data.items())
@@ -106,7 +106,7 @@ def _output_from_cell(cell):
     if cell.cell_type!='code': return ''
     return getattr(cell, 'outputs', [])
 
-# %% ../nbs/01_ipynb.ipynb #1b43e94f
+# %% ../nbs/02_ipynb.ipynb #1b43e94f
 @patch
 def cell2msg(self:Dialog, cell):
     "Convert single notebook cell to message object"
@@ -121,13 +121,13 @@ def cell2msg(self:Dialog, cell):
     id = getattr(cell, 'id', None) or rtoken_hex(4)
     return self.msg_cls(content, id=id, output=output, msg_type=msg_type, dlg=self, attachments=atts, meta=meta, **kwargs)
 
-# %% ../nbs/01_ipynb.ipynb #aef05275
+# %% ../nbs/02_ipynb.ipynb #aef05275
 @patch
 def from_cells(self:Dialog, cells):
     self.messages = Msgs(cells).map(self.cell2msg)
     return self
 
-# %% ../nbs/01_ipynb.ipynb #6bdaa293
+# %% ../nbs/02_ipynb.ipynb #6bdaa293
 def reads_ipynb(txt, cls=Dialog, name='dialog'):
     "Read a dialog from notebook JSON string `txt`, constructing via `cls`"
     nb = json.loads(txt)
@@ -135,7 +135,7 @@ def reads_ipynb(txt, cls=Dialog, name='dialog'):
     nb = dict2nb(nb)
     return cls(name=name, meta=dict(nb.get('metadata', {}))).from_cells(nb.cells)
 
-# %% ../nbs/01_ipynb.ipynb #7e8912e8
+# %% ../nbs/02_ipynb.ipynb #7e8912e8
 def read_ipynb(fname, cls=Dialog, name=None):
     "Read a dialog from notebook file `fname` (`.ipynb` added if missing), constructing via `cls`; `name` defaults to the file stem"
     f = Path(fname).expanduser()
@@ -146,7 +146,7 @@ def read_ipynb(fname, cls=Dialog, name=None):
     res.path_ = f
     return res
 
-# %% ../nbs/01_ipynb.ipynb #87786976
+# %% ../nbs/02_ipynb.ipynb #87786976
 @patch
 def save(self:Dialog, fname=None):
     "Write back to `fname`, or to the `path_` stamped by `read_ipynb`"

aidialog-0.0.3/aidialog/msg_parts.py

@@ -0,0 +1,340 @@
+"""The canonical LLM message model - `Msg`, `Part`, `ToolCall` - and its formatted text form
+
+Docs: https://AnswerDotAI.github.io/aidialog/msg_parts.html.md"""
+
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_msg_parts.ipynb.
+
+# %% auto #0
+__all__ = ['PartType', 'tool_info', 'usage_info', 'think_start', 'think_end', 're_think', 'Part', 'Msg', 'ToolCall',
+           'display_list', 'mk_tool_res_msg', 'sys_text', 'part_txt', 'data_url', 'url_mime', 'MediaUrl', 'parse_tools',
+           'strip_tools', 'conv_tools', 'fmt2hist', 'StopResponse', 'FullResponse', 'mk_tr_details', 'hist2fmt']
+
+# %% ../nbs/00_msg_parts.ipynb #a616b4f5
+import json
+from json import dumps
+from dataclasses import dataclass, field
+from fastcore.utils import *
+from fastcore.xtras import detect_mime
+
+# %% ../nbs/00_msg_parts.ipynb #2212704a
+@dataclass
+class Part:
+    "A normalized content part."
+    type: str
+    text: str = None
+    data: dict = None
+
+# %% ../nbs/00_msg_parts.ipynb #5342db55
+PartType = str_enum('PartType', 'text', 'thinking', 'refusal', 'tool_use', 'server_tool_result', 'tool_result',
+                    'input_image', 'input_audio', 'input_video', 'input_file')
+
+# %% ../nbs/00_msg_parts.ipynb #026bfff6
+def _trunc_strs(o, n=200):
+    "Truncate str or dict"
+    if not o: return o
+    if isinstance(o,str) and len(o)>n: return o[:100]+'...'
+    if isinstance(o,dict): return {k: (v[:100]+'...' if isinstance(v,str) and len(v)>n else v) for k,v in o.items()}
+    return o
+
+@patch
+def _repr_markdown_(self: Part):
+    body = _trunc_strs(self.text) if self.text else ''
+    data = _trunc_strs(self.data)
+    return f"""**Part** (`{self.type}`)
+
+{body}
+
+<details markdown='1'>
+
+- data: `{data}`
+
+</details>"""
+
+# %% ../nbs/00_msg_parts.ipynb #ae349c18
+@dataclass
+class Msg:
+    "A normalized message."
+    role: str
+    content: List[Part]
+
+    @property
+    def text(self): return ''.join(p.text or '' for p in self.content if p.type == PartType.text)
+
+    def _repr_markdown_(self):
+        return f"""**Msg**
+
+- role: `{self.role}`
+
+<contents>
+
+{'\n\n'.join(p._repr_markdown_() for p in self.content)}
+
+</contents>"""
+
+# %% ../nbs/00_msg_parts.ipynb #3cbac5be
+@dataclass
+class ToolCall:
+    "Normalized tool call."
+    id: str
+    name: str
+    arguments: dict = field(default_factory=dict)
+    server: bool = False
+    extra: dict = field(default_factory=dict)
+
+# %% ../nbs/00_msg_parts.ipynb #195b4d89
+@patch
+def _repr_markdown_(self: ToolCall):
+    extra = _trunc_strs(self.extra)
+    return f"""🔧 **{self.name}**(`{self.arguments}`)
+
+<details markdown='1'>
+
+- id: `{self.id}`
+- server: `{self.server}`
+- extra: `{extra}`
+
+</details>"""
+
+def display_list(l): 
+    from IPython.display import Markdown, display
+    display(Markdown('\n\n'.join(o._repr_markdown_() for o in l)))
+
+# %% ../nbs/00_msg_parts.ipynb #7c88fce1
+def mk_tool_res_msg(tool_calls:list[ToolCall], results:list[str|list]):
+    'A util to prepare parallel tool call with str or media list results'
+    parts = []
+    for tc,res in zip(tool_calls, results):
+        data = dict(id=tc.id, name=tc.name, arguments=tc.arguments, server=tc.server)
+        parts.append(Part(type=PartType.tool_result, text=res, data=data))
+    return Msg(role="tool", content=parts)
+
+# %% ../nbs/00_msg_parts.ipynb #07b612b3
+def sys_text(system):
+    "Extract text from system (str or Part)."
+    if system is None: return None
+    return system if isinstance(system, str) else system.text
+
+def part_txt(p): return p.text if isinstance(p,Part) else p
+
+# %% ../nbs/00_msg_parts.ipynb #f30c7962
+@flexicache(time_policy(24*3600))
+def _fetch_url_partial(url, nbytes=512): 
+    "Fetch remote media bytes, optionally only first `nbytes`."
+    import httpx  # deliberately lazy: keeps aidialog's deps to fastcore alone (may re-base on fastcore.net later)
+    try:
+        with httpx.stream('GET', url, headers={'Range': f'bytes=0-{nbytes-1}'}, follow_redirects=True) as r:
+            if r.status_code not in (200, 206): return
+            return r.read()
+    except (httpx.HTTPError, httpx.InvalidURL): return
+
+# %% ../nbs/00_msg_parts.ipynb #b6f61273
+_ext_mime = {
+    '.jpg':'image/jpeg', '.jpeg':'image/jpeg', '.png':'image/png', '.gif':'image/gif', '.webp':'image/webp',
+    '.pdf':'application/pdf',
+    '.mp3':'audio/mpeg', '.wav':'audio/wav', '.ogg':'audio/ogg', '.flac':'audio/flac', '.m4a':'audio/mp4',
+    '.mp4':'video/mp4', '.mov':'video/quicktime', '.webm':'video/webm',
+}
+
+def data_url(url):
+    "Parse data:mime;base64,data URL into (mime, b64_data), or None."
+    if not isinstance(url, str) or not url.startswith('data:') or ',' not in url: return None
+    header, body = url.split(',', 1)
+    if ';base64' not in header or not body: return None
+    return header[5:].split(';',1)[0].strip() or 'application/octet-stream', body
+
+def url_mime(url, default='application/octet-stream'):
+    "Guess mime from URL extension, and optional bytes fallback."
+    if "youtube.com" in url or "youtu.be" in url: return "video/mp4"
+    ext = '.' + url.rsplit('.', 1)[-1].split('?')[0].lower() if '.' in url.split('?')[0].split('/')[-1] else ''
+    if (mime:=_ext_mime.get(ext)) is None: return detect_mime(_fetch_url_partial(url))
+    return ifnone(mime, default)
+
+# %% ../nbs/00_msg_parts.ipynb #189772f5
+class MediaUrl(BasicRepr):
+    "Direct URL media reference"
+    def __init__(self, url, mime=None): self.url, self.mime = url, ifnone(mime, url_mime(url))
+
+# %% ../nbs/00_msg_parts.ipynb #ee9c825e
+tool_info = 'json {.tool}'     # fence info string of a tool block: {id, name, args, result} (+server; `error` reserved)
+usage_info = 'json {.usage}'   # fence info string of a usage block: UsageStats fields
+
+def parse_tools(s):
+    "Split `s` into `(text, data)` segments: `data` is a parsed `{.tool}` block dict, `None` for the final segment"
+    res, pos = [], 0
+    for info,body,start,end in fenced_blocks(s):
+        if info != tool_info: continue
+        try: d = json.loads(body)
+        except Exception: continue
+        res.append((s[pos:start], d))
+        pos = end
+    return res + [(s[pos:], None)]
+
+def strip_tools(s, tools=True, usage=True):
+    "Remove `{.tool}` (and `{.usage}`) blocks from `s`"
+    out, pos = [], 0
+    for info,body,start,end in fenced_blocks(s):
+        if not (tools and info == tool_info) and not (usage and info == usage_info): continue
+        out.append(s[pos:start])
+        pos = end
+    out.append(s[pos:])
+    return ''.join(out)
+
+think_start,think_end = '<!--think_start-->','<!--think_end-->'
+re_think = re.compile(rf'{re.escape(think_start)}.*?{re.escape(think_end)}\n?', re.DOTALL)
+
+# Frozen legacy envelope recognition, used only by `conv_tools`. These are the
+# exact patterns fastllm shipped for the released `<details markdown='1'>`
+# envelopes plus the never-released `::: {.details}` spelling.
+_lg_tool_tag = "<details class='tool-usage-details' markdown='1'>"
+_lg_token_tag = "<details class='token-usage-details' markdown='1'>"
+_lg_tool_attrs, _lg_token_attrs = "{.details .tool-usage-details}", "{.details .token-usage-details}"
+_lg_tools = re.compile(
+    fr"^(?:{_lg_tool_tag}\n*(?:<summary>(?P<summ1>.*?)</summary>\n*)?\n*```json\n+(?P<json1>.*?)\n+```\n+</details>"
+    fr"|(?P<fence>:{{3,}}) {re.escape(_lg_tool_attrs)}\n+(?:## (?P<summ2>.*?)\n+)?```json\n+(?P<json2>.*?)\n+```\n+(?P=fence)$)",
+    flags=re.DOTALL|re.MULTILINE)
+_lg_token = re.compile(
+    fr"^(?:{re.escape(_lg_token_tag)}\n*<summary>(?P<tsumm1>.*?)</summary>\n*\n*`(?P<trepr1>.*?)`\n*\n*</details>"
+    fr"|(?P<tfence>:{{3,}}) {re.escape(_lg_token_attrs)}\n+## (?P<tsumm2>.*?)\n+`(?P<trepr2>.*?)`\n+(?P=tfence)$)\n?",
+    flags=re.DOTALL|re.MULTILINE)
+
+def conv_tools(s):
+    "Convert legacy tool/usage envelopes in `s` (both historical spellings) to the fenced JSON wire format. Idempotent."
+    def _tool(m):
+        tj = m['json1'] if m['json1'] is not None else m['json2']
+        try: d = json.loads(tj.strip())
+        except Exception: return m[0]
+        call = d.get('call') or {}
+        res = dict(id=d.get('id'), name=call.get('function'), args=call.get('arguments') or {}, result=d.get('result'))
+        if d.get('server'): res['server'] = True
+        return fenced(dumps(res, indent=2, ensure_ascii=False), tool_info)
+    def _tok(m):
+        summ = m['tsumm1'] if m['tsumm1'] is not None else m['tsumm2']
+        det = m['trepr1'] if m['trepr1'] is not None else m['trepr2']
+        return fenced(dumps(dict(summary=summ, detail=det), ensure_ascii=False), usage_info)
+    return _lg_token.sub(_tok, _lg_tools.sub(_tool, s))
+
+# %% ../nbs/00_msg_parts.ipynb #7ae2959f
+_fence_back = '`````'
+_result_re = re.compile(f'\n{_fence_back}result\n(.*?)\n{_fence_back}\n', re.DOTALL)
+
+
+# %% ../nbs/00_msg_parts.ipynb #6d60c135
+def _mk_result_fence(output): return f"\n{_fence_back}result\n{output}\n{_fence_back}\n"
+
+def _split_msg_on_fences(msg):
+    "Split an assistant Msg on result fences, return list of Msgs"
+    if msg.role != 'assistant': return [msg]
+    if not _result_re.search(msg.text): return [msg]
+    res, asst_parts, tool_parts = [], [], []
+    for msg_part in msg.content:
+        if msg_part.type == PartType.thinking: asst_parts.append(msg_part)
+        elif msg_part.type == PartType.tool_use: tool_parts.append(msg_part)
+        elif parts := _result_re.split(msg_part.text or ''):
+            for i,p in enumerate(parts):
+                if not p: continue
+                if i % 2 == 0: res.append(Msg(role='assistant', content=asst_parts+[Part(type=PartType.text, text=p.strip())]))
+                else:          res.append(Msg(role='user', content=[Part(type=PartType.text, text=_mk_result_fence(p))]))
+    if tool_parts: res.append(Msg(role='assistant', content=tool_parts))
+    return res
+
+def _split_fence_msgs(msgs):
+    "Split all assistant msgs on result fences for wire protocol"
+    res = []
+    for m in msgs: res.extend(_split_msg_on_fences(m))
+    return res
+
+# %% ../nbs/00_msg_parts.ipynb #ade55a7b
+def _extract_tool_parts(d:dict):
+    "Build (tool_use_part, tool_result_part) from a parsed `{.tool}` block"
+    # Skip server tool calls in deserialization (round trip issues with Gemini/Anthropic)
+    if not d or d.get('server') or d.get('id') is None: return None
+    tu = Part(type=PartType.tool_use, text=None, data=dict(id=d['id'], name=d['name'], arguments=d.get('args') or {}))
+    tr = Part(type=PartType.tool_result, text=str(d.get('result')), data={'id': d['id'], 'name': d['name']})
+    return tu, tr
+
+# %% ../nbs/00_msg_parts.ipynb #916f8df0
+def fmt2hist(outp:str)->list[Msg]:
+    "Transform a formatted output string into fastllm canonical Msgs"
+    if usage_info in outp: outp = strip_tools(outp, tools=False)
+    if think_start in outp: outp = re_think.sub('', outp)
+    if tool_info not in outp:
+        msg = Msg(role='assistant', content=[Part(type=PartType.text, text=outp.strip() or '.')])
+        return _split_msg_on_fences(msg)
+    hist, asst_parts, tool_parts = [], [], []
+    def flush():
+        if tool_parts:
+            hist.append(Msg(role='assistant', content=asst_parts.copy()))
+            hist.append(Msg(role='tool',      content=tool_parts.copy()))
+            asst_parts.clear()
+            tool_parts.clear()
+    for txt,d in parse_tools(outp.strip()):
+        if txt and txt.strip():
+            if tool_parts: flush()
+            asst_parts.append(Part(type=PartType.text, text=txt.strip() or '.'))
+        if d and (tp := _extract_tool_parts(d)):
+            asst_parts.append(tp[0])
+            tool_parts.append(tp[1])
+    flush()
+    if asst_parts: hist.append(Msg(role='assistant', content=asst_parts))
+    if not hist: hist.append(Msg(role='assistant', content=[Part(type=PartType.text, text='.')]))
+    result = []
+    for msg in hist:
+        if msg.role == 'assistant': result.extend(_split_msg_on_fences(msg))
+        else: result.append(msg)
+    if result[-1].role == 'tool': result.append(Msg(role='assistant', content=[Part(type=PartType.text, text='.')])) 
+    return result
+
+# %% ../nbs/00_msg_parts.ipynb #86265805
+class StopResponse(str): pass
+class FullResponse(str): pass
+
+# %% ../nbs/00_msg_parts.ipynb #4f105e4d
+def _trunc_str(s, mx=2000, skip=10, replace="TRUNCATED"):
+    "Truncate `s` to `mx` chars max, adding `replace` if truncated; `mx=None` disables truncation"
+    if mx is None or isinstance_str(s, ('FullResponse','Safe','PrettyString')): return s
+    if not isinstance(s, str): s = str(s)
+    s = type(s)(s.rstrip())
+    if len(s)>2 and s[0]=='𝍁' and s[-1]=='𝍁':
+        s = s[1:-1]
+        if replace: return s
+    if mx is None or len(s)<=mx: return s
+    s = s[skip:mx-skip]
+    ss = s.split(' ')
+    if len(ss[-1])>150: ss[-1] = ss[-1][:5]
+    s = ' '.join(ss)
+    if skip: s = f"…{s}"
+    s = f"{s}…"
+    if replace: s = f"<{replace}>{s}</{replace}>"
+    return s
+
+# %% ../nbs/00_msg_parts.ipynb #3f2a30e7
+def _trunc_content(content, mx):
+    "Truncate tool result content, respecting '_full' flag"
+    if isinstance(content, dict) and '_full' in content and len(content)==1: return content['_full']
+    return _trunc_str(content, mx=mx)
+
+# %% ../nbs/00_msg_parts.ipynb #f08c8cbd
+def mk_tr_details(tr, mx=2000):
+    "Create the `{.tool}` wire block for a tool call; `mx=None` disables truncation"
+    args = {k:_trunc_str(v, mx=None if mx is None else mx*5) if isinstance(v, str) else v for k,v in tr.data['arguments'].items()}
+    res = dict(id=tr.data['id'], name=tr.data['name'], args=args, result=_trunc_content(tr.text, mx=mx))
+    if tr.data.get('server'): res['server'] = True
+    return "\n\n" + fenced(dumps(res, indent=2, ensure_ascii=False), tool_info) + "\n\n"
+
+# %% ../nbs/00_msg_parts.ipynb #e5acd16a
+def hist2fmt(msgs:list[Msg], mx=2000)->str:
+    "Render assistant/tool `msgs` as one formatted output string, the inverse of `fmt2hist`"
+    tus, out = {}, []
+    for m in msgs:
+        if m.role == 'assistant':
+            for p in m.content:
+                if p.type == PartType.text and p.text: out.append(p.text.strip())
+                elif p.type == PartType.tool_use: tus[p.data['id']] = p
+        elif m.role == 'tool':
+            for p in m.content:
+                if p.type != PartType.tool_result: continue
+                tu = tus.get(p.data['id'])
+                d = dict(p.data, arguments=tu.data.get('arguments', {}) if tu else {})
+                out.append(mk_tr_details(Part(type=PartType.tool_result, text=p.text, data=d), mx=mx).strip())
+        else: raise ValueError(f"hist2fmt renders assistant and tool messages only, got {m.role!r}")
+    return '\n\n'.join(o for o in out if o)

{aidialog-0.0.2 → aidialog-0.0.3/aidialog.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aidialog
-Version: 0.0.2
+Version: 0.0.3
 Summary: Read, search, edit, and persist AI dialogs stored as Jupyter notebooks
 Author-email: Jeremy Howard <github@jhoward.fastmail.fm>
 License: Apache-2.0
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: fastcore>=2.1.9
+Requires-Dist: fastcore>=2.1.12
 Provides-Extra: dev
 Requires-Dist: exhash; extra == "dev"
 Requires-Dist: nbformat; extra == "dev"
@@ -51,7 +51,7 @@ The [`Dialog`](https://AnswerDotAI.github.io/aidialog/dialog.html#dialog) is the
 | ipynb file | storage, pragmatically lossless | [`read_ipynb`](https://AnswerDotAI.github.io/aidialog/ipynb.html#read_ipynb) | [`write_ipynb`](https://AnswerDotAI.github.io/aidialog/ipynb.html#write_ipynb) |
 | Claude Code session | storage | `sess2dlg` | `dlg2sess` |
 | Codex thread | storage (write-only so far) |  | `dlg2thread` |
-| fastllm chat (`Msg`/`Part`) | transmission, normalizing | `chat2dlg` | `dlg2chat` |
+| fastllm chat ([`Msg`](https://AnswerDotAI.github.io/aidialog/msg_parts.html#msg)/[`Part`](https://AnswerDotAI.github.io/aidialog/msg_parts.html#part)) | transmission, normalizing | `chat2dlg` | `dlg2chat` |
 | fastllm hist (live call input) | transmission, one-way |  | `dlg2hist` |
 | a prompt’s reply | self-similar | `reply2dlg` | `dlg2reply` |
 | XML views | display, one-way |  | [`view_dlg`](https://AnswerDotAI.github.io/aidialog/dlgskill.html#view_dlg), [`msg2xml`](https://AnswerDotAI.github.io/aidialog/dlgskill.html#msg2xml) |

{aidialog-0.0.2 → aidialog-0.0.3}/aidialog.egg-info/SOURCES.txt

@@ -7,6 +7,7 @@ aidialog/_modidx.py
 aidialog/dialog.py
 aidialog/dlgskill.py
 aidialog/ipynb.py
+aidialog/msg_parts.py
 aidialog.egg-info/PKG-INFO
 aidialog.egg-info/SOURCES.txt
 aidialog.egg-info/dependency_links.txt

{aidialog-0.0.2 → aidialog-0.0.3}/aidialog.egg-info/requires.txt

@@ -1,4 +1,4 @@
-fastcore>=2.1.9
+fastcore>=2.1.12
 
 [dev]
 exhash

{aidialog-0.0.2 → aidialog-0.0.3}/pyproject.toml

@@ -15,7 +15,7 @@ classifiers = [
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
 ]
-dependencies = ['fastcore>=2.1.9']
+dependencies = ['fastcore>=2.1.12']
 
 [project.optional-dependencies]
 dev = ['exhash', 'nbformat']

aidialog-0.0.2/aidialog/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.0.2"