toolslm 0.3.2__py3-none-any.whl → 0.3.7__py3-none-any.whl

toolslm/__init__.py

@@ -1 +1 @@
-__version__ = "0.3.2"
+__version__ = "0.3.7"

toolslm/_modidx.py

@@ -14,7 +14,7 @@ d = { 'settings': { 'branch': 'main',
                                   'toolslm.download.read_html': ('download.html#read_html', 'toolslm/download.py'),
                                   'toolslm.download.read_md': ('download.html#read_md', 'toolslm/download.py'),
                                   'toolslm.download.split_url': ('download.html#split_url', 'toolslm/download.py')},
-            'toolslm.funccall': { 'toolslm.funccall.PathArg': ('funccall.html#patharg', 'toolslm/funccall.py'),
+            'toolslm.funccall': { 'toolslm.funccall._coerce_inputs': ('funccall.html#_coerce_inputs', 'toolslm/funccall.py'),
                                   'toolslm.funccall._copy_loc': ('funccall.html#_copy_loc', 'toolslm/funccall.py'),
                                   'toolslm.funccall._get_nested_schema': ('funccall.html#_get_nested_schema', 'toolslm/funccall.py'),
                                   'toolslm.funccall._handle_container': ('funccall.html#_handle_container', 'toolslm/funccall.py'),
@@ -29,16 +29,25 @@ d = { 'settings': { 'branch': 'main',
                                   'toolslm.funccall.call_func_async': ('funccall.html#call_func_async', 'toolslm/funccall.py'),
                                   'toolslm.funccall.get_schema': ('funccall.html#get_schema', 'toolslm/funccall.py'),
                                   'toolslm.funccall.mk_ns': ('funccall.html#mk_ns', 'toolslm/funccall.py'),
-                                  'toolslm.funccall.python': ('funccall.html#python', 'toolslm/funccall.py')},
+                                  'toolslm.funccall.mk_param': ('funccall.html#mk_param', 'toolslm/funccall.py'),
+                                  'toolslm.funccall.mk_tool': ('funccall.html#mk_tool', 'toolslm/funccall.py'),
+                                  'toolslm.funccall.python': ('funccall.html#python', 'toolslm/funccall.py'),
+                                  'toolslm.funccall.schema2sig': ('funccall.html#schema2sig', 'toolslm/funccall.py')},
             'toolslm.md_hier': {},
             'toolslm.shell': { 'toolslm.shell.TerminalInteractiveShell.run_cell': ( 'shell.html#terminalinteractiveshell.run_cell',
                                                                                     'toolslm/shell.py'),
                                'toolslm.shell.get_shell': ('shell.html#get_shell', 'toolslm/shell.py')},
             'toolslm.xml': { 'toolslm.xml._add_nls': ('xml.html#_add_nls', 'toolslm/xml.py'),
+                             'toolslm.xml.cell2out': ('xml.html#cell2out', 'toolslm/xml.py'),
+                             'toolslm.xml.cell2xml': ('xml.html#cell2xml', 'toolslm/xml.py'),
                              'toolslm.xml.docs_xml': ('xml.html#docs_xml', 'toolslm/xml.py'),
                              'toolslm.xml.files2ctx': ('xml.html#files2ctx', 'toolslm/xml.py'),
                              'toolslm.xml.folder2ctx': ('xml.html#folder2ctx', 'toolslm/xml.py'),
                              'toolslm.xml.folder2ctx_cli': ('xml.html#folder2ctx_cli', 'toolslm/xml.py'),
+                             'toolslm.xml.get_mime_text': ('xml.html#get_mime_text', 'toolslm/xml.py'),
                              'toolslm.xml.json_to_xml': ('xml.html#json_to_xml', 'toolslm/xml.py'),
                              'toolslm.xml.mk_doc': ('xml.html#mk_doc', 'toolslm/xml.py'),
-                             'toolslm.xml.mk_doctype': ('xml.html#mk_doctype', 'toolslm/xml.py')}}}
+                             'toolslm.xml.mk_doctype': ('xml.html#mk_doctype', 'toolslm/xml.py'),
+                             'toolslm.xml.nb2xml': ('xml.html#nb2xml', 'toolslm/xml.py'),
+                             'toolslm.xml.read_file': ('xml.html#read_file', 'toolslm/xml.py'),
+                             'toolslm.xml.repo2ctx': ('xml.html#repo2ctx', 'toolslm/xml.py')}}}

toolslm/funccall.py

@@ -1,15 +1,20 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../01_funccall.ipynb.
 
 # %% auto 0
-__all__ = ['empty', 'custom_types', 'get_schema', 'PathArg', 'python', 'mk_ns', 'call_func', 'call_func_async']
+__all__ = ['empty', 'custom_types', 'get_schema', 'python', 'mk_ns', 'call_func', 'call_func_async', 'mk_param', 'schema2sig',
+           'mk_tool']
 
 # %% ../01_funccall.ipynb
-import inspect
+import inspect, json
 from collections import abc
 from fastcore.utils import *
 from fastcore.docments import docments
-from typing import get_origin, get_args, Dict, List, Optional, Tuple, Union
+from typing import get_origin, get_args, Dict, List, Optional, Tuple, Union, Any
 from types import UnionType
+from typing import get_type_hints
+from inspect import Parameter, Signature
+from decimal import Decimal
+from uuid import UUID
 
 # %% ../01_funccall.ipynb
 empty = inspect.Parameter.empty
@@ -41,7 +46,7 @@ def _param(name, info):
     return pschema
 
 # %% ../01_funccall.ipynb
-custom_types = {Path}
+custom_types = {Path, bytes, Decimal, UUID}
 
 def _handle_type(t, defs):
     "Handle a single type, creating nested schemas if necessary"
@@ -107,7 +112,9 @@ def _get_nested_schema(obj):
         if n != 'return' and n != 'self':
             _process_property(n, o, props, req, defs)
 
-    schema = dict(type='object', properties=props, title=obj.__name__ if isinstance(obj, type) else None)
+    tkw = {}
+    if isinstance(obj, type): tkw['title']=obj.__name__
+    schema = dict(type='object', properties=props, **tkw)
     if req: schema['required'] = list(req)
     if defs: schema['$defs'] = defs
     return schema
@@ -121,14 +128,9 @@ def get_schema(f:Union[callable,dict], pname='input_schema')->dict:
     assert desc, "Docstring missing!"
     d = docments(f, full=True)
     ret = d.pop('return')
-    if ret.anno is not empty: desc += f'\n\nReturns:\n- type: {_types(ret.anno)[0]}'
+    if (ret.anno is not empty) and (ret.anno is not None): desc += f'\n\nReturns:\n- type: {_types(ret.anno)[0]}'
     return {"name": f.__name__, "description": desc, pname: schema}
 
-# %% ../01_funccall.ipynb
-def PathArg(
-    path: str  # A filesystem path
-): return Path(path)
-
 # %% ../01_funccall.ipynb
 import ast, time, signal, traceback
 from fastcore.utils import *
@@ -191,14 +193,26 @@ def mk_ns(fs):
         elif callable(o) and hasattr(o, '__name__'): merged |= {o.__name__: o}
     return merged
 
+# %% ../01_funccall.ipynb
+def _coerce_inputs(func, inputs):
+    "Coerce inputs based on function type annotations"
+    hints = get_type_hints(func) if hasattr(func, '__annotations__') else {}
+    res = {}
+    for k,v in inputs.items():
+        ann = hints.get(k)
+        if ann in custom_types: res[k] = ann(v)
+        elif isinstance(v, dict) and callable(ann): res[k] = ann(**v)
+        else: res[k] = v
+    return res
+
 # %% ../01_funccall.ipynb
 def call_func(fc_name, fc_inputs, ns, raise_on_err=True):
     "Call the function `fc_name` with the given `fc_inputs` using namespace `ns`."
     if not isinstance(ns, abc.Mapping): ns = mk_ns(ns)
     func = ns[fc_name]
-    # Clean up bad param names
     inps = {re.sub(r'\W', '', k):v for k,v in fc_inputs.items()}
-    try: return func(**fc_inputs)
+    inps = _coerce_inputs(func, inps)
+    try: return func(**inps)
     except Exception as e:
         if raise_on_err: raise e from None
         else: return traceback.format_exc()
@@ -213,3 +227,35 @@ async def call_func_async(fc_name, fc_inputs, ns, raise_on_err=True):
             if raise_on_err: raise e from None
             else: return traceback.format_exc()
     return res
+
+# %% ../01_funccall.ipynb
+def mk_param(nm, props, req):
+    "Create a `Parameter` for `nm` with schema `props`"
+    kind = Parameter.POSITIONAL_OR_KEYWORD if nm in req else Parameter.KEYWORD_ONLY
+    default = Parameter.empty if nm in req else props.get('default')    
+    if props.get('type') == 'array' and 'items' in props:
+        item_type = type_map.get(props['items'].get('type'), Any)
+        anno = list[item_type]
+    else: anno = type_map.get(props.get('type'), Any)
+    return Parameter(nm, kind, default=default, annotation=anno)
+
+# %% ../01_funccall.ipynb
+def schema2sig(tool):
+    "Convert json schema `tool` to a `Signature`"
+    props, req = tool.inputSchema['properties'], tool.inputSchema.get('required', [])
+    params = sorted([mk_param(k, v, req) for k, v in props.items()], key=lambda p: p.kind)
+    return Signature(params)
+
+# %% ../01_funccall.ipynb
+def mk_tool(dispfn, tool):
+    "Create a callable function from a JSON schema tool definition"
+    sig = schema2sig(tool)
+    props = tool.inputSchema['properties']
+    def fn(*args, **kwargs):
+        bound = sig.bind(*args, **kwargs)
+        return dispfn(tool.name, **bound.arguments)
+    fn.__doc__ = tool.description
+    fn.__signature__ = sig
+    fn.__name__ = fn.__qualname__ = tool.name
+    fn.__annotations__ = {k: p.annotation for k, p in sig.parameters.items()}
+    return fn

toolslm/md_hier.py

@@ -14,20 +14,32 @@ def create_heading_dict(text, rm_fenced=True):
     original_text = text
     original_lines = text.splitlines()
     
-    # Use fenced-removed text only for finding headings
-    text_for_headings = text
-    if rm_fenced: text_for_headings = re.sub(r'```[\s\S]*?```', '', text)
+    # Build line mapping when removing fenced blocks
+    if rm_fenced:
+        filtered_lines,original_line_map,in_fenced = [],{},False
+        for i, line in enumerate(original_lines):
+            if line.strip().startswith('```'):
+                in_fenced = not in_fenced
+                continue
+            if not in_fenced:
+                original_line_map[len(filtered_lines)] = i
+                filtered_lines.append(line)
+
+        lines_for_headings = filtered_lines
+    else:
+        lines_for_headings = original_lines
+        original_line_map = {i: i for i in range(len(original_lines))}
 
-    lines_for_headings = text_for_headings.splitlines()
     headings = []
 
-    # Parse headings with their levels and line numbers
+    # Parse headings with their levels and original line numbers
     for idx, line in enumerate(lines_for_headings):
         match = re.match(r'^(#{1,6})\s+\S.*', line)
         if match:
             level = len(match.group(1))
             title = line.strip('#').strip()
-            headings.append({'level': level, 'title': title, 'line': idx})
+            original_line_idx = original_line_map[idx]
+            headings.append({'level': level, 'title': title, 'line': original_line_idx})
 
     # Assign text content to each heading using original lines
     for i, h in enumerate(headings):
@@ -270,9 +282,26 @@ More content after code."""
         assert '💻 Computer' in result['🚀 Rocket Heading']
         assert result.text == md_content
 
+    def test_fenced_blocks_between_headings():
+        md_content = """# First Header
+Content here.
+
+```python
+# A comment that should be ignored for structure
+def example():
+    pass
+```
+
+# Second Header
+This content belongs to Second Header."""
+        result = create_heading_dict(md_content)
+        assert "def example():" not in result['Second Header'].text
+        assert result['Second Header'].text == "# Second Header\nThis content belongs to Second Header."
+
     test_empty_input()
     test_whitespace_only()
     test_malformed_headings()
     test_unicode_and_emojis()
+    test_fenced_blocks_between_headings()
     print('tests passed')
 

toolslm/xml.py

@@ -1,19 +1,19 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../00_xml.ipynb.
 
 # %% auto 0
-__all__ = ['doctype', 'json_to_xml', 'mk_doctype', 'mk_doc', 'docs_xml', 'files2ctx', 'folder2ctx', 'folder2ctx_cli']
+__all__ = ['doctype', 'json_to_xml', 'get_mime_text', 'cell2out', 'cell2xml', 'nb2xml', 'mk_doctype', 'mk_doc', 'docs_xml',
+           'read_file', 'files2ctx', 'folder2ctx', 'repo2ctx', 'folder2ctx_cli']
 
 # %% ../00_xml.ipynb
 import hashlib,xml.etree.ElementTree as ET
 from collections import namedtuple
+from ghapi.all import GhApi
 
 from fastcore.utils import *
 from fastcore.meta import delegates
 from fastcore.xtras import hl_md
-from fastcore.xml import to_xml, Document, Documents, Document_content, Src
+from fastcore.xml import to_xml, Document, Documents, Document_content, Src, Source,Out,Outs,Cell,Notebook,Md,Code
 from fastcore.script import call_parse
-try: from IPython import display
-except: display=None
 
 # %% ../00_xml.ipynb
 def json_to_xml(d:dict, # JSON dictionary to convert
@@ -31,6 +31,42 @@ def json_to_xml(d:dict, # JSON dictionary to convert
     ET.indent(root)
     return ET.tostring(root, encoding='unicode')
 
+# %% ../00_xml.ipynb
+def get_mime_text(data):
+    "Get text from MIME bundle, preferring markdown over plain"
+    if 'text/markdown' in data: return ''.join(list(data['text/markdown']))
+    if 'text/plain' in data: return ''.join(list(data['text/plain']))
+
+# %% ../00_xml.ipynb
+def cell2out(o):
+    "Convert single notebook output to XML format"
+    if hasattr(o, 'data'): 
+        txt = get_mime_text(o.data)
+        if txt: return Out(txt, mime='markdown' if 'text/markdown' in o.data else 'plain')
+    if hasattr(o, 'text'):
+        txt = o.text if isinstance(o.text, str) else ''.join(o.text)
+        return Out(txt, type='stream', name=o.get('name', 'stdout'))
+    if hasattr(o, 'ename'): return Out(f"{o.ename}: {o.evalue}", type='error')
+
+# %% ../00_xml.ipynb
+def cell2xml(cell, out=True):
+    "Convert notebook cell to concise XML format"
+    src = ''.join(getattr(cell, 'source', ''))
+    f = Code if cell.cell_type=='code' else Md
+    if not out: return f(src)
+    parts = [Source(src)]
+    out_items = L(getattr(cell,'outputs',[])).map(cell2out).filter()
+    if out_items: parts.append(Outs(*out_items))
+    return f(*parts)
+
+# %% ../00_xml.ipynb
+def nb2xml(fname=None, nb=None, out=True):
+    "Convert notebook to XML format"
+    assert bool(fname)^bool(nb), "Pass either `fname` or `nb`"
+    if not nb: nb = dict2obj(fname.read_json())
+    cells_xml = [to_xml(cell2xml(c, out=out), do_escape=False) for c in nb.cells if c.cell_type in ('code','markdown')]
+    return to_xml(Notebook(*cells_xml), do_escape=False)
+
 # %% ../00_xml.ipynb
 doctype = namedtuple('doctype', ['src', 'content'])
 
@@ -73,32 +109,68 @@ def docs_xml(docs:list[str],  # The content of each document
     if srcs is None: srcs = [None]*len(docs)
     if details is None: details = [{}]*len(docs)
     docs = (mk_doc(i+1, d, s, **kw) for i,(d,s,kw) in enumerate(zip(docs,srcs,details)))
-    return pre + to_xml(Documents(docs))
+    return pre + to_xml(Documents(docs), do_escape=False)
+
+# %% ../00_xml.ipynb
+def read_file(fname, out=True):
+    "Read file content, converting notebooks to XML if needed"
+    fname = Path(fname)
+    if fname.suffix == '.ipynb': return nb2xml(fname, out=out)
+    return fname.read_text()
 
 # %% ../00_xml.ipynb
 def files2ctx(
     fnames:list[Union[str,Path]], # List of file names to add to context
-    prefix:bool=True # Include Anthropic's suggested prose intro?
+    prefix:bool=True, # Include Anthropic's suggested prose intro?
+    out:bool=True, # Include notebook cell outputs?
+    srcs:Optional[list]=None # Use the labels instead of `fnames`
 )->str: # XML for LM context
+    "Convert files to XML context, handling notebooks"
     fnames = [Path(o) for o in fnames]
-    contents = [o.read_text() for o in fnames]
-    return docs_xml(contents, fnames, prefix=prefix)
+    contents = [read_file(o, out=out) for o in fnames]
+    return docs_xml(contents, srcs or fnames, prefix=prefix)
 
 # %% ../00_xml.ipynb
 @delegates(globtastic)
 def folder2ctx(
-    folder:Union[str,Path], # Folder name containing files to add to context
-    prefix:bool=True, # Include Anthropic's suggested prose intro?
-    **kwargs # Passed to `globtastic`
-)->str: # XML for Claude context
+    folder:Union[str,Path],
+    prefix:bool=True,
+    out:bool=True,
+    include_base:bool=True,
+    **kwargs
+)->str:
+    "Convert folder contents to XML context, handling notebooks"
+    folder = Path(folder)
     fnames = globtastic(folder, **kwargs)
-    return files2ctx(fnames, prefix=prefix)
+    srcs = fnames if include_base else [Path(f).relative_to(folder) for f in fnames]
+    return files2ctx(fnames, prefix=prefix, out=out, srcs=srcs)
+
+# %% ../00_xml.ipynb
+@delegates(folder2ctx)
+def repo2ctx(
+    owner:str,  # GitHub repo owner
+    repo:str,   # GitHub repo name
+    ref:str=None,  # Git ref (branch/tag/sha); defaults to repo's default branch
+    **kwargs  # Passed to `folder2ctx`
+)->str:  # XML for LM context
+    "Convert GitHub repo to XML context without cloning"
+    import tempfile, tarfile, io
+    api = GhApi()
+    if ref is None: ref = api.repos.get(owner, repo).default_branch
+    data = api.repos.download_tarball_archive(owner, repo, ref)
+    tf = tarfile.open(fileobj=io.BytesIO(data))
+    with tempfile.TemporaryDirectory() as tmp:
+        tf.extractall(tmp, filter='data')
+        subdir = Path(tmp) / tf.getmembers()[0].name.split('/')[0]
+        return folder2ctx(subdir, include_base=False, **kwargs)
 
 # %% ../00_xml.ipynb
 @call_parse
 @delegates(folder2ctx)
 def folder2ctx_cli(
     folder:str, # Folder name containing files to add to context
+    out:bool=True, # Include notebook cell outputs?
     **kwargs # Passed to `folder2ctx`
 )->str: # XML for Claude context
-    print(folder2ctx(folder, **kwargs))
+    "CLI to convert folder contents to XML context, handling notebooks"
+    print(folder2ctx(folder, out=out, **kwargs))

{toolslm-0.3.2.dist-info → toolslm-0.3.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: toolslm
-Version: 0.3.2
+Version: 0.3.7
 Summary: Tools to make language models a bit easier to use
 Home-page: https://github.com/AnswerDotAI/toolslm
 Author: Jeremy Howard
@@ -16,7 +16,7 @@ Classifier: License :: OSI Approved :: Apache Software License
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: fastcore>=1.5.47
+Requires-Dist: fastcore>=1.9.6
 Requires-Dist: httpx
 Provides-Extra: dev
 Dynamic: author

toolslm-0.3.7.dist-info/RECORD

@@ -0,0 +1,13 @@
+toolslm/__init__.py,sha256=J0I0c7-a50EOnWXMryTu_E6xhXSYFBPjVpeYP_a3vRI,22
+toolslm/_modidx.py,sha256=kpgsDpj-Tvn90wezrHaMttyzhNcyNVgw_dQgK10qotI,5308
+toolslm/download.py,sha256=g3BxUSxylC_575M7RFSJ1GI3Co3EwPDdEeWzxaf2Czk,4451
+toolslm/funccall.py,sha256=0OBrx6KzI0KK13L-5Hn69yah9oZhgTsKchmMenCoT0A,10421
+toolslm/md_hier.py,sha256=r_NPezhgfxjRmSYFlu_ND42hXt1qSbaPWHTcjbviOn4,11010
+toolslm/shell.py,sha256=dGInuRKvexu21VmtZkw_0S3BGiTsbAongUG-yG4YHpc,1566
+toolslm/xml.py,sha256=TO3i6QD1g_ya8B7Wxwib2ZWv7pwVpfyaAalw1qrKb74,7148
+toolslm-0.3.7.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+toolslm-0.3.7.dist-info/METADATA,sha256=IaRtHzIR_YzPP-XOAcO5EdSKWChZwgpSOz5HCdgGqGc,2403
+toolslm-0.3.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+toolslm-0.3.7.dist-info/entry_points.txt,sha256=xFz0Eymlo5X7BGpaO6DI9gMxvN5A7faebzrlr8ctp5I,95
+toolslm-0.3.7.dist-info/top_level.txt,sha256=4hRTrFWayz_Kz5221XjvlpCwVFrW3WPi1P0fllkTq9s,8
+toolslm-0.3.7.dist-info/RECORD,,

toolslm-0.3.2.dist-info/RECORD

@@ -1,13 +0,0 @@
-toolslm/__init__.py,sha256=vNiWJ14r_cw5t_7UDqDQIVZvladKFGyHH2avsLpN7Vg,22
-toolslm/_modidx.py,sha256=-D-B5o30VGs11gBKf96lpADVXnZhdiVEshJpLzmUnDs,4378
-toolslm/download.py,sha256=g3BxUSxylC_575M7RFSJ1GI3Co3EwPDdEeWzxaf2Czk,4451
-toolslm/funccall.py,sha256=7nPfbcvDRMWiVKBKMLlCOMInoUJgDs5e38ef2T7QBHY,8485
-toolslm/md_hier.py,sha256=Havk9Hf0t2Xt67n_r7ZxCsS0pciR85iLcE5quShvkTg,10032
-toolslm/shell.py,sha256=dGInuRKvexu21VmtZkw_0S3BGiTsbAongUG-yG4YHpc,1566
-toolslm/xml.py,sha256=D665Nk7NzyZlXyXrpnIRqfK2xQ-6Gf0bCSgocjF7zik,4061
-toolslm-0.3.2.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-toolslm-0.3.2.dist-info/METADATA,sha256=5lWEv7BWTwdd5cvXgGsQXqr0j6tk8UIcGpRTlcjV3V4,2404
-toolslm-0.3.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-toolslm-0.3.2.dist-info/entry_points.txt,sha256=xFz0Eymlo5X7BGpaO6DI9gMxvN5A7faebzrlr8ctp5I,95
-toolslm-0.3.2.dist-info/top_level.txt,sha256=4hRTrFWayz_Kz5221XjvlpCwVFrW3WPi1P0fllkTq9s,8
-toolslm-0.3.2.dist-info/RECORD,,