toolslm 0.3.17__py3-none-any.whl → 0.3.23__py3-none-any.whl

toolslm/__init__.py

@@ -1 +1 @@
-__version__ = "0.3.17"
+__version__ = "0.3.23"

toolslm/_modidx.py

@@ -40,18 +40,22 @@ d = { 'settings': { 'branch': 'main',
             'toolslm.xml': { 'toolslm.xml._add_nls': ('xml.html#_add_nls', 'toolslm/xml.py'),
                              'toolslm.xml.cell2out': ('xml.html#cell2out', 'toolslm/xml.py'),
                              'toolslm.xml.cell2xml': ('xml.html#cell2xml', 'toolslm/xml.py'),
+                             'toolslm.xml.cells2xml': ('xml.html#cells2xml', 'toolslm/xml.py'),
                              'toolslm.xml.docs_xml': ('xml.html#docs_xml', 'toolslm/xml.py'),
                              'toolslm.xml.files2ctx': ('xml.html#files2ctx', 'toolslm/xml.py'),
                              'toolslm.xml.folder2ctx': ('xml.html#folder2ctx', 'toolslm/xml.py'),
                              'toolslm.xml.folder2ctx_cli': ('xml.html#folder2ctx_cli', 'toolslm/xml.py'),
+                             'toolslm.xml.get_docstring': ('xml.html#get_docstring', 'toolslm/xml.py'),
                              'toolslm.xml.get_mime_text': ('xml.html#get_mime_text', 'toolslm/xml.py'),
                              'toolslm.xml.json_to_xml': ('xml.html#json_to_xml', 'toolslm/xml.py'),
                              'toolslm.xml.mk_doc': ('xml.html#mk_doc', 'toolslm/xml.py'),
                              'toolslm.xml.mk_doctype': ('xml.html#mk_doctype', 'toolslm/xml.py'),
                              'toolslm.xml.nb2xml': ('xml.html#nb2xml', 'toolslm/xml.py'),
                              'toolslm.xml.parse_gh_url': ('xml.html#parse_gh_url', 'toolslm/xml.py'),
+                             'toolslm.xml.py2sigs': ('xml.html#py2sigs', 'toolslm/xml.py'),
                              'toolslm.xml.read_file': ('xml.html#read_file', 'toolslm/xml.py'),
                              'toolslm.xml.repo2ctx': ('xml.html#repo2ctx', 'toolslm/xml.py'),
+                             'toolslm.xml.repo2ctx_cli': ('xml.html#repo2ctx_cli', 'toolslm/xml.py'),
                              'toolslm.xml.sym2file': ('xml.html#sym2file', 'toolslm/xml.py'),
                              'toolslm.xml.sym2folderctx': ('xml.html#sym2folderctx', 'toolslm/xml.py'),
                              'toolslm.xml.sym2pkgctx': ('xml.html#sym2pkgctx', 'toolslm/xml.py'),

toolslm/funccall.py

@@ -141,7 +141,12 @@ def get_schema(
     assert desc, "Docstring missing!"
     d = docments(f, full=True)
     ret = d.pop('return')
-    if (ret.anno is not empty) and (ret.anno is not None): desc += f'\n\nReturns:\n- type: {_types(ret.anno)[0]}'
+    has_type = (ret.anno is not empty) and (ret.anno is not None)
+    has_doc = ret.docment
+    if has_type or has_doc:
+        type_str = f'type: {_types(ret.anno)[0]}' if has_type else None
+        ret_str = f'{ret.docment} ({type_str})' if has_type and has_doc else (type_str if has_type else ret.docment)
+        desc += f'\n\nReturns:\n- {ret_str}'
     return {"name": f.__name__, "description": desc, pname: schema}
 
 # %% ../01_funccall.ipynb

toolslm/xml.py

@@ -1,19 +1,19 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../00_xml.ipynb.
 
 # %% auto 0
-__all__ = ['doctype', 'json_to_xml', 'get_mime_text', 'cell2out', 'cell2xml', 'nb2xml', 'mk_doctype', 'mk_doc', 'docs_xml',
-           'read_file', 'files2ctx', 'folder2ctx', 'sym2file', 'sym2folderctx', 'sym2pkgpath', 'sym2pkgctx',
-           'folder2ctx_cli', 'parse_gh_url', 'repo2ctx']
+__all__ = ['doctype', 'json_to_xml', 'get_mime_text', 'cell2out', 'cell2xml', 'cells2xml', 'nb2xml', 'get_docstring', 'py2sigs',
+           'mk_doctype', 'mk_doc', 'docs_xml', 'read_file', 'files2ctx', 'folder2ctx', 'sym2file', 'sym2folderctx',
+           'sym2pkgpath', 'sym2pkgctx', 'folder2ctx_cli', 'parse_gh_url', 'repo2ctx', 'repo2ctx_cli']
 
 # %% ../00_xml.ipynb
-import hashlib, inspect, xml.etree.ElementTree as ET
+import hashlib, inspect, xml.etree.ElementTree as ET, ast
 from collections import namedtuple
 from ghapi.all import GhApi
 
 from fastcore.utils import *
 from fastcore.meta import delegates
 from fastcore.xtras import hl_md
-from fastcore.xml import to_xml, Document, Documents, Document_content, Src, Source,Out,Outs,Cell,Notebook,Md,Code
+from fastcore.xml import to_xml, Document, Documents, Document_content, Src, Source,Out,Outs,Cell,Notebook,Md,Code,Raw
 from fastcore.script import call_parse
 
 # %% ../00_xml.ipynb
@@ -50,10 +50,13 @@ def cell2out(o):
     if hasattr(o, 'ename'): return Out(f"{o.ename}: {o.evalue}", type='error')
 
 # %% ../00_xml.ipynb
-def cell2xml(cell, out=True, ids=True):
+_ctfuns = {'code': Code, 'markdown': Md, 'raw': Raw}
+
+def cell2xml(cell, out=True, ids=True, nums=False):
     "Convert notebook cell to concise XML format"
     src = ''.join(getattr(cell, 'source', ''))
-    f = Code if cell.cell_type=='code' else Md
+    if nums: src = '\n'.join(f'{i+1:6d} │ {l}' for i,l in enumerate(src.splitlines()))
+    f = _ctfuns[cell.cell_type]
     kw = dict(id=cell.id) if ids and hasattr(cell, 'id') else {}
     if not out: return f(src, **kw)
     parts = [Source(src)]
@@ -62,12 +65,40 @@ def cell2xml(cell, out=True, ids=True):
     return f(*parts, **kw)
 
 # %% ../00_xml.ipynb
-def nb2xml(fname=None, nb=None, out=True, ids=True):
+@delegates(cell2xml)
+def cells2xml(cells, wrap=Notebook, **kwargs):
+    "Convert notebook to XML format"
+    res = [cell2xml(c, **kwargs) for c in cells]
+    return to_xml(wrap(*res), do_escape=False)
+
+@delegates(cell2xml)
+def nb2xml(fname=None, nb=None, **kwargs):
     "Convert notebook to XML format"
     assert bool(fname)^bool(nb), "Pass either `fname` or `nb`"
     if not nb: nb = dict2obj(fname.read_json())
-    cells_xml = [to_xml(cell2xml(c, out=out, ids=ids), do_escape=False) for c in nb.cells if c.cell_type in ('code','markdown')]
-    return to_xml(Notebook(*cells_xml), do_escape=False)
+    return cells2xml(nb.cells, **kwargs)
+
+# %% ../00_xml.ipynb
+def get_docstring(node, lines):
+    "Get docstring from source lines if present"
+    if not (node.body and isinstance(node.body[0], ast.Expr) and isinstance(node.body[0].value, ast.Constant)): return None
+    doc_node = node.body[0]
+    return '\n'.join(lines[doc_node.lineno-1:doc_node.end_lineno])
+
+def py2sigs(fname=None, src=None):
+    "Return signature+docstring text for all functions and class methods in source"
+    if fname: src = Path(fname).expanduser().read_text()
+    tree = ast.parse(src)
+    lines = src.splitlines()
+    res = []
+    for node in ast.walk(tree):
+        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+            body_start = max(node.body[0].lineno - 1, node.lineno)
+            sig = '\n'.join(lines[node.lineno-1:body_start])
+            doc = get_docstring(node, lines)
+            cts = f"{sig}\n{doc}" if doc else sig
+            res.append(cts.strip('\r\n'))
+    return '\n\n'.join(res)
 
 # %% ../00_xml.ipynb
 doctype = namedtuple('doctype', ['src', 'content'])
@@ -111,15 +142,17 @@ def docs_xml(docs:list[str],  # The content of each document
     pre = 'Here are some documents for you to reference for your task:\n\n' if prefix else ''
     if srcs is None: srcs = [None]*len(docs)
     if details is None: details = [{}]*len(docs)
-    docs = (mk_doc(i+1, d, s, **kw) for i,(d,s,kw) in enumerate(zip(docs,srcs,details)))
+    docs = (mk_doc(i+1, d, s, **kw) for i,(d,s,kw) in enumerate(zip(docs,srcs,details)) if d.strip())
     kw = dict(title=title) if title else {}
     return pre + to_xml(Documents(*docs, **kw), do_escape=False)
 
 # %% ../00_xml.ipynb
-def read_file(fname, out=True, max_size=None, ids=True):
+@delegates(nb2xml)
+def read_file(fname, max_size=None, sigs_only=False, **kwargs):
     "Read file content, converting notebooks to XML if needed"
-    fname = Path(fname)
-    if fname.suffix == '.ipynb': res = nb2xml(fname, out=out, ids=ids)
+    fname = Path(fname).expanduser()
+    if fname.suffix == '.ipynb': res = nb2xml(fname, **kwargs)
+    elif fname.suffix == '.py' and sigs_only: res = py2sigs(fname)
     else: res = fname.read_text()
     if max_size and len(res)>max_size: return f"[Skipped: {fname.name} exceeds {max_size} bytes]"
     return res
@@ -128,21 +161,23 @@ def read_file(fname, out=True, max_size=None, ids=True):
 @delegates(docs_xml)
 def files2ctx(
     fnames:list[Union[str,Path]], # List of file names to add to context
-    out:bool=True, # Include notebook cell outputs?
     srcs:Optional[list]=None, # Use the labels instead of `fnames`
     max_size:int=None, # Skip files larger than this (bytes)
+    out:bool=True, # Include notebook cell outputs?
     ids:bool=True,  # Include cell ids in notebooks?
+    nums:bool=False, # Include line numbers in notebook cell source?
+    sigs_only:bool=False, # For .py files, only include signatures and docstrings
     **kwargs
 )->str: # XML for LM context
     "Convert files to XML context, handling notebooks"
-    fnames = [Path(o) for o in fnames]
-    contents = [read_file(o, out=out, max_size=max_size, ids=ids) for o in fnames]
+    fnames = [Path(o).expanduser() for o in listify(fnames)]
+    contents = [read_file(o, max_size=max_size, out=out, ids=ids, sigs_only=sigs_only, nums=nums) for o in fnames]
     return docs_xml(contents, srcs or fnames, **kwargs)
 
 # %% ../00_xml.ipynb
-@delegates(globtastic)
+@delegates(globtastic, but='func')
 def folder2ctx(
-    folder:Union[str,Path], # Folder to read
+    path:Union[str,Path], # Folder to read
     prefix:bool=False, # Include Anthropic's suggested prose intro?
     out:bool=True, # Include notebook cell outputs?
     include_base:bool=True, # Include full path in src?
@@ -151,16 +186,17 @@ def folder2ctx(
     max_total:int=10_000_000,  # Max total output size in bytes
     readme_first:bool=False,  # Prioritize README files at start of context?
     files_only:bool=False,  # Return dict of {filename: size} instead of context?
+    sigs_only:bool=False,  # Return signatures instead of full text for python files?
     ids:bool=True,  # Include cell ids in notebooks?
     **kwargs
 )->Union[str,dict]:
     "Convert folder contents to XML context, handling notebooks"
-    folder = Path(folder)
+    folder = Path(path).expanduser()
     fnames = pglob(folder, **kwargs)
     if files_only: return {str(f.relative_to(folder)): f.stat().st_size for f in fnames}
     if readme_first: fnames = sorted(fnames, key=lambda f: (0 if 'readme' in f.name.lower() else 1, f))
     srcs = fnames if include_base else [f.relative_to(folder) for f in fnames]
-    res = files2ctx(fnames, prefix=prefix, out=out, srcs=srcs, title=title, max_size=max_size, ids=ids)
+    res = files2ctx(fnames, prefix=prefix, out=out, srcs=srcs, title=title, max_size=max_size, sigs_only=sigs_only, ids=ids)
     suf = f"\n\n[TRUNCATED: output size {{_outsz_}} exceeded max size {max_total} bytes]"
     if max_total and len(res) > max_total: res = truncstr(res, max_total, suf=suf, sizevar='_outsz_')
     return res
@@ -197,12 +233,12 @@ def sym2pkgctx(sym, types:str|list='py', skip_file_re=r'^_mod', **kwargs):
 @call_parse
 @delegates(folder2ctx)
 def folder2ctx_cli(
-    folder:str, # Folder name containing files to add to context
+    path:str='.', # Folder name containing files to add to context
     out:bool=True, # Include notebook cell outputs?
     **kwargs # Passed to `folder2ctx`
 )->str: # XML for Claude context
     "CLI to convert folder contents to XML context, handling notebooks"
-    print(folder2ctx(folder, out=out, **kwargs))
+    print(folder2ctx(path, out=out, **kwargs))
 
 # %% ../00_xml.ipynb
 def parse_gh_url(url):
@@ -244,3 +280,17 @@ def repo2ctx(
         subdir = Path(tmp) / tf.getmembers()[0].name.split('/')[0]
         if folder: subdir = subdir/folder
         return folder2ctx(subdir, include_base=False, title=title, readme_first=True, **kwargs)
+
+# %% ../00_xml.ipynb
+@call_parse
+@delegates(repo2ctx, but='include_base,title,readme_first')
+def repo2ctx_cli(
+    owner:str,  # GitHub repo owner or "owner/repo" or a full github URL
+    repo:str=None,   # GitHub repo name (leave empty if using "owner/repo" or URL format)
+    ref:str=None,  # Git ref (branch/tag/sha)
+    folder:str=None,  # Only include files under this path
+    out:bool=True, # Include notebook cell outputs?
+    **kwargs # Passed to `repo2ctx`
+)->str: # XML for Claude context
+    "CLI to convert GitHub repo contents to XML context"
+    print(repo2ctx(owner, repo, ref=ref, folder=folder, out=out, **kwargs))

{toolslm-0.3.17.dist-info → toolslm-0.3.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: toolslm
-Version: 0.3.17
+Version: 0.3.23
 Summary: Tools to make language models a bit easier to use
 Home-page: https://github.com/AnswerDotAI/toolslm
 Author: Jeremy Howard

toolslm-0.3.23.dist-info/RECORD

@@ -0,0 +1,13 @@
+toolslm/__init__.py,sha256=YEBEORM81hFUTm-XaZztBUzkIBn7JXmZ6MQPnR7XWdE,23
+toolslm/_modidx.py,sha256=Gf9CRS_oX2eqJeO_9MedzlVEtUSZEeVnMQJZuWcTWw4,6200
+toolslm/download.py,sha256=yMhyY3u26XRr6a4eZuCCmkprS7LQhHASl01Zn2B4q_o,4481
+toolslm/funccall.py,sha256=Xoulo5xmYUpuqxm6ssvpABeGs8Hx3nyDah-FI9HlxlY,11400
+toolslm/md_hier.py,sha256=r_NPezhgfxjRmSYFlu_ND42hXt1qSbaPWHTcjbviOn4,11010
+toolslm/shell.py,sha256=dGInuRKvexu21VmtZkw_0S3BGiTsbAongUG-yG4YHpc,1566
+toolslm/xml.py,sha256=WLKnM_c7If6rG7MarFpYz9_0nsyaXSP8FUVfPv1sC7I,13487
+toolslm-0.3.23.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+toolslm-0.3.23.dist-info/METADATA,sha256=n-qbMh7K29vHc50KxHqZxho4iWRRHFvRFKHgk4dW3XU,2425
+toolslm-0.3.23.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+toolslm-0.3.23.dist-info/entry_points.txt,sha256=L77QoeUC_BjrE3BVY-Wpi4RMq_iwfX_1eAWchc6Zsmw,131
+toolslm-0.3.23.dist-info/top_level.txt,sha256=4hRTrFWayz_Kz5221XjvlpCwVFrW3WPi1P0fllkTq9s,8
+toolslm-0.3.23.dist-info/RECORD,,

{toolslm-0.3.17.dist-info → toolslm-0.3.23.dist-info}/entry_points.txt

@@ -1,5 +1,6 @@
 [console_scripts]
 folder2ctx = toolslm.xml:folder2ctx_cli
+repo2ctx = toolslm.xml:repo2ctx_cli
 
 [nbdev]
 toolslm = toolslm._modidx:d

toolslm-0.3.17.dist-info/RECORD

@@ -1,13 +0,0 @@
-toolslm/__init__.py,sha256=HXhmv802-3PQnM5q29vlyrO10zGWKWZQ7xsx3qPYVRM,23
-toolslm/_modidx.py,sha256=EC1pFuHb5MbfRMml7RXx1sxGXlTiczjUimXICuXUMn0,5806
-toolslm/download.py,sha256=yMhyY3u26XRr6a4eZuCCmkprS7LQhHASl01Zn2B4q_o,4481
-toolslm/funccall.py,sha256=_5TyhTjWaWLi-eJ96-4P3_faFv6Ft07nO60UjCF-bPU,11160
-toolslm/md_hier.py,sha256=r_NPezhgfxjRmSYFlu_ND42hXt1qSbaPWHTcjbviOn4,11010
-toolslm/shell.py,sha256=dGInuRKvexu21VmtZkw_0S3BGiTsbAongUG-yG4YHpc,1566
-toolslm/xml.py,sha256=I2lRJPVG6us1g_gOTOKbbnZdlGB3g2-6MYGcLSqkFrM,11173
-toolslm-0.3.17.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-toolslm-0.3.17.dist-info/METADATA,sha256=jSPVU5a6Qe0eYmuxTqj30o0YP81UgHOH_gIAOeeQDaE,2425
-toolslm-0.3.17.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-toolslm-0.3.17.dist-info/entry_points.txt,sha256=xFz0Eymlo5X7BGpaO6DI9gMxvN5A7faebzrlr8ctp5I,95
-toolslm-0.3.17.dist-info/top_level.txt,sha256=4hRTrFWayz_Kz5221XjvlpCwVFrW3WPi1P0fllkTq9s,8
-toolslm-0.3.17.dist-info/RECORD,,