toolslm 0.3.23__tar.gz → 0.3.25__tar.gz

{toolslm-0.3.23/toolslm.egg-info → toolslm-0.3.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: toolslm
-Version: 0.3.23
+Version: 0.3.25
 Summary: Tools to make language models a bit easier to use
 Home-page: https://github.com/AnswerDotAI/toolslm
 Author: Jeremy Howard
@@ -19,7 +19,9 @@ License-File: LICENSE
 Requires-Dist: fastcore>=1.9.7
 Requires-Dist: httpx
 Requires-Dist: ghapi
+Requires-Dist: codesigs
 Provides-Extra: dev
+Requires-Dist: ipython; extra == "dev"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{toolslm-0.3.23 → toolslm-0.3.25}/settings.ini

@@ -1,11 +1,12 @@
 [DEFAULT]
 repo = toolslm
 lib_name = toolslm
-version = 0.3.23
+version = 0.3.25
 min_python = 3.9
 license = apache2
 black_formatting = False
-requirements = fastcore>=1.9.7 httpx ghapi
+requirements = fastcore>=1.9.7 httpx ghapi codesigs
+dev_requirements = ipython
 doc_path = _docs
 lib_path = toolslm
 nbs_path = .
@@ -31,7 +32,6 @@ readme_nb = index.ipynb
 allowed_metadata_keys = 
 allowed_cell_metadata_keys = 
 jupyter_hooks = True
-clean_ids = True
 clear_all = False
 conda_user = fastai
 console_scripts = folder2ctx=toolslm.xml:folder2ctx_cli

toolslm-0.3.25/toolslm/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.25"

{toolslm-0.3.23 → toolslm-0.3.25}/toolslm/_modidx.py

@@ -33,6 +33,30 @@ d = { 'settings': { 'branch': 'main',
                                   'toolslm.funccall.mk_tool': ('funccall.html#mk_tool', 'toolslm/funccall.py'),
                                   'toolslm.funccall.python': ('funccall.html#python', 'toolslm/funccall.py'),
                                   'toolslm.funccall.schema2sig': ('funccall.html#schema2sig', 'toolslm/funccall.py')},
+            'toolslm.inspecttools': { 'toolslm.inspecttools.SymbolNotFound': ( 'inspecttools.html#symbolnotfound',
+                                                                               'toolslm/inspecttools.py'),
+                                      'toolslm.inspecttools.SymbolNotFound.__repr__': ( 'inspecttools.html#symbolnotfound.__repr__',
+                                                                                        'toolslm/inspecttools.py'),
+                                      'toolslm.inspecttools._find_frame_dict': ( 'inspecttools.html#_find_frame_dict',
+                                                                                 'toolslm/inspecttools.py'),
+                                      'toolslm.inspecttools._src_from_lines': ( 'inspecttools.html#_src_from_lines',
+                                                                                'toolslm/inspecttools.py'),
+                                      'toolslm.inspecttools.importmodule': ('inspecttools.html#importmodule', 'toolslm/inspecttools.py'),
+                                      'toolslm.inspecttools.resolve': ('inspecttools.html#resolve', 'toolslm/inspecttools.py'),
+                                      'toolslm.inspecttools.symdir': ('inspecttools.html#symdir', 'toolslm/inspecttools.py'),
+                                      'toolslm.inspecttools.symfiles_folder': ( 'inspecttools.html#symfiles_folder',
+                                                                                'toolslm/inspecttools.py'),
+                                      'toolslm.inspecttools.symfiles_package': ( 'inspecttools.html#symfiles_package',
+                                                                                 'toolslm/inspecttools.py'),
+                                      'toolslm.inspecttools.symlen': ('inspecttools.html#symlen', 'toolslm/inspecttools.py'),
+                                      'toolslm.inspecttools.symnth': ('inspecttools.html#symnth', 'toolslm/inspecttools.py'),
+                                      'toolslm.inspecttools.symsearch': ('inspecttools.html#symsearch', 'toolslm/inspecttools.py'),
+                                      'toolslm.inspecttools.symset': ('inspecttools.html#symset', 'toolslm/inspecttools.py'),
+                                      'toolslm.inspecttools.symslice': ('inspecttools.html#symslice', 'toolslm/inspecttools.py'),
+                                      'toolslm.inspecttools.symsrc': ('inspecttools.html#symsrc', 'toolslm/inspecttools.py'),
+                                      'toolslm.inspecttools.symtype': ('inspecttools.html#symtype', 'toolslm/inspecttools.py'),
+                                      'toolslm.inspecttools.symtype_val': ('inspecttools.html#symtype_val', 'toolslm/inspecttools.py'),
+                                      'toolslm.inspecttools.symval': ('inspecttools.html#symval', 'toolslm/inspecttools.py')},
             'toolslm.md_hier': {},
             'toolslm.shell': { 'toolslm.shell.TerminalInteractiveShell.run_cell': ( 'shell.html#terminalinteractiveshell.run_cell',
                                                                                     'toolslm/shell.py'),
@@ -45,14 +69,12 @@ d = { 'settings': { 'branch': 'main',
                              'toolslm.xml.files2ctx': ('xml.html#files2ctx', 'toolslm/xml.py'),
                              'toolslm.xml.folder2ctx': ('xml.html#folder2ctx', 'toolslm/xml.py'),
                              'toolslm.xml.folder2ctx_cli': ('xml.html#folder2ctx_cli', 'toolslm/xml.py'),
-                             'toolslm.xml.get_docstring': ('xml.html#get_docstring', 'toolslm/xml.py'),
                              'toolslm.xml.get_mime_text': ('xml.html#get_mime_text', 'toolslm/xml.py'),
                              'toolslm.xml.json_to_xml': ('xml.html#json_to_xml', 'toolslm/xml.py'),
                              'toolslm.xml.mk_doc': ('xml.html#mk_doc', 'toolslm/xml.py'),
                              'toolslm.xml.mk_doctype': ('xml.html#mk_doctype', 'toolslm/xml.py'),
                              'toolslm.xml.nb2xml': ('xml.html#nb2xml', 'toolslm/xml.py'),
                              'toolslm.xml.parse_gh_url': ('xml.html#parse_gh_url', 'toolslm/xml.py'),
-                             'toolslm.xml.py2sigs': ('xml.html#py2sigs', 'toolslm/xml.py'),
                              'toolslm.xml.read_file': ('xml.html#read_file', 'toolslm/xml.py'),
                              'toolslm.xml.repo2ctx': ('xml.html#repo2ctx', 'toolslm/xml.py'),
                              'toolslm.xml.repo2ctx_cli': ('xml.html#repo2ctx_cli', 'toolslm/xml.py'),

{toolslm-0.3.23 → toolslm-0.3.25}/toolslm/download.py

@@ -1,29 +1,29 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../03_download.ipynb.
 
-# %% auto 0
+# %% auto #0
 __all__ = ['clean_md', 'read_md', 'html2md', 'read_html', 'get_llmstxt', 'split_url', 'find_docs', 'read_docs']
 
-# %% ../03_download.ipynb
+# %% ../03_download.ipynb #e58d8c43
 from fastcore.utils import *
 from httpx import get
 from fastcore.meta import delegates
 from urllib.parse import urlparse, urljoin
 from .xml import parse_gh_url
 
-# %% ../03_download.ipynb
+# %% ../03_download.ipynb #95c4cab1
 def clean_md(text, rm_comments=True, rm_details=True):
     "Remove comments and `<details>` sections from `text`"
     if rm_comments: text = re.sub(r'\n?<!--.*?-->\n?', '', text, flags=re.DOTALL)
     if rm_details: text = re.sub(r'\n?<details>.*?</details>\n?', '', text, flags=re.DOTALL)
     return text
 
-# %% ../03_download.ipynb
+# %% ../03_download.ipynb #0f3d5c69
 @delegates(get)
 def read_md(url, rm_comments=True, rm_details=True, **kwargs):
     "Read text from `url` and clean with `clean_docs`"
     return clean_md(get(url, **kwargs).text, rm_comments=rm_comments, rm_details=rm_details)
 
-# %% ../03_download.ipynb
+# %% ../03_download.ipynb #d8d61937
 def html2md(s:str, ignore_links=True):
     "Convert `s` from HTML to markdown"
     import html2text
@@ -33,7 +33,7 @@ def html2md(s:str, ignore_links=True):
     o.ignore_images = True
     return o.handle(s)
 
-# %% ../03_download.ipynb
+# %% ../03_download.ipynb #5e897053
 def read_html(url, # URL to read
               sel=None, # Read only outerHTML of CSS selector `sel`
               rm_comments=True, # Removes HTML comments
@@ -55,7 +55,7 @@ def read_html(url, # URL to read
     if wrap_tag: return '\n'.join([f"\n<{wrap_tag}>\n{o}</{wrap_tag}>\n" for o in mds])
     else: return'\n'.join(mds)
 
-# %% ../03_download.ipynb
+# %% ../03_download.ipynb #066b5532
 def get_llmstxt(url, optional=False, n_workers=None):
     "Get llms.txt file from and expand it with `llms_txt.create_ctx()`"
     if not url.endswith('llms.txt'): return None
@@ -64,7 +64,7 @@ def get_llmstxt(url, optional=False, n_workers=None):
     if resp.status_code!=200: return None
     return llms_txt.create_ctx(resp.text, optional=optional, n_workers=n_workers)
 
-# %% ../03_download.ipynb
+# %% ../03_download.ipynb #a2fc5a55
 def split_url(url):
     "Split `url` into base, path, and file name, normalising name to '/' if empty"
     parsed = urlparse(url.strip('/'))
@@ -74,13 +74,13 @@ def split_url(url):
     if not path and not fname: path='/'
     return base,path,fname
 
-# %% ../03_download.ipynb
+# %% ../03_download.ipynb #5337c0a2
 def _tryget(url):
     "Return response from `url` if `status_code!=404`, otherwise `None`"
     res = get(url)
     return None if res.status_code==404 else url
 
-# %% ../03_download.ipynb
+# %% ../03_download.ipynb #189f5b24
 def find_docs(url):
     "If available, return LLM-friendly llms.txt context or markdown file location from `url`"
     base,path,fname = split_url(url)
@@ -100,7 +100,7 @@ def find_docs(url):
     if parsed_url.path == '/' or not parsed_url.path: return None
     return find_docs(urljoin(url, '..'))
 
-# %% ../03_download.ipynb
+# %% ../03_download.ipynb #771d1208
 def read_docs(url, optional=False, n_workers=None, rm_comments=True, rm_details=True):
     "If available, return LLM-friendly llms.txt context or markdown file response for `url`"
     url = find_docs(url)

{toolslm-0.3.23 → toolslm-0.3.25}/toolslm/funccall.py

@@ -1,10 +1,10 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../01_funccall.ipynb.
 
-# %% auto 0
+# %% auto #0
 __all__ = ['empty', 'custom_types', 'get_schema', 'python', 'mk_ns', 'call_func', 'call_func_async', 'mk_param', 'schema2sig',
            'mk_tool']
 
-# %% ../01_funccall.ipynb
+# %% ../01_funccall.ipynb #e5ad6b86
 import inspect, json, ast
 from collections import abc
 from fastcore.utils import *
@@ -16,10 +16,10 @@ from inspect import Parameter, Signature
 from decimal import Decimal
 from uuid import UUID
 
-# %% ../01_funccall.ipynb
+# %% ../01_funccall.ipynb #a9f43047
 empty = inspect.Parameter.empty
 
-# %% ../01_funccall.ipynb
+# %% ../01_funccall.ipynb #e7bf4025
 def _types(t:type)->tuple[str,Optional[str]]:
     "Tuple of json schema type name and (if appropriate) array item name."
     if t is empty: raise TypeError('Missing type')
@@ -36,7 +36,7 @@ def _types(t:type)->tuple[str,Optional[str]]:
     # if t is the type itself like int, use the __name__ representation as the key
     else: return tmap.get(t.__name__, "object"), None
 
-# %% ../01_funccall.ipynb
+# %% ../01_funccall.ipynb #4d5dc245
 def _param(
     name, # param name
     info, # dict from docments
@@ -53,7 +53,7 @@ def _param(
         else: pschema["default"] = info.default
     return pschema
 
-# %% ../01_funccall.ipynb
+# %% ../01_funccall.ipynb #f77bea1c
 custom_types = {Path, bytes, Decimal, UUID}
 
 def _handle_type(t, defs):
@@ -66,17 +66,17 @@ def _handle_type(t, defs):
         return {'$ref': f'#/$defs/{t.__name__}'}
     return {'type': _types(t)[0]}
 
-# %% ../01_funccall.ipynb
+# %% ../01_funccall.ipynb #7fd6cd29
 def _is_container(t):
     "Check if type is a container (list, dict, tuple, set, Union)"
     origin = get_origin(t)
-    return origin in (list, dict, tuple, set, Union) if origin else False
+    return origin in (list, dict, tuple, set, Union, UnionType) if origin else False
 
 def _is_parameterized(t):
     "Check if type has arguments (e.g. list[int] vs list, dict[str, int] vs dict)"
     return _is_container(t) and (get_args(t) != ())
 
-# %% ../01_funccall.ipynb
+# %% ../01_funccall.ipynb #c1153f02
 def _handle_container(origin, args, defs):
     "Handle container types like dict, list, tuple, set, and Union"
     if origin is Union or origin is UnionType:
@@ -97,7 +97,7 @@ def _handle_container(origin, args, defs):
         return schema
     return None
 
-# %% ../01_funccall.ipynb
+# %% ../01_funccall.ipynb #e0840bf5
 def _process_property(name, obj, props, req, defs, evalable=False):
     "Process a single property of the schema"
     p = _param(name, obj, evalable=evalable)
@@ -110,7 +110,7 @@ def _process_property(name, obj, props, req, defs, evalable=False):
         # Non-container type or container without arguments
         p.update(_handle_type(obj.anno, defs))
 
-# %% ../01_funccall.ipynb
+# %% ../01_funccall.ipynb #38b0f97e
 def _get_nested_schema(obj, evalable=False, skip_hidden=False):
     "Generate nested JSON schema for a class or function"
     d = docments(obj, full=True)
@@ -127,7 +127,7 @@ def _get_nested_schema(obj, evalable=False, skip_hidden=False):
     if defs: schema['$defs'] = defs
     return schema
 
-# %% ../01_funccall.ipynb
+# %% ../01_funccall.ipynb #748da965
 def get_schema(
     f:Union[callable,dict], # Function to get schema for
     pname='input_schema',   # Key name for parameters
@@ -149,11 +149,11 @@ def get_schema(
         desc += f'\n\nReturns:\n- {ret_str}'
     return {"name": f.__name__, "description": desc, pname: schema}
 
-# %% ../01_funccall.ipynb
+# %% ../01_funccall.ipynb #873000d7
 import ast, time, signal, traceback
 from fastcore.utils import *
 
-# %% ../01_funccall.ipynb
+# %% ../01_funccall.ipynb #4703296a
 def _copy_loc(new, orig):
     "Copy location information from original node to new node and all children."
     new = ast.copy_location(new, orig)
@@ -162,7 +162,7 @@ def _copy_loc(new, orig):
         elif isinstance(o, list): setattr(new, field, [_copy_loc(value, orig) for value in o])
     return new
 
-# %% ../01_funccall.ipynb
+# %% ../01_funccall.ipynb #1574585f
 def _run(code:str, glb:dict=None, loc:dict=None):
     "Run `code`, returning final expression (similar to IPython)"
     tree = ast.parse(code)
@@ -185,7 +185,7 @@ def _run(code:str, glb:dict=None, loc:dict=None):
     if _result is not None: return _result
     return stdout_buffer.getvalue().strip()
 
-# %% ../01_funccall.ipynb
+# %% ../01_funccall.ipynb #81857615
 def python(
     code:str, # Code to execute
     glb:Optional[dict]=None, # Globals namespace
@@ -202,7 +202,7 @@ def python(
     except Exception as e: return traceback.format_exc()
     finally: signal.alarm(0)
 
-# %% ../01_funccall.ipynb
+# %% ../01_funccall.ipynb #782c4415
 def mk_ns(fs):
     if isinstance(fs, abc.Mapping): return fs
     merged = {}
@@ -211,7 +211,7 @@ def mk_ns(fs):
         elif callable(o) and hasattr(o, '__name__'): merged |= {o.__name__: o}
     return merged
 
-# %% ../01_funccall.ipynb
+# %% ../01_funccall.ipynb #6ef23838
 def _coerce_inputs(func, inputs):
     "Coerce inputs based on function type annotations"
     hints = get_type_hints(func) if hasattr(func, '__annotations__') else {}
@@ -223,7 +223,7 @@ def _coerce_inputs(func, inputs):
         else: res[k] = v
     return res
 
-# %% ../01_funccall.ipynb
+# %% ../01_funccall.ipynb #d817ab01
 def call_func(fc_name, fc_inputs, ns, raise_on_err=True):
     "Call the function `fc_name` with the given `fc_inputs` using namespace `ns`."
     if not isinstance(ns, abc.Mapping): ns = mk_ns(ns)
@@ -235,7 +235,7 @@ def call_func(fc_name, fc_inputs, ns, raise_on_err=True):
         if raise_on_err: raise e from None
         else: return traceback.format_exc()
 
-# %% ../01_funccall.ipynb
+# %% ../01_funccall.ipynb #7ac04e80-7bb9-4b52-8285-454684605d47
 async def call_func_async(fc_name, fc_inputs, ns, raise_on_err=True):
     "Awaits the function `fc_name` with the given `fc_inputs` using namespace `ns`."
     res = call_func(fc_name, fc_inputs, ns, raise_on_err=raise_on_err)
@@ -246,7 +246,7 @@ async def call_func_async(fc_name, fc_inputs, ns, raise_on_err=True):
             else: return traceback.format_exc()
     return res
 
-# %% ../01_funccall.ipynb
+# %% ../01_funccall.ipynb #ede7ea66
 def mk_param(nm, props, req):
     "Create a `Parameter` for `nm` with schema `props`"
     kind = Parameter.POSITIONAL_OR_KEYWORD if nm in req else Parameter.KEYWORD_ONLY
@@ -257,14 +257,14 @@ def mk_param(nm, props, req):
     else: anno = type_map.get(props.get('type'), Any)
     return Parameter(nm, kind, default=default, annotation=anno)
 
-# %% ../01_funccall.ipynb
+# %% ../01_funccall.ipynb #a8befff6
 def schema2sig(tool):
     "Convert json schema `tool` to a `Signature`"
     props, req = tool.inputSchema['properties'], tool.inputSchema.get('required', [])
     params = sorted([mk_param(k, v, req) for k, v in props.items()], key=lambda p: p.kind)
     return Signature(params)
 
-# %% ../01_funccall.ipynb
+# %% ../01_funccall.ipynb #bb16561a
 def mk_tool(dispfn, tool):
     "Create a callable function from a JSON schema tool definition"
     sig = schema2sig(tool)

toolslm-0.3.25/toolslm/inspecttools.py

@@ -0,0 +1,239 @@
+"""LLM tools for inspecting the symbol table and importing modules"""
+
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../05_inspecttools.ipynb.
+
+# %% auto #0
+__all__ = ['importmodule', 'SymbolNotFound', 'resolve', 'symsrc', 'symtype', 'symval', 'symtype_val', 'symdir', 'symnth',
+           'symlen', 'symslice', 'symsearch', 'symset', 'symfiles_folder', 'symfiles_package']
+
+# %% ../05_inspecttools.ipynb #5ae7ad05
+from fastcore.utils import *
+from fastcore.meta import delegates
+import inspect, re, sys, ast, builtins, os, linecache
+from importlib import import_module
+from .xml import *
+
+# %% ../05_inspecttools.ipynb #9778fca8
+def _find_frame_dict(var:str):
+    "Find the dict (globals or locals) containing var"
+    frame = inspect.currentframe().f_back
+    while frame:
+        if var in frame.f_globals: return frame.f_globals
+        frame = frame.f_back
+    raise ValueError(f"Could not find {var} in any scope")
+
+# %% ../05_inspecttools.ipynb #7fd415fa
+def importmodule(
+    mod: str, # The module to import (e.g. 'torch.nn.functional')
+    caller_symbol:str = '__msg_id'  # The name of the special variable to find the correct caller namespace
+):
+    """Import a module into the caller's global namespace so it's available for `symsrc`, `symval`, `symdir`, etc.
+    Use this before inspecting or using symbols from modules not yet imported."""
+    g = _find_frame_dict(caller_symbol)
+    import_module(mod)
+    g[mod.split('.')[0]] = import_module(mod.split('.')[0])
+
+# %% ../05_inspecttools.ipynb #e40f68c9
+class SymbolNotFound(Exception):
+    def __repr__(self): return f"SymbolNotFound({self.args[0]})"
+    __str__ = __repr__
+
+_last = None
+
+def resolve(
+    sym: str  # Dotted symbol path, with optional [n] indexing, e.g. "module.attr.subattr[1]" or "_last" for previous result
+):
+    """Resolve a dotted symbol string to its Python object, with optional [n] indexing.
+    Sets global `_last` to the resolved object for chaining.
+    Pass `"_last"` to reference the result of the previous tool call.
+
+    Examples:
+
+    - `resolve("sympy.sets.sets.Interval")` -> `<class 'sympy.sets.sets.Interval'>`
+    - `resolve("mylist[2]")` -> third element of mylist"""
+    global _last
+    if (sym := sym.strip()) == '_last': return _last
+    g = _find_frame_dict('__msg_id')
+    if match := re.match(r'^(\w+)\[(\d+)\]$', sym):
+        attr, idx = match.groups()
+        parts, _last = ['_last'], _last[int(idx)] if attr == '_last' else g[attr][int(idx)]
+    else: parts = re.split(r'\.(?![^\[]*\])', sym)
+    try: obj = _last if parts[0] == '_last' else g[parts[0]]
+    except KeyError: raise SymbolNotFound(f"Symbol '{parts[0]}' not found. Consider using `importmodule` first.")
+    for part in parts[1:]:
+        match = re.match(r'(\w+)\[(\d+)\]$', part)
+        if match:
+            attr, idx = match.groups()
+            obj = getattr(obj, attr)[int(idx)]
+        else: obj = getattr(obj, part)
+    _last = obj
+    return obj
+
+# %% ../05_inspecttools.ipynb #659bf879
+def _src_from_lines(lines, start):
+    "Extract a single definition from lines starting at start (0-indexed)"
+    src = ''.join(lines[start:])
+    try: tree = ast.parse(src)
+    except SyntaxError: return None
+    if not tree.body: return None
+    return ''.join(lines[start:start + tree.body[0].end_lineno])
+
+# %% ../05_inspecttools.ipynb #41ecbd5c
+def symsrc(
+    sym: str  # Dotted symbol path (e.g `Interval` or `sympy.sets.sets.Interval`) or "_last" for previous result
+):
+    """Get the source code for a symbol.
+
+    Examples:
+
+    - `symsrc("Interval")` -> source code of Interval class if it's already imported
+    - `symsrc("sympy.sets.sets.Interval")` -> source code of Interval class
+    - `symsrc("_last")` -> source of object from previous tool call
+    - For dispatchers or registries of callables: `symnth("module.dispatcher.funcs", n) then symsrc("_last")`"""
+    try: obj = resolve(sym)
+    except SymbolNotFound as e: return str(e)
+    if isinstance(obj, type) or callable(obj): pass
+    elif hasattr(obj, '__module__') and not inspect.ismodule(obj): obj = obj.__class__
+    try: fname = inspect.getfile(obj)
+    except (OSError, TypeError): fname = "<session>"
+    try: return f"File: {fname}\n\n{inspect.getsource(obj)}"
+    except (OSError, TypeError): pass
+    name = getattr(obj, '__name__', None)
+    if not name: raise OSError(f"Cannot get source for {sym}")
+    pat = rf'^(class|def)\s+{name}\b'
+    for fname, (_, _, lines, _) in linecache.cache.items():
+        src = ''.join(lines)
+        if match := re.search(pat, src, re.MULTILINE):
+            start = src[:match.start()].count('\n')
+            if extracted := _src_from_lines(lines, start): return f"File: {fname}\n\n{extracted}"
+    raise OSError(f"Source for {name} not found")
+
+# %% ../05_inspecttools.ipynb #bbf67405
+def symtype(
+    syms: str  # Comma separated str list of dotted symbol paths (e.g `'Interval,a'` or `'sympy.sets.sets.Interval'`); "_last" for prev result
+):
+    """Get the type of a symbol and set `_last`.
+
+    Examples:
+
+    - `symtype("sympy.sets.sets.Interval")` -> `<class 'type'>`
+    - `symtype("doesnotexist")` -> `'SymbolNotFound`
+    - `symtype("_last")` -> type of previous result"""
+    def f(o):
+        try: return type(resolve(o))
+        except SymbolNotFound as e: return str(e)
+    return [f(o) for o in re.split(r'\,\s*', syms)]
+
+# %% ../05_inspecttools.ipynb #dd4279b8
+def symval(
+    syms: str  # Comma separated str list of dotted symbol paths (e.g `Interval` or `sympy.sets.sets.Interval`); "_last" for prev result
+):
+    """List of repr of symbols' values.
+
+    Examples:
+    
+    - `symval("sympy.sets.sets.Interval")` -> `[<class 'sympy.sets.sets.Interval'>]`
+    - `symval("some_dict.keys")` -> `[dict_keys([...])]`
+    - `symval("a,notexist")` -> `['foo','SymbolNotFound']`"""
+    def f(o):
+        try: return repr(resolve(o))
+        except SymbolNotFound as e: return str(e)
+    return [f(o) for o in re.split(r'\,\s*', syms)]
+
+# %% ../05_inspecttools.ipynb #3cb05b78
+def symtype_val(
+    syms: str  # Comma separated str list of dotted symbol paths (e.g `Interval` or `sympy.sets.sets.Interval`); "_last" for prev result
+):
+    """List of 2-ple of (type,repr) of symbols' values.
+
+    Examples:
+    
+    - `symtype_val("a,c,notexist")` -> `[(<class 'str'>,'foo'),(<class 'int'>,1), 'SymbolNotFound']`"""
+    def f(o):
+        try: r = resolve(o)
+        except SymbolNotFound as e: return 'SymbolNotFound'
+        return (type(r), repr(r))
+    return [f(o) for o in re.split(r'\,\s*', syms)]
+
+# %% ../05_inspecttools.ipynb #1cd34596
+def symdir(
+    sym: str,  # Dotted symbol path (e.g `Interval` or `sympy.sets.sets.Interval`) or "_last" for previous result
+    exclude_private: bool=False # Filter out attrs starting with "_"
+):
+    """Get dir() listing of a symbol's attributes and set `_last`. E.g: `symdir("sympy.Interval")` -> `['__add__', '__and__', ...]`"""
+    res = dir(resolve(sym))
+    if not exclude_private: return res
+    return [o for o in res if o[0]!='_']
+
+# %% ../05_inspecttools.ipynb #9542de0b
+def symnth(
+    sym: str,  # Dotted symbol path to a dict or object with .values()
+    n: int     # Index into the values (0-based)
+):
+    """Get the nth value from a dict (or any object with .values()). Sets `_last` so you can chain with `symsrc("_last")` etc.
+
+    Examples:
+    
+    - `symnth("dispatcher.funcs", 12)` -> 13th registered function
+    - `symnth("dispatcher.funcs", 0); symsrc("_last")` -> source of first handler"""
+    global _last
+    _last = list(resolve(sym).values())[n]
+    return _last
+
+# %% ../05_inspecttools.ipynb #4ac6ca2d
+def symlen(
+    sym: str  # Dotted symbol path or "_last" for previous result
+):
+    "Returns the length of the given symbol"
+    return len(resolve(sym))
+
+# %% ../05_inspecttools.ipynb #a5dfbe8f
+def symslice(
+    sym: str,   # Dotted symbol path or "_last" for previous result
+    start: int, # Starting index for slice
+    end: int    # Ending index for slice
+):
+    "Returns the contents of the symbol from the given start to the end."
+    try: return resolve(sym)[start:end]
+    except Exception as e: return f'Error: {e}'
+
+# %% ../05_inspecttools.ipynb #5fca4d70
+def symsearch(
+    sym:str,      # Dotted symbol path or "_last" for previous result
+    term:str,     # Search term (exact string or regex pattern)
+    regex:bool=True,  # If True, regex search; if False, exact match
+    flags:int=0   # Regex flags (e.g., re.IGNORECASE)
+):
+    """Search contents of symbol, which is assumed to be str for regex, or iterable for non-regex.
+    Regex mode returns (match, start, end) tuples; otherwise returns (item, index) tuples"""
+    if regex: return str([(m.group(), m.start(), m.end()) for m in re.finditer(term, resolve(sym), flags)])
+    else: return str([(x, i) for i, x in enumerate(resolve(sym)) if x == term])
+
+# %% ../05_inspecttools.ipynb #02c09e1a
+def symset(
+    val: str  # Value to assign to _ai_sym
+):
+    "Set _ai_sym to the given value"
+    _find_frame_dict('__msg_id')['_ai_sym'] = val
+
+# %% ../05_inspecttools.ipynb #72e5f0a8
+@delegates(sym2folderctx)
+def symfiles_folder(
+    sym:str,      # Dotted symbol path or "_last" for previous result
+    **kwargs
+):
+    "Return XML context of files in the folder containing `sym`'s definition"
+    try: s = resolve(sym)
+    except SymbolNotFound as e: return str(e)
+    return sym2folderctx(s, **kwargs)
+
+# %% ../05_inspecttools.ipynb #fdd0990e
+@delegates(sym2pkgctx)
+def symfiles_package(
+    sym:str,      # Dotted symbol path or "_last" for previous result
+    **kwargs
+):
+    "Return XML context of all files in `sym`'s top-level package"
+    try: s = resolve(sym)
+    except SymbolNotFound as e: return str(e)
+    return sym2pkgctx(s, **kwargs)

{toolslm-0.3.23 → toolslm-0.3.25}/toolslm/shell.py

@@ -1,20 +1,20 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../02_shell.ipynb.
 
-# %% auto 0
+# %% auto #0
 __all__ = ['get_shell']
 
-# %% ../02_shell.ipynb
+# %% ../02_shell.ipynb #1328ef69
 import ast, time, signal, traceback
 from fastcore.utils import *
 
-# %% ../02_shell.ipynb
+# %% ../02_shell.ipynb #6bbf062d
 from IPython.terminal.interactiveshell import TerminalInteractiveShell
 from IPython.utils.capture import capture_output
 
-# %% ../02_shell.ipynb
+# %% ../02_shell.ipynb #34099c2f
 TerminalInteractiveShell.orig_run = TerminalInteractiveShell.run_cell
 
-# %% ../02_shell.ipynb
+# %% ../02_shell.ipynb #d6aa8e7b
 @patch
 def run_cell(self:TerminalInteractiveShell, cell, timeout=None):
     "Wrapper for original `run_cell` which adds timeout and output capture"
@@ -31,7 +31,7 @@ def run_cell(self:TerminalInteractiveShell, cell, timeout=None):
     finally:
         if timeout: signal.alarm(0)
 
-# %% ../02_shell.ipynb
+# %% ../02_shell.ipynb #cdadbb12
 def get_shell()->TerminalInteractiveShell:
     "Get a `TerminalInteractiveShell` with minimal functionality"
     sh = TerminalInteractiveShell()

{toolslm-0.3.23 → toolslm-0.3.25}/toolslm/xml.py

@@ -1,11 +1,11 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../00_xml.ipynb.
 
-# %% auto 0
-__all__ = ['doctype', 'json_to_xml', 'get_mime_text', 'cell2out', 'cell2xml', 'cells2xml', 'nb2xml', 'get_docstring', 'py2sigs',
-           'mk_doctype', 'mk_doc', 'docs_xml', 'read_file', 'files2ctx', 'folder2ctx', 'sym2file', 'sym2folderctx',
-           'sym2pkgpath', 'sym2pkgctx', 'folder2ctx_cli', 'parse_gh_url', 'repo2ctx', 'repo2ctx_cli']
+# %% auto #0
+__all__ = ['doctype', 'json_to_xml', 'get_mime_text', 'cell2out', 'cell2xml', 'cells2xml', 'nb2xml', 'mk_doctype', 'mk_doc',
+           'docs_xml', 'read_file', 'files2ctx', 'folder2ctx', 'sym2file', 'sym2folderctx', 'sym2pkgpath', 'sym2pkgctx',
+           'folder2ctx_cli', 'parse_gh_url', 'repo2ctx', 'repo2ctx_cli']
 
-# %% ../00_xml.ipynb
+# %% ../00_xml.ipynb #033c76fd
 import hashlib, inspect, xml.etree.ElementTree as ET, ast
 from collections import namedtuple
 from ghapi.all import GhApi
@@ -16,7 +16,9 @@ from fastcore.xtras import hl_md
 from fastcore.xml import to_xml, Document, Documents, Document_content, Src, Source,Out,Outs,Cell,Notebook,Md,Code,Raw
 from fastcore.script import call_parse
 
-# %% ../00_xml.ipynb
+from codesigs import file_sigs
+
+# %% ../00_xml.ipynb #2795f9fc
 def json_to_xml(d:dict, # JSON dictionary to convert
                 rnm:str # Root name
                )->str:
@@ -32,13 +34,13 @@ def json_to_xml(d:dict, # JSON dictionary to convert
     ET.indent(root)
     return ET.tostring(root, encoding='unicode')
 
-# %% ../00_xml.ipynb
+# %% ../00_xml.ipynb #338af2a5
 def get_mime_text(data):
     "Get text from MIME bundle, preferring markdown over plain"
     if 'text/markdown' in data: return ''.join(list(data['text/markdown']))
     if 'text/plain' in data: return ''.join(list(data['text/plain']))
 
-# %% ../00_xml.ipynb
+# %% ../00_xml.ipynb #aff831f8
 def cell2out(o):
     "Convert single notebook output to XML format"
     if hasattr(o, 'data'): 
@@ -49,7 +51,7 @@ def cell2out(o):
         return Out(txt, type='stream', name=o.get('name', 'stdout'))
     if hasattr(o, 'ename'): return Out(f"{o.ename}: {o.evalue}", type='error')
 
-# %% ../00_xml.ipynb
+# %% ../00_xml.ipynb #ac3ed87b
 _ctfuns = {'code': Code, 'markdown': Md, 'raw': Raw}
 
 def cell2xml(cell, out=True, ids=True, nums=False):
@@ -64,7 +66,7 @@ def cell2xml(cell, out=True, ids=True, nums=False):
     if out_items: parts.append(Outs(*out_items))
     return f(*parts, **kw)
 
-# %% ../00_xml.ipynb
+# %% ../00_xml.ipynb #f780f576
 @delegates(cell2xml)
 def cells2xml(cells, wrap=Notebook, **kwargs):
     "Convert notebook to XML format"
@@ -78,32 +80,10 @@ def nb2xml(fname=None, nb=None, **kwargs):
     if not nb: nb = dict2obj(fname.read_json())
     return cells2xml(nb.cells, **kwargs)
 
-# %% ../00_xml.ipynb
-def get_docstring(node, lines):
-    "Get docstring from source lines if present"
-    if not (node.body and isinstance(node.body[0], ast.Expr) and isinstance(node.body[0].value, ast.Constant)): return None
-    doc_node = node.body[0]
-    return '\n'.join(lines[doc_node.lineno-1:doc_node.end_lineno])
-
-def py2sigs(fname=None, src=None):
-    "Return signature+docstring text for all functions and class methods in source"
-    if fname: src = Path(fname).expanduser().read_text()
-    tree = ast.parse(src)
-    lines = src.splitlines()
-    res = []
-    for node in ast.walk(tree):
-        if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
-            body_start = max(node.body[0].lineno - 1, node.lineno)
-            sig = '\n'.join(lines[node.lineno-1:body_start])
-            doc = get_docstring(node, lines)
-            cts = f"{sig}\n{doc}" if doc else sig
-            res.append(cts.strip('\r\n'))
-    return '\n\n'.join(res)
-
-# %% ../00_xml.ipynb
+# %% ../00_xml.ipynb #a01dc320
 doctype = namedtuple('doctype', ['src', 'content'])
 
-# %% ../00_xml.ipynb
+# %% ../00_xml.ipynb #ce853491
 def _add_nls(s):
     "Add newlines to start and end of `s` if missing"
     if not s: return s
@@ -111,7 +91,7 @@ def _add_nls(s):
     if s[-1]!='\n': s = s+'\n'
     return s
 
-# %% ../00_xml.ipynb
+# %% ../00_xml.ipynb #932e8858
 def mk_doctype(content:str,  # The document content
            src:Optional[str]=None # URL, filename, etc; defaults to `md5(content)` if not provided
           ) -> namedtuple:
@@ -119,7 +99,7 @@ def mk_doctype(content:str,  # The document content
     if src is None: src = hashlib.md5(content.encode()).hexdigest()[:8]
     return doctype(_add_nls(str(src).strip()), _add_nls(content.strip()))
 
-# %% ../00_xml.ipynb
+# %% ../00_xml.ipynb #15e454db
 def mk_doc(index:int,  # The document index
            content:str,  # The document content
            src:Optional[str]=None, # URL, filename, etc; defaults to `md5(content)` if not provided
@@ -131,7 +111,7 @@ def mk_doc(index:int,  # The document index
     src = Src(NotStr(dt.src))
     return Document(src, content, index=index, **kwargs)
 
-# %% ../00_xml.ipynb
+# %% ../00_xml.ipynb #32237f0a
 def docs_xml(docs:list[str],  # The content of each document
              srcs:Optional[list]=None,  # URLs, filenames, etc; each one defaults to `md5(content)` if not provided
              prefix:bool=False, # Include Anthropic's suggested prose intro?
@@ -146,18 +126,20 @@ def docs_xml(docs:list[str],  # The content of each document
     kw = dict(title=title) if title else {}
     return pre + to_xml(Documents(*docs, **kw), do_escape=False)
 
-# %% ../00_xml.ipynb
+# %% ../00_xml.ipynb #278b484b
 @delegates(nb2xml)
 def read_file(fname, max_size=None, sigs_only=False, **kwargs):
     "Read file content, converting notebooks to XML if needed"
     fname = Path(fname).expanduser()
     if fname.suffix == '.ipynb': res = nb2xml(fname, **kwargs)
-    elif fname.suffix == '.py' and sigs_only: res = py2sigs(fname)
-    else: res = fname.read_text()
+    elif sigs_only: res = '\n'.join(str(s) for s in file_sigs(fname))
+    else:
+        try: res = fname.read_text()
+        except UnicodeDecodeError: return f"[Skipped: {fname.name} is binary]"
     if max_size and len(res)>max_size: return f"[Skipped: {fname.name} exceeds {max_size} bytes]"
     return res
 
-# %% ../00_xml.ipynb
+# %% ../00_xml.ipynb #7d92255e
 @delegates(docs_xml)
 def files2ctx(
     fnames:list[Union[str,Path]], # List of file names to add to context
@@ -166,7 +148,7 @@ def files2ctx(
     out:bool=True, # Include notebook cell outputs?
     ids:bool=True,  # Include cell ids in notebooks?
     nums:bool=False, # Include line numbers in notebook cell source?
-    sigs_only:bool=False, # For .py files, only include signatures and docstrings
+    sigs_only:bool=False, # Only include signatures and docstrings (where supported by `codesigs` lib)
     **kwargs
 )->str: # XML for LM context
     "Convert files to XML context, handling notebooks"
@@ -174,7 +156,7 @@ def files2ctx(
     contents = [read_file(o, max_size=max_size, out=out, ids=ids, sigs_only=sigs_only, nums=nums) for o in fnames]
     return docs_xml(contents, srcs or fnames, **kwargs)
 
-# %% ../00_xml.ipynb
+# %% ../00_xml.ipynb #f97bc488
 @delegates(globtastic, but='func')
 def folder2ctx(
     path:Union[str,Path], # Folder to read
@@ -186,7 +168,7 @@ def folder2ctx(
     max_total:int=10_000_000,  # Max total output size in bytes
     readme_first:bool=False,  # Prioritize README files at start of context?
     files_only:bool=False,  # Return dict of {filename: size} instead of context?
-    sigs_only:bool=False,  # Return signatures instead of full text for python files?
+    sigs_only:bool=False,  # Return signatures instead of full text? (where supported by `codesigs` lib)
     ids:bool=True,  # Include cell ids in notebooks?
     **kwargs
 )->Union[str,dict]:
@@ -201,13 +183,13 @@ def folder2ctx(
     if max_total and len(res) > max_total: res = truncstr(res, max_total, suf=suf, sizevar='_outsz_')
     return res
 
-# %% ../00_xml.ipynb
+# %% ../00_xml.ipynb #9dc68935
 def sym2file(sym):
     "Return md string with filepath and contents for a symbol's source file"
     f = Path(inspect.getfile(sym))
     return f"- `{f}`\n\n````\n{f.read_text()}\n````"
 
-# %% ../00_xml.ipynb
+# %% ../00_xml.ipynb #daef58e8
 @delegates(folder2ctx)
 def sym2folderctx(
     sym,
@@ -217,19 +199,20 @@ def sym2folderctx(
     "Return folder context for a symbol's source file location"
     return folder2ctx(Path(inspect.getfile(sym)).parent, types=types, skip_file_re=skip_file_re, **kwargs)
 
-# %% ../00_xml.ipynb
+# %% ../00_xml.ipynb #9afa301c
 def sym2pkgpath(sym):
     "Get root package path for a symbol"
-    root = sym.__module__.split('.')[0]
+    mod = sym.__name__ if inspect.ismodule(sym) else sym.__module__
+    root = mod.split('.')[0]
     return Path(sys.modules[root].__path__[0])
 
-# %% ../00_xml.ipynb
+# %% ../00_xml.ipynb #798ca61f
 @delegates(folder2ctx)
-def sym2pkgctx(sym, types:str|list='py', skip_file_re=r'^_mod', **kwargs):
-    "Return repo context for a symbol's root package"
-    return folder2ctx(sym2pkgpath(sym), types=types, skip_file_re=skip_file_re, **kwargs)
+def sym2pkgctx(sym, types:str|list='py', skip_file_re=r'^_mod', skip_folder_re=r'^(\.|__)', **kwargs):
+    "Return contents of files in a symbol's root package"
+    return folder2ctx(sym2pkgpath(sym), types=types, skip_file_re=skip_file_re, skip_folder_re=skip_folder_re, **kwargs)
 
-# %% ../00_xml.ipynb
+# %% ../00_xml.ipynb #6a786c55
 @call_parse
 @delegates(folder2ctx)
 def folder2ctx_cli(
@@ -240,13 +223,13 @@ def folder2ctx_cli(
     "CLI to convert folder contents to XML context, handling notebooks"
     print(folder2ctx(path, out=out, **kwargs))
 
-# %% ../00_xml.ipynb
+# %% ../00_xml.ipynb #d10ed2c1
 def parse_gh_url(url):
     "Parse GitHub URL into (owner, repo, type, ref, path) or None"
     m = re.match(r'https?://(?:www\.)?github\.com/([^/]+)/([^/]+)(?:/([^/]+)(?:/([^/]+)(?:/(.+))?)?)?', url)
     return dict(zip('owner repo typ ref path'.split(), m.groups())) if m else None
 
-# %% ../00_xml.ipynb
+# %% ../00_xml.ipynb #d91934db
 @delegates(folder2ctx)
 def repo2ctx(
     owner:str,  # GitHub repo owner or "owner/repo" or a full github URL
@@ -281,7 +264,7 @@ def repo2ctx(
         if folder: subdir = subdir/folder
         return folder2ctx(subdir, include_base=False, title=title, readme_first=True, **kwargs)
 
-# %% ../00_xml.ipynb
+# %% ../00_xml.ipynb #7654e554
 @call_parse
 @delegates(repo2ctx, but='include_base,title,readme_first')
 def repo2ctx_cli(

{toolslm-0.3.23 → toolslm-0.3.25/toolslm.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: toolslm
-Version: 0.3.23
+Version: 0.3.25
 Summary: Tools to make language models a bit easier to use
 Home-page: https://github.com/AnswerDotAI/toolslm
 Author: Jeremy Howard
@@ -19,7 +19,9 @@ License-File: LICENSE
 Requires-Dist: fastcore>=1.9.7
 Requires-Dist: httpx
 Requires-Dist: ghapi
+Requires-Dist: codesigs
 Provides-Extra: dev
+Requires-Dist: ipython; extra == "dev"
 Dynamic: author
 Dynamic: author-email
 Dynamic: classifier

{toolslm-0.3.23 → toolslm-0.3.25}/toolslm.egg-info/SOURCES.txt

@@ -8,6 +8,7 @@ toolslm/__init__.py
 toolslm/_modidx.py
 toolslm/download.py
 toolslm/funccall.py
+toolslm/inspecttools.py
 toolslm/md_hier.py
 toolslm/shell.py
 toolslm/xml.py

{toolslm-0.3.23 → toolslm-0.3.25}/toolslm.egg-info/requires.txt

@@ -1,5 +1,7 @@
 fastcore>=1.9.7
 httpx
 ghapi
+codesigs
 
 [dev]
+ipython

toolslm-0.3.23/toolslm/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.3.23"