toolslm 0.2.1__tar.gz → 0.2.3__tar.gz

{toolslm-0.2.1/toolslm.egg-info → toolslm-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: toolslm
-Version: 0.2.1
+Version: 0.2.3
 Summary: Tools to make language models a bit easier to use
 Home-page: https://github.com/AnswerDotAI/toolslm
 Author: Jeremy Howard

{toolslm-0.2.1 → toolslm-0.2.3}/settings.ini

@@ -1,7 +1,7 @@
 [DEFAULT]
 repo = toolslm
 lib_name = toolslm
-version = 0.2.1
+version = 0.2.3
 min_python = 3.9
 license = apache2
 black_formatting = False
@@ -35,7 +35,7 @@ clean_ids = True
 clear_all = False
 conda_user = fastai
 console_scripts = folder2ctx=toolslm.xml:folder2ctx_cli
-cell_number = True
+cell_number = False
 skip_procs = 
 update_pyproject = True
 

toolslm-0.2.3/toolslm/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.3"

{toolslm-0.2.1 → toolslm-0.2.3}/toolslm/download.py

@@ -3,26 +3,26 @@
 # %% auto 0
 __all__ = ['clean_md', 'read_md', 'html2md', 'read_html', 'get_llmstxt', 'split_url', 'find_docs', 'read_docs']
 
-# %% ../03_download.ipynb 2
+# %% ../03_download.ipynb
 from fastcore.utils import *
 from httpx import get
 from fastcore.meta import delegates
 from urllib.parse import urlparse, urljoin
 
-# %% ../03_download.ipynb 4
+# %% ../03_download.ipynb
 def clean_md(text, rm_comments=True, rm_details=True):
     "Remove comments and `<details>` sections from `text`"
     if rm_comments: text = re.sub(r'\n?<!--.*?-->\n?', '', text, flags=re.DOTALL)
     if rm_details: text = re.sub(r'\n?<details>.*?</details>\n?', '', text, flags=re.DOTALL)
     return text
 
-# %% ../03_download.ipynb 5
+# %% ../03_download.ipynb
 @delegates(get)
 def read_md(url, rm_comments=True, rm_details=True, **kwargs):
     "Read text from `url` and clean with `clean_docs`"
     return clean_md(get(url, **kwargs).text, rm_comments=rm_comments, rm_details=rm_details)
 
-# %% ../03_download.ipynb 7
+# %% ../03_download.ipynb
 def html2md(s:str, ignore_links=True):
     "Convert `s` from HTML to markdown"
     import html2text
@@ -32,7 +32,7 @@ def html2md(s:str, ignore_links=True):
     o.ignore_images = True
     return o.handle(s)
 
-# %% ../03_download.ipynb 8
+# %% ../03_download.ipynb
 def read_html(url, # URL to read
               sel=None, # Read only outerHTML of CSS selector `sel`
               rm_comments=True, # Removes HTML comments
@@ -54,7 +54,7 @@ def read_html(url, # URL to read
     if wrap_tag: return '\n'.join([f"\n<{wrap_tag}>\n{o}</{wrap_tag}>\n" for o in mds])
     else: return'\n'.join(mds)
 
-# %% ../03_download.ipynb 13
+# %% ../03_download.ipynb
 def get_llmstxt(url, optional=False, n_workers=None):
     "Get llms.txt file from and expand it with `llms_txt.create_ctx()`"
     if not url.endswith('llms.txt'): return None
@@ -63,7 +63,7 @@ def get_llmstxt(url, optional=False, n_workers=None):
     if resp.status_code!=200: return None
     return llms_txt.create_ctx(resp.text, optional=optional, n_workers=n_workers)
 
-# %% ../03_download.ipynb 15
+# %% ../03_download.ipynb
 def split_url(url):
     "Split `url` into base, path, and file name, normalising name to '/' if empty"
     parsed = urlparse(url.strip('/'))
@@ -73,13 +73,13 @@ def split_url(url):
     if not path and not fname: path='/'
     return base,path,fname
 
-# %% ../03_download.ipynb 17
+# %% ../03_download.ipynb
 def _tryget(url):
     "Return response from `url` if `status_code!=404`, otherwise `None`"
     res = get(url)
     return None if res.status_code==404 else url
 
-# %% ../03_download.ipynb 18
+# %% ../03_download.ipynb
 def find_docs(url):
     "If available, return LLM-friendly llms.txt context or markdown file location from `url`"
     base,path,fname = split_url(url)
@@ -99,7 +99,7 @@ def find_docs(url):
     if parsed_url.path == '/' or not parsed_url.path: return None
     return find_docs(urljoin(url, '..'))
 
-# %% ../03_download.ipynb 23
+# %% ../03_download.ipynb
 def read_docs(url, optional=False, n_workers=None, rm_comments=True, rm_details=True):
     "If available, return LLM-friendly llms.txt context or markdown file response for `url`"
     url = find_docs(url)

{toolslm-0.2.1 → toolslm-0.2.3}/toolslm/funccall.py

@@ -3,7 +3,7 @@
 # %% auto 0
 __all__ = ['empty', 'custom_types', 'get_schema', 'PathArg', 'python', 'mk_ns', 'call_func', 'call_func_async']
 
-# %% ../01_funccall.ipynb 2
+# %% ../01_funccall.ipynb
 import inspect
 from collections import abc
 from fastcore.utils import *
@@ -11,10 +11,10 @@ from fastcore.docments import docments
 from typing import get_origin, get_args, Dict, List, Optional, Tuple, Union
 from types import UnionType
 
-# %% ../01_funccall.ipynb 4
+# %% ../01_funccall.ipynb
 empty = inspect.Parameter.empty
 
-# %% ../01_funccall.ipynb 12
+# %% ../01_funccall.ipynb
 def _types(t:type)->tuple[str,Optional[str]]:
     "Tuple of json schema type name and (if appropriate) array item name."
     if t is empty: raise TypeError('Missing type')
@@ -31,7 +31,7 @@ def _types(t:type)->tuple[str,Optional[str]]:
     # if t is the type itself like int, use the __name__ representation as the key
     else: return tmap.get(t.__name__, "object"), None
 
-# %% ../01_funccall.ipynb 19
+# %% ../01_funccall.ipynb
 def _param(name, info):
     "json schema parameter given `name` and `info` from docments full dict."
     paramt,itemt = _types(info.anno)
@@ -40,19 +40,20 @@ def _param(name, info):
     if info.default is not empty: pschema["default"] = info.default
     return pschema
 
-# %% ../01_funccall.ipynb 22
+# %% ../01_funccall.ipynb
 custom_types = {Path}
 
 def _handle_type(t, defs):
     "Handle a single type, creating nested schemas if necessary"
     if t is NoneType: return {'type': 'null'}
     if t in custom_types: return {'type':'string', 'format':t.__name__}
+    if t in (dict, list, tuple, set): return {'type': _types(t)[0]}
     if isinstance(t, type) and not issubclass(t, (int, float, str, bool)) or inspect.isfunction(t):
         defs[t.__name__] = _get_nested_schema(t)
         return {'$ref': f'#/$defs/{t.__name__}'}
     return {'type': _types(t)[0]}
 
-# %% ../01_funccall.ipynb 24
+# %% ../01_funccall.ipynb
 def _is_container(t):
     "Check if type is a container (list, dict, tuple, set, Union)"
     origin = get_origin(t)
@@ -62,7 +63,7 @@ def _is_parameterized(t):
     "Check if type has arguments (e.g. list[int] vs list, dict[str, int] vs dict)"
     return _is_container(t) and (get_args(t) != ())
 
-# %% ../01_funccall.ipynb 30
+# %% ../01_funccall.ipynb
 def _handle_container(origin, args, defs):
     "Handle container types like dict, list, tuple, set, and Union"
     if origin is Union or origin is UnionType:
@@ -83,7 +84,7 @@ def _handle_container(origin, args, defs):
         return schema
     return None
 
-# %% ../01_funccall.ipynb 31
+# %% ../01_funccall.ipynb
 def _process_property(name, obj, props, req, defs):
     "Process a single property of the schema"
     p = _param(name, obj)
@@ -91,12 +92,12 @@ def _process_property(name, obj, props, req, defs):
     if obj.default is empty: req[name] = True
 
     if _is_container(obj.anno) and _is_parameterized(obj.anno):
-            p.update(_handle_container(get_origin(obj.anno), get_args(obj.anno), defs))        
+        p.update(_handle_container(get_origin(obj.anno), get_args(obj.anno), defs))        
     else:
         # Non-container type or container without arguments
         p.update(_handle_type(obj.anno, defs))
 
-# %% ../01_funccall.ipynb 32
+# %% ../01_funccall.ipynb
 def _get_nested_schema(obj):
     "Generate nested JSON schema for a class or function"
     d = docments(obj, full=True)
@@ -111,7 +112,7 @@ def _get_nested_schema(obj):
     if defs: schema['$defs'] = defs
     return schema
 
-# %% ../01_funccall.ipynb 36
+# %% ../01_funccall.ipynb
 def get_schema(f:Union[callable,dict], pname='input_schema')->dict:
     "Generate JSON schema for a class, function, or method"
     if isinstance(f, dict): return f
@@ -123,16 +124,16 @@ def get_schema(f:Union[callable,dict], pname='input_schema')->dict:
     if ret.anno is not empty: desc += f'\n\nReturns:\n- type: {_types(ret.anno)[0]}'
     return {"name": f.__name__, "description": desc, pname: schema}
 
-# %% ../01_funccall.ipynb 47
+# %% ../01_funccall.ipynb
 def PathArg(
     path: str  # A filesystem path
 ): return Path(path)
 
-# %% ../01_funccall.ipynb 67
+# %% ../01_funccall.ipynb
 import ast, time, signal, traceback
 from fastcore.utils import *
 
-# %% ../01_funccall.ipynb 68
+# %% ../01_funccall.ipynb
 def _copy_loc(new, orig):
     "Copy location information from original node to new node and all children."
     new = ast.copy_location(new, orig)
@@ -141,7 +142,7 @@ def _copy_loc(new, orig):
         elif isinstance(o, list): setattr(new, field, [_copy_loc(value, orig) for value in o])
     return new
 
-# %% ../01_funccall.ipynb 70
+# %% ../01_funccall.ipynb
 def _run(code:str, glb:dict=None, loc:dict=None):
     "Run `code`, returning final expression (similar to IPython)"
     tree = ast.parse(code)
@@ -164,14 +165,14 @@ def _run(code:str, glb:dict=None, loc:dict=None):
     if _result is not None: return _result
     return stdout_buffer.getvalue().strip()
 
-# %% ../01_funccall.ipynb 75
-def python(code:str, # Code to execute
-           glb:Optional[dict]=None, # Globals namespace
-           loc:Optional[dict]=None, # Locals namespace
-           timeout:int=3600 # Maximum run time in seconds before a `TimeoutError` is raised
-          ): # Result of last node, if it's an expression, or `None` otherwise
-    """Executes python `code` with `timeout` and returning final expression (similar to IPython).
-    Raised exceptions are returned as a string, with a stack trace."""
+# %% ../01_funccall.ipynb
+def python(
+    code:str, # Code to execute
+    glb:Optional[dict]=None, # Globals namespace
+    loc:Optional[dict]=None, # Locals namespace
+    timeout:int=3600 # Maximum run time in seconds
+):
+    "Executes python `code` with `timeout` and returning final expression (similar to IPython)."
     def handler(*args): raise TimeoutError()
     if glb is None: glb = inspect.currentframe().f_back.f_globals
     if loc is None: loc=glb
@@ -181,32 +182,36 @@ def python(code:str, # Code to execute
     except Exception as e: return traceback.format_exc()
     finally: signal.alarm(0)
 
-# %% ../01_funccall.ipynb 86
-def mk_ns(*funcs_or_objs):
+# %% ../01_funccall.ipynb
+def mk_ns(fs):
+    if isinstance(fs, abc.Mapping): return fs
     merged = {}
-    for o in funcs_or_objs:
-        if isinstance(o, type): merged |= {n:getattr(o,n) for n,m in o.__dict__.items() if isinstance(m, (staticmethod, classmethod))}
-        if isinstance(o, object): merged |= {n:getattr(o,n) for n, m in inspect.getmembers(o, inspect.ismethod)} | {n:m for n,m in o.__class__.__dict__.items() if isinstance(m, staticmethod)}
-        if callable(o) and hasattr(o, '__name__'): merged |= {o.__name__: o}
+    for o in listify(fs):
+        if isinstance(o, dict): merged |= o
+        elif isinstance(o, type): merged |= {n:getattr(o,n) for n,m in o.__dict__.items() if isinstance(m, (staticmethod, classmethod))}
+        elif callable(o) and hasattr(o, '__name__'): merged |= {o.__name__: o}
+        merged |= {n:getattr(o,n) for n, m in inspect.getmembers(o, inspect.ismethod)} | {n:m for n,m in o.__class__.__dict__.items() if isinstance(m, staticmethod)}
     return merged
 
-# %% ../01_funccall.ipynb 95
+# %% ../01_funccall.ipynb
 def call_func(fc_name, fc_inputs, ns, raise_on_err=True):
     "Call the function `fc_name` with the given `fc_inputs` using namespace `ns`."
-    if not isinstance(ns, abc.Mapping): ns = mk_ns(*ns)
+    if not isinstance(ns, abc.Mapping): ns = mk_ns(ns)
     func = ns[fc_name]
+    # Clean up bad param names
+    inps = {re.sub(r'\W', '', k):v for k,v in fc_inputs.items()}
     try: return func(**fc_inputs)
     except Exception as e:
-        if raise_on_err: raise e
+        if raise_on_err: raise e from None
         else: return traceback.format_exc()
 
-# %% ../01_funccall.ipynb 106
+# %% ../01_funccall.ipynb
 async def call_func_async(fc_name, fc_inputs, ns, raise_on_err=True):
     "Awaits the function `fc_name` with the given `fc_inputs` using namespace `ns`."
     res = call_func(fc_name, fc_inputs, ns, raise_on_err=raise_on_err)
     if inspect.iscoroutine(res):
         try: res = await res
         except Exception as e:
-            if raise_on_err: raise e
+            if raise_on_err: raise e from None
             else: return traceback.format_exc()
     return res

{toolslm-0.2.1 → toolslm-0.2.3}/toolslm/shell.py

@@ -3,18 +3,18 @@
 # %% auto 0
 __all__ = ['get_shell']
 
-# %% ../02_shell.ipynb 2
+# %% ../02_shell.ipynb
 import ast, time, signal, traceback
 from fastcore.utils import *
 
-# %% ../02_shell.ipynb 4
+# %% ../02_shell.ipynb
 from IPython.terminal.interactiveshell import TerminalInteractiveShell
 from IPython.utils.capture import capture_output
 
-# %% ../02_shell.ipynb 7
+# %% ../02_shell.ipynb
 TerminalInteractiveShell.orig_run = TerminalInteractiveShell.run_cell
 
-# %% ../02_shell.ipynb 8
+# %% ../02_shell.ipynb
 @patch
 def run_cell(self:TerminalInteractiveShell, cell, timeout=None):
     "Wrapper for original `run_cell` which adds timeout and output capture"
@@ -31,7 +31,7 @@ def run_cell(self:TerminalInteractiveShell, cell, timeout=None):
     finally:
         if timeout: signal.alarm(0)
 
-# %% ../02_shell.ipynb 9
+# %% ../02_shell.ipynb
 def get_shell()->TerminalInteractiveShell:
     "Get a `TerminalInteractiveShell` with minimal functionality"
     sh = TerminalInteractiveShell()

{toolslm-0.2.1 → toolslm-0.2.3}/toolslm/xml.py

@@ -3,7 +3,7 @@
 # %% auto 0
 __all__ = ['doctype', 'json_to_xml', 'mk_doctype', 'mk_doc', 'docs_xml', 'files2ctx', 'folder2ctx', 'folder2ctx_cli']
 
-# %% ../00_xml.ipynb 3
+# %% ../00_xml.ipynb
 import hashlib,xml.etree.ElementTree as ET
 from collections import namedtuple
 
@@ -15,7 +15,7 @@ from fastcore.script import call_parse
 try: from IPython import display
 except: display=None
 
-# %% ../00_xml.ipynb 4
+# %% ../00_xml.ipynb
 def json_to_xml(d:dict, # JSON dictionary to convert
                 rnm:str # Root name
                )->str:
@@ -31,10 +31,10 @@ def json_to_xml(d:dict, # JSON dictionary to convert
     ET.indent(root)
     return ET.tostring(root, encoding='unicode')
 
-# %% ../00_xml.ipynb 9
+# %% ../00_xml.ipynb
 doctype = namedtuple('doctype', ['src', 'content'])
 
-# %% ../00_xml.ipynb 11
+# %% ../00_xml.ipynb
 def _add_nls(s):
     "Add newlines to start and end of `s` if missing"
     if not s: return s
@@ -42,7 +42,7 @@ def _add_nls(s):
     if s[-1]!='\n': s = s+'\n'
     return s
 
-# %% ../00_xml.ipynb 16
+# %% ../00_xml.ipynb
 def mk_doctype(content:str,  # The document content
            src:Optional[str]=None # URL, filename, etc; defaults to `md5(content)` if not provided
           ) -> namedtuple:
@@ -50,7 +50,7 @@ def mk_doctype(content:str,  # The document content
     if src is None: src = hashlib.md5(content.encode()).hexdigest()[:8]
     return doctype(_add_nls(str(src).strip()), _add_nls(content.strip()))
 
-# %% ../00_xml.ipynb 19
+# %% ../00_xml.ipynb
 def mk_doc(index:int,  # The document index
            content:str,  # The document content
            src:Optional[str]=None, # URL, filename, etc; defaults to `md5(content)` if not provided
@@ -62,7 +62,7 @@ def mk_doc(index:int,  # The document index
     src = Src(NotStr(dt.src))
     return Document(src, content, index=index, **kwargs)
 
-# %% ../00_xml.ipynb 22
+# %% ../00_xml.ipynb
 def docs_xml(docs:list[str],  # The content of each document
              srcs:Optional[list]=None,  # URLs, filenames, etc; each one defaults to `md5(content)` if not provided
              prefix:bool=True, # Include Anthropic's suggested prose intro?
@@ -75,7 +75,7 @@ def docs_xml(docs:list[str],  # The content of each document
     docs = (mk_doc(i+1, d, s, **kw) for i,(d,s,kw) in enumerate(zip(docs,srcs,details)))
     return pre + to_xml(Documents(docs))
 
-# %% ../00_xml.ipynb 29
+# %% ../00_xml.ipynb
 def files2ctx(
     fnames:list[Union[str,Path]], # List of file names to add to context
     prefix:bool=True # Include Anthropic's suggested prose intro?
@@ -84,7 +84,7 @@ def files2ctx(
     contents = [o.read_text() for o in fnames]
     return docs_xml(contents, fnames, prefix=prefix)
 
-# %% ../00_xml.ipynb 32
+# %% ../00_xml.ipynb
 @delegates(globtastic)
 def folder2ctx(
     folder:Union[str,Path], # Folder name containing files to add to context
@@ -94,7 +94,7 @@ def folder2ctx(
     fnames = globtastic(folder, **kwargs)
     return files2ctx(fnames, prefix=prefix)
 
-# %% ../00_xml.ipynb 34
+# %% ../00_xml.ipynb
 @call_parse
 @delegates(folder2ctx)
 def folder2ctx_cli(

{toolslm-0.2.1 → toolslm-0.2.3/toolslm.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: toolslm
-Version: 0.2.1
+Version: 0.2.3
 Summary: Tools to make language models a bit easier to use
 Home-page: https://github.com/AnswerDotAI/toolslm
 Author: Jeremy Howard

toolslm-0.2.1/toolslm/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.2.1"