toolslm 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

toolslm/__init__.py

@@ -1 +1 @@
-__version__ = "0.1.0"
+__version__ = "0.1.2"

toolslm/_modidx.py

@@ -14,15 +14,19 @@ d = { 'settings': { 'branch': 'main',
                                   'toolslm.download.read_html': ('download.html#read_html', 'toolslm/download.py'),
                                   'toolslm.download.read_md': ('download.html#read_md', 'toolslm/download.py'),
                                   'toolslm.download.split_url': ('download.html#split_url', 'toolslm/download.py')},
-            'toolslm.funccall': { 'toolslm.funccall._copy_loc': ('funccall.html#_copy_loc', 'toolslm/funccall.py'),
+            'toolslm.funccall': { 'toolslm.funccall.PathArg': ('funccall.html#patharg', 'toolslm/funccall.py'),
+                                  'toolslm.funccall._copy_loc': ('funccall.html#_copy_loc', 'toolslm/funccall.py'),
                                   'toolslm.funccall._get_nested_schema': ('funccall.html#_get_nested_schema', 'toolslm/funccall.py'),
                                   'toolslm.funccall._handle_container': ('funccall.html#_handle_container', 'toolslm/funccall.py'),
                                   'toolslm.funccall._handle_type': ('funccall.html#_handle_type', 'toolslm/funccall.py'),
+                                  'toolslm.funccall._is_container': ('funccall.html#_is_container', 'toolslm/funccall.py'),
+                                  'toolslm.funccall._is_parameterized': ('funccall.html#_is_parameterized', 'toolslm/funccall.py'),
                                   'toolslm.funccall._param': ('funccall.html#_param', 'toolslm/funccall.py'),
                                   'toolslm.funccall._process_property': ('funccall.html#_process_property', 'toolslm/funccall.py'),
                                   'toolslm.funccall._run': ('funccall.html#_run', 'toolslm/funccall.py'),
                                   'toolslm.funccall._types': ('funccall.html#_types', 'toolslm/funccall.py'),
                                   'toolslm.funccall.call_func': ('funccall.html#call_func', 'toolslm/funccall.py'),
+                                  'toolslm.funccall.call_func_async': ('funccall.html#call_func_async', 'toolslm/funccall.py'),
                                   'toolslm.funccall.get_schema': ('funccall.html#get_schema', 'toolslm/funccall.py'),
                                   'toolslm.funccall.mk_ns': ('funccall.html#mk_ns', 'toolslm/funccall.py'),
                                   'toolslm.funccall.python': ('funccall.html#python', 'toolslm/funccall.py')},

toolslm/download.py

@@ -27,10 +27,10 @@ def read_md(url, rm_comments=True, rm_details=True, **kwargs):
     return clean_md(get(url, **kwargs).text, rm_comments=rm_comments, rm_details=rm_details)
 
 # %% ../03_download.ipynb 7
-def html2md(s:str):
+def html2md(s:str, ignore_links=True):
     "Convert `s` from HTML to markdown"
     o = HTML2Text(bodywidth=5000)
-    o.ignore_links = True
+    o.ignore_links = ignore_links
     o.mark_code = True
     o.ignore_images = True
     return o.handle(s)
@@ -42,6 +42,7 @@ def read_html(url, # URL to read
               rm_details=True, # Removes `<details>` tags
               multi=False, # Get all matches to `sel` or first one  
               wrap_tag=None, #If multi, each selection wrapped with <wrap_tag>content</wrap_tag>
+              ignore_links=True,
              ): # Cleaned markdown
     "Get `url`, optionally selecting CSS selector `sel`, and convert to clean markdown"
     page = get(url).text
@@ -51,12 +52,12 @@ def read_html(url, # URL to read
             page = [str(el) for el in soup.select(sel)]
             if not wrap_tag: page = "\n".join(page)
         else: page = str(soup.select_one(sel))
-    mds = map(lambda x: clean_md(html2md(x), rm_comments, rm_details=rm_details), tuplify(page))
+    mds = map(lambda x: clean_md(html2md(x, ignore_links=ignore_links), rm_comments, rm_details=rm_details), tuplify(page))
     if wrap_tag: return '\n'.join([f"\n<{wrap_tag}>\n{o}</{wrap_tag}>\n" for o in mds])
     else: return'\n'.join(mds)
 
 
-# %% ../03_download.ipynb 12
+# %% ../03_download.ipynb 13
 def get_llmstxt(url, optional=False, n_workers=None):
     "Get llms.txt file from and expand it with `llms_txt.create_ctx()`"
     if not url.endswith('llms.txt'): return None
@@ -64,7 +65,7 @@ def get_llmstxt(url, optional=False, n_workers=None):
     if resp.status_code!=200: return None
     return create_ctx(resp.text, optional=optional, n_workers=n_workers)
 
-# %% ../03_download.ipynb 14
+# %% ../03_download.ipynb 15
 def split_url(url):
     "Split `url` into base, path, and file name, normalising name to '/' if empty"
     parsed = urlparse(url.strip('/'))
@@ -74,13 +75,13 @@ def split_url(url):
     if not path and not fname: path='/'
     return base,path,fname
 
-# %% ../03_download.ipynb 16
+# %% ../03_download.ipynb 17
 def _tryget(url):
     "Return response from `url` if `status_code!=404`, otherwise `None`"
     res = get(url)
     return None if res.status_code==404 else url
 
-# %% ../03_download.ipynb 17
+# %% ../03_download.ipynb 18
 def find_docs(url):
     "If available, return LLM-friendly llms.txt context or markdown file location from `url`"
     base,path,fname = split_url(url)
@@ -100,7 +101,7 @@ def find_docs(url):
     if parsed_url.path == '/' or not parsed_url.path: return None
     return find_docs(urljoin(url, '..'))
 
-# %% ../03_download.ipynb 22
+# %% ../03_download.ipynb 23
 def read_docs(url, optional=False, n_workers=None, rm_comments=True, rm_details=True):
     "If available, return LLM-friendly llms.txt context or markdown file response for `url`"
     url = find_docs(url)

toolslm/funccall.py

@@ -1,13 +1,15 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../01_funccall.ipynb.
 
 # %% auto 0
-__all__ = ['empty', 'get_schema', 'python', 'mk_ns', 'call_func']
+__all__ = ['empty', 'custom_types', 'get_schema', 'PathArg', 'python', 'mk_ns', 'call_func', 'call_func_async']
 
 # %% ../01_funccall.ipynb 2
 import inspect
 from collections import abc
 from fastcore.utils import *
 from fastcore.docments import docments
+from typing import get_origin, get_args, Dict, List, Optional, Tuple, Union
+from types import UnionType
 
 # %% ../01_funccall.ipynb 3
 empty = inspect.Parameter.empty
@@ -18,11 +20,18 @@ def _types(t:type)->tuple[str,Optional[str]]:
     if t is empty: raise TypeError('Missing type')
     tmap = {int:"integer", float:"number", str:"string", bool:"boolean", list:"array", dict:"object"}
     tmap.update({k.__name__: v for k, v in tmap.items()})
-    if getattr(t, '__origin__', None) in (list,tuple): return "array", tmap.get(t.__args__[0].__name__, "object")
+    if getattr(t, '__origin__', None) in (list,tuple):
+        args = getattr(t, '__args__', None)
+        item_type = "object" if not args else tmap.get(t.__args__[0].__name__, "object")
+        return "array", item_type
+    # if t is a string like 'int', directly use the string as the key
     elif isinstance(t, str): return tmap.get(t, "object"), None
+    # if t is the type itself and a container
+    elif get_origin(t): return tmap.get(get_origin(t).__name__, "object"), None
+    # if t is the type itself like int, use the __name__ representation as the key
     else: return tmap.get(t.__name__, "object"), None
 
-# %% ../01_funccall.ipynb 16
+# %% ../01_funccall.ipynb 18
 def _param(name, info):
     "json schema parameter given `name` and `info` from docments full dict."
     paramt,itemt = _types(info.anno)
@@ -31,17 +40,33 @@ def _param(name, info):
     if info.default is not empty: pschema["default"] = info.default
     return pschema
 
-# %% ../01_funccall.ipynb 19
+# %% ../01_funccall.ipynb 21
+custom_types = {Path}
+
 def _handle_type(t, defs):
     "Handle a single type, creating nested schemas if necessary"
-    if isinstance(t, type) and not issubclass(t, (int, float, str, bool)):
+    if t is NoneType: return {'type': 'null'}
+    if t in custom_types: return {'type':'string', 'format':t.__name__}
+    if isinstance(t, type) and not issubclass(t, (int, float, str, bool)) or inspect.isfunction(t):
         defs[t.__name__] = _get_nested_schema(t)
         return {'$ref': f'#/$defs/{t.__name__}'}
     return {'type': _types(t)[0]}
 
-# %% ../01_funccall.ipynb 20
+# %% ../01_funccall.ipynb 23
+def _is_container(t):
+    "Check if type is a container (list, dict, tuple, set, Union)"
+    origin = get_origin(t)
+    return origin in (list, dict, tuple, set, Union) if origin else False
+
+def _is_parameterized(t):
+    "Check if type has arguments (e.g. list[int] vs list, dict[str, int] vs dict)"
+    return _is_container(t) and (get_args(t) != ())
+
+# %% ../01_funccall.ipynb 29
 def _handle_container(origin, args, defs):
-    "Handle container types like dict, list, tuple, set"
+    "Handle container types like dict, list, tuple, set, and Union"
+    if origin is Union or origin is UnionType:
+        return {"anyOf": [_handle_type(arg, defs) for arg in args]}
     if origin is dict:
         value_type = args[1].__args__[0] if hasattr(args[1], '__args__') else args[1]
         return {
@@ -58,34 +83,35 @@ def _handle_container(origin, args, defs):
         return schema
     return None
 
-# %% ../01_funccall.ipynb 21
+# %% ../01_funccall.ipynb 30
 def _process_property(name, obj, props, req, defs):
     "Process a single property of the schema"
     p = _param(name, obj)
     props[name] = p
     if obj.default is empty: req[name] = True
-    
-    if hasattr(obj.anno, '__origin__'):
-        p.update(_handle_container(obj.anno.__origin__, obj.anno.__args__, defs))
+
+    if _is_container(obj.anno) and _is_parameterized(obj.anno):
+            p.update(_handle_container(get_origin(obj.anno), get_args(obj.anno), defs))        
     else:
+        # Non-container type or container without arguments
         p.update(_handle_type(obj.anno, defs))
 
-# %% ../01_funccall.ipynb 22
+# %% ../01_funccall.ipynb 31
 def _get_nested_schema(obj):
     "Generate nested JSON schema for a class or function"
     d = docments(obj, full=True)
     props, req, defs = {}, {}, {}
-    
+
     for n, o in d.items():
         if n != 'return' and n != 'self':
             _process_property(n, o, props, req, defs)
-    
+
     schema = dict(type='object', properties=props, title=obj.__name__ if isinstance(obj, type) else None)
     if req: schema['required'] = list(req)
     if defs: schema['$defs'] = defs
     return schema
 
-# %% ../01_funccall.ipynb 26
+# %% ../01_funccall.ipynb 35
 def get_schema(f:callable, pname='input_schema')->dict:
     "Generate JSON schema for a class, function, or method"
     schema = _get_nested_schema(f)
@@ -96,11 +122,16 @@ def get_schema(f:callable, pname='input_schema')->dict:
     if ret.anno is not empty: desc += f'\n\nReturns:\n- type: {_types(ret.anno)[0]}'
     return {"name": f.__name__, "description": desc, pname: schema}
 
-# %% ../01_funccall.ipynb 39
+# %% ../01_funccall.ipynb 46
+def PathArg(
+    path: str  # A filesystem path
+): return Path(path)
+
+# %% ../01_funccall.ipynb 66
 import ast, time, signal, traceback
 from fastcore.utils import *
 
-# %% ../01_funccall.ipynb 40
+# %% ../01_funccall.ipynb 67
 def _copy_loc(new, orig):
     "Copy location information from original node to new node and all children."
     new = ast.copy_location(new, orig)
@@ -109,7 +140,7 @@ def _copy_loc(new, orig):
         elif isinstance(o, list): setattr(new, field, [_copy_loc(value, orig) for value in o])
     return new
 
-# %% ../01_funccall.ipynb 42
+# %% ../01_funccall.ipynb 69
 def _run(code:str ):
     "Run `code`, returning final expression (similar to IPython)"
     tree = ast.parse(code)
@@ -132,7 +163,7 @@ def _run(code:str ):
     if _result is not None: return _result
     return stdout_buffer.getvalue().strip()
 
-# %% ../01_funccall.ipynb 47
+# %% ../01_funccall.ipynb 74
 def python(code, # Code to execute
            timeout=5 # Maximum run time in seconds before a `TimeoutError` is raised
           ): # Result of last node, if it's an expression, or `None` otherwise
@@ -145,7 +176,7 @@ def python(code, # Code to execute
     except Exception as e: return traceback.format_exc()
     finally: signal.alarm(0)
 
-# %% ../01_funccall.ipynb 54
+# %% ../01_funccall.ipynb 81
 def mk_ns(*funcs_or_objs):
     merged = {}
     for o in funcs_or_objs:
@@ -154,9 +185,16 @@ def mk_ns(*funcs_or_objs):
         if callable(o) and hasattr(o, '__name__'): merged |= {o.__name__: o}
     return merged
 
-# %% ../01_funccall.ipynb 63
+# %% ../01_funccall.ipynb 90
 def call_func(fc_name, fc_inputs, ns):
     "Call the function `fc_name` with the given `fc_inputs` using namespace `ns`."
     if not isinstance(ns, abc.Mapping): ns = mk_ns(*ns)
     func = ns[fc_name]
     return func(**fc_inputs)
+
+# %% ../01_funccall.ipynb 97
+async def call_func_async(fc_name, fc_inputs, ns):
+    "Awaits the function `fc_name` with the given `fc_inputs` using namespace `ns`."
+    if not isinstance(ns, abc.Mapping): ns = mk_ns(*ns)
+    func = ns[fc_name]
+    return await func(**fc_inputs)

toolslm/md_hier.py

@@ -7,9 +7,15 @@ def markdown_to_dict(markdown_content):
 
     lines = markdown_content.splitlines()
     headings = []
+    in_code_block = False
 
     # Parse headings with their levels and line numbers
     for idx, line in enumerate(lines):
+        # Toggle code block state when encountering fence
+        if line.strip().startswith('```'): in_code_block = not in_code_block
+        
+        # Only detect headings when not in a code block
+        if in_code_block: continue
         match = re.match(r'^(#{1,6})\s*(.*)', line)
         if match:
             level = len(match.group(1))
@@ -29,13 +35,15 @@ def markdown_to_dict(markdown_content):
 
     # Build the dictionary with hierarchical keys
     result,stack = {},[]
+    first_level = headings[0]['level']
     for h in headings:
-        stack = stack[:h['level'] - 1] + [clean_heading(h['text'])]
+        stack = stack[:h['level'] - first_level] + [clean_heading(h['text'])]
         key = '.'.join(stack)
         result[key] = h['content']
     return dict2obj(result)
 
 def create_heading_dict(text):
+    text = re.sub(r'```[\s\S]*?```', '', text)
     headings = re.findall(r'^#+.*', text, flags=re.MULTILINE)
     result = {}
     stack = [result]
@@ -126,11 +134,42 @@ Admin users management.
         assert result['Parent.Child'] == '## Child\nChild content.\n### Grandchild\nGrandchild content.'
         assert result['Parent.Child.Grandchild'] == '### Grandchild\nGrandchild content.'
 
+    def test_multiple_level2_siblings():
+        md_content = "##Sib 1\n##Sib 2\n##Sib 3\n##Sib 4\n##Sib 5'"
+        result = markdown_to_dict(md_content)
+        assert 'Sib 1' in result
+        assert 'Sib 2' in result
+        assert 'Sib 3' in result
+        assert 'Sib 4' in result
+        assert 'Sib 5' in result
+        
+    def test_code_chunks_escaped():
+        md_content = "# Parent\nParent content.\n## Child\nChild content.\n```python\n# Code comment\nprint('Hello, world!')\n```"
+        result = markdown_to_dict(md_content)
+        assert 'Code comment' not in result
+        assert "# Code comment" in result['Parent.Child']
+
     test_empty_content()
     test_special_characters()
     test_duplicate_headings()
     test_no_content()
     test_different_levels()
     test_parent_includes_subheadings()
+    test_multiple_level2_siblings()
+    test_code_chunks_escaped()
     print('tests passed')
 
+    def test_nested_headings():    
+        md_content = "# Parent\nParent content.\n## Child\nChild content.\n### Grandchild\nGrandchild content."
+        result = create_heading_dict(md_content)
+        assert 'Child' in result['Parent']
+        assert 'Grandchild' in result['Parent']['Child']
+
+    def test_code_chunks_escaped():
+        md_content = "# Parent\nParent content.\n## Child\nChild content.\n```python\n# Code comment\nprint('Hello, world!')\n```"
+        result = create_heading_dict(md_content)
+        assert 'Code comment' not in result
+    
+    test_nested_headings()
+    test_code_chunks_escaped()
+    print('tests passed')

{toolslm-0.1.0.dist-info → toolslm-0.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: toolslm
-Version: 0.1.0
+Version: 0.1.2
 Summary: Tools to make language models a bit easier to use
 Home-page: https://github.com/AnswerDotAI/toolslm
 Author: Jeremy Howard
@@ -20,8 +20,21 @@ Requires-Dist: fastcore>=1.5.47
 Requires-Dist: beautifulsoup4
 Requires-Dist: html2text
 Requires-Dist: httpx
-Requires-Dist: llms-txt
+Requires-Dist: llms_txt
 Provides-Extra: dev
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # toolslm
 

toolslm-0.1.2.dist-info/RECORD

@@ -0,0 +1,13 @@
+toolslm/__init__.py,sha256=YvuYzWnKtqBb-IqG8HAu-nhIYAsgj9Vmc_b9o7vO-js,22
+toolslm/_modidx.py,sha256=-D-B5o30VGs11gBKf96lpADVXnZhdiVEshJpLzmUnDs,4378
+toolslm/download.py,sha256=d84O8PCX7uAbLg0HTbNNfCdANPcnhXFu4qzOtcfJfHU,4465
+toolslm/funccall.py,sha256=yAZmu2gIRtND-5pOuLCp7P4dor0FV_xntcacnZ81j0M,8210
+toolslm/md_hier.py,sha256=4uC12443tPBduYJgIZZIcEat2VG0x7JYC8-SwDdS2JY,6360
+toolslm/shell.py,sha256=GVqfL74NHw66zzZ7jvGVLjE55ZNJGBPvEb8kLz4aoYc,1576
+toolslm/xml.py,sha256=QNwUavoMkFK84D7dMwnBjqlYJwN-pJ7u3BxOeDuNAmk,4088
+toolslm-0.1.2.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+toolslm-0.1.2.dist-info/METADATA,sha256=gGurWuj5SWNPQvCPKbd8oc5Iz9x4NPxv8bx_C8w3Kps,2483
+toolslm-0.1.2.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+toolslm-0.1.2.dist-info/entry_points.txt,sha256=xFz0Eymlo5X7BGpaO6DI9gMxvN5A7faebzrlr8ctp5I,95
+toolslm-0.1.2.dist-info/top_level.txt,sha256=4hRTrFWayz_Kz5221XjvlpCwVFrW3WPi1P0fllkTq9s,8
+toolslm-0.1.2.dist-info/RECORD,,

{toolslm-0.1.0.dist-info → toolslm-0.1.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.3.0)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

toolslm-0.1.0.dist-info/RECORD

@@ -1,13 +0,0 @@
-toolslm/__init__.py,sha256=kUR5RAFc7HCeiqdlX36dZOHkUI5wI6V_43RpEcD8b-0,22
-toolslm/_modidx.py,sha256=EIl2FBWhcZUS46r1AU0wURYg2O6Z3aXTPUr3p8Smrqk,3882
-toolslm/download.py,sha256=tXhq77GCqwVFDzTtzjcSAjxUWRiyPsjlXzkMjleH3dQ,4378
-toolslm/funccall.py,sha256=hSvBvfMv-YcBSUUs4-NrYu1f8jg4gfu2s82cPyIHVkU,6534
-toolslm/md_hier.py,sha256=hkCjuOfIFWuMEiM2_XCoD9QIBjy9huLOSvpX_bMdn0Y,4645
-toolslm/shell.py,sha256=GVqfL74NHw66zzZ7jvGVLjE55ZNJGBPvEb8kLz4aoYc,1576
-toolslm/xml.py,sha256=QNwUavoMkFK84D7dMwnBjqlYJwN-pJ7u3BxOeDuNAmk,4088
-toolslm-0.1.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-toolslm-0.1.0.dist-info/METADATA,sha256=tQBydygSCJdH_wQaIbzC8Z8rZanQTJOdmpe1nEETkdE,2205
-toolslm-0.1.0.dist-info/WHEEL,sha256=P9jw-gEje8ByB7_hXoICnHtVCrEwMQh-630tKvQWehc,91
-toolslm-0.1.0.dist-info/entry_points.txt,sha256=xFz0Eymlo5X7BGpaO6DI9gMxvN5A7faebzrlr8ctp5I,95
-toolslm-0.1.0.dist-info/top_level.txt,sha256=4hRTrFWayz_Kz5221XjvlpCwVFrW3WPi1P0fllkTq9s,8
-toolslm-0.1.0.dist-info/RECORD,,