toolslm 0.0.6__py3-none-any.whl → 0.0.7__py3-none-any.whl

toolslm/__init__.py

@@ -1 +1 @@
-__version__ = "0.0.6"
+__version__ = "0.0.7"

toolslm/_modidx.py

@@ -15,11 +15,18 @@ d = { 'settings': { 'branch': 'main',
                                   'toolslm.download.read_md': ('download.html#read_md', 'toolslm/download.py'),
                                   'toolslm.download.split_url': ('download.html#split_url', 'toolslm/download.py')},
             'toolslm.funccall': { 'toolslm.funccall._copy_loc': ('funccall.html#_copy_loc', 'toolslm/funccall.py'),
+                                  'toolslm.funccall._get_nested_schema': ('funccall.html#_get_nested_schema', 'toolslm/funccall.py'),
+                                  'toolslm.funccall._handle_container': ('funccall.html#_handle_container', 'toolslm/funccall.py'),
+                                  'toolslm.funccall._handle_type': ('funccall.html#_handle_type', 'toolslm/funccall.py'),
                                   'toolslm.funccall._param': ('funccall.html#_param', 'toolslm/funccall.py'),
+                                  'toolslm.funccall._process_property': ('funccall.html#_process_property', 'toolslm/funccall.py'),
                                   'toolslm.funccall._run': ('funccall.html#_run', 'toolslm/funccall.py'),
                                   'toolslm.funccall._types': ('funccall.html#_types', 'toolslm/funccall.py'),
+                                  'toolslm.funccall.call_func': ('funccall.html#call_func', 'toolslm/funccall.py'),
                                   'toolslm.funccall.get_schema': ('funccall.html#get_schema', 'toolslm/funccall.py'),
+                                  'toolslm.funccall.mk_ns': ('funccall.html#mk_ns', 'toolslm/funccall.py'),
                                   'toolslm.funccall.python': ('funccall.html#python', 'toolslm/funccall.py')},
+            'toolslm.md_hier': {},
             'toolslm.shell': { 'toolslm.shell.TerminalInteractiveShell.run_cell': ( 'shell.html#terminalinteractiveshell.run_cell',
                                                                                     'toolslm/shell.py'),
                                'toolslm.shell.get_shell': ('shell.html#get_shell', 'toolslm/shell.py')},

toolslm/download.py

@@ -36,16 +36,27 @@ def html2md(s:str):
     return o.handle(s)
 
 # %% ../03_download.ipynb 8
-def read_html(url, sel=None, rm_comments=True, rm_details=True):
+def read_html(url, # URL to read
+              sel=None, # Read only outerHTML of CSS selector `sel`
+              rm_comments=True, # Removes HTML comments
+              rm_details=True, # Removes `<details>` tags
+              multi=False, # Get all matches to `sel` or first one  
+              wrap_tag=None, #If multi, each selection wrapped with <wrap_tag>content</wrap_tag>
+             ): # Cleaned markdown
     "Get `url`, optionally selecting CSS selector `sel`, and convert to clean markdown"
     page = get(url).text
     if sel:
         soup = BeautifulSoup(page, 'html.parser')
-        page = str(soup.find(sel))
-    md = html2md(page)
-    return clean_md(md, rm_comments, rm_details=rm_details)
+        if multi:
+            page = [str(el) for el in soup.select(sel)]
+            if not wrap_tag: page = "\n".join(page)
+        else: page = str(soup.select_one(sel))
+    mds = map(lambda x: clean_md(html2md(x), rm_comments, rm_details=rm_details), tuplify(page))
+    if wrap_tag: return '\n'.join([f"\n<{wrap_tag}>\n{o}</{wrap_tag}>\n" for o in mds])
+    else: return'\n'.join(mds)
 
-# %% ../03_download.ipynb 10
+
+# %% ../03_download.ipynb 12
 def get_llmstxt(url, optional=False, n_workers=None):
     "Get llms.txt file from and expand it with `llms_txt.create_ctx()`"
     if not url.endswith('llms.txt'): return None
@@ -53,7 +64,7 @@ def get_llmstxt(url, optional=False, n_workers=None):
     if resp.status_code!=200: return None
     return create_ctx(resp.text, optional=optional, n_workers=n_workers)
 
-# %% ../03_download.ipynb 12
+# %% ../03_download.ipynb 14
 def split_url(url):
     "Split `url` into base, path, and file name, normalising name to '/' if empty"
     parsed = urlparse(url.strip('/'))
@@ -63,20 +74,20 @@ def split_url(url):
     if not path and not fname: path='/'
     return base,path,fname
 
-# %% ../03_download.ipynb 14
+# %% ../03_download.ipynb 16
 def _tryget(url):
     "Return response from `url` if `status_code!=404`, otherwise `None`"
     res = get(url)
     return None if res.status_code==404 else url
 
-# %% ../03_download.ipynb 15
+# %% ../03_download.ipynb 17
 def find_docs(url):
     "If available, return LLM-friendly llms.txt context or markdown file location from `url`"
     base,path,fname = split_url(url)
     url = (base+path+fname).strip('/')
     if fname=='/llms.txt': return url
     if Path(fname).suffix in('.md', '.txt', '.rst'): return _tryget(url)
-    if '.' in fname: return _tryget(url+'.md')
+    if '.' in fname: return _tryget(url+'.md') or find_docs(url[:url.rfind('/')])
     res = _tryget(url+'/llms.txt')
     if res: return res
     res = _tryget(url+'/index.md')
@@ -85,13 +96,14 @@ def find_docs(url):
     if res: return res
     res = _tryget(url+'/index-commonmark.md')
     if res: return res
-    return None
+    parsed_url = urlparse(url)
+    if parsed_url.path == '/' or not parsed_url.path: return None
+    return find_docs(urljoin(url, '..'))
 
-# %% ../03_download.ipynb 19
+# %% ../03_download.ipynb 23
 def read_docs(url, optional=False, n_workers=None, rm_comments=True, rm_details=True):
     "If available, return LLM-friendly llms.txt context or markdown file response for `url`"
     url = find_docs(url)
-    if not url: return
     if url.endswith('/llms.txt'): res = get_llmstxt(url, optional=optional, n_workers=n_workers)
     else: res = get(url).text
     return clean_md(res, rm_comments=rm_comments, rm_details=rm_details)

toolslm/funccall.py

@@ -1,10 +1,11 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../01_funccall.ipynb.
 
 # %% auto 0
-__all__ = ['empty', 'get_schema', 'python']
+__all__ = ['empty', 'get_schema', 'python', 'mk_ns', 'call_func']
 
 # %% ../01_funccall.ipynb 2
 import inspect
+from collections import abc
 from fastcore.utils import *
 from fastcore.docments import docments
 
@@ -17,10 +18,11 @@ def _types(t:type)->tuple[str,Optional[str]]:
     if t is empty: raise TypeError('Missing type')
     tmap = {int:"integer", float:"number", str:"string", bool:"boolean", list:"array", dict:"object"}
     tmap.update({k.__name__: v for k, v in tmap.items()})
-    if getattr(t, '__origin__', None) in  (list,tuple): return "array", tmap.get(t.__args__[0], "object")
-    else: return tmap[t], None
+    if getattr(t, '__origin__', None) in (list,tuple): return "array", tmap.get(t.__args__[0].__name__, "object")
+    elif isinstance(t, str): return tmap.get(t, "object"), None
+    else: return tmap.get(t.__name__, "object"), None
 
-# %% ../01_funccall.ipynb 14
+# %% ../01_funccall.ipynb 16
 def _param(name, info):
     "json schema parameter given `name` and `info` from docments full dict."
     paramt,itemt = _types(info.anno)
@@ -29,28 +31,76 @@ def _param(name, info):
     if info.default is not empty: pschema["default"] = info.default
     return pschema
 
-# %% ../01_funccall.ipynb 17
+# %% ../01_funccall.ipynb 19
+def _handle_type(t, defs):
+    "Handle a single type, creating nested schemas if necessary"
+    if isinstance(t, type) and not issubclass(t, (int, float, str, bool)):
+        defs[t.__name__] = _get_nested_schema(t)
+        return {'$ref': f'#/$defs/{t.__name__}'}
+    return {'type': _types(t)[0]}
+
+# %% ../01_funccall.ipynb 20
+def _handle_container(origin, args, defs):
+    "Handle container types like dict, list, tuple, set"
+    if origin is dict:
+        value_type = args[1].__args__[0] if hasattr(args[1], '__args__') else args[1]
+        return {
+            'type': 'object',
+            'additionalProperties': (
+                {'type': 'array', 'items': _handle_type(value_type, defs)}
+                if hasattr(args[1], '__origin__') else _handle_type(args[1], defs)
+            )
+        }
+    elif origin in (list, tuple, set):
+        schema = {'type': 'array', 'items': _handle_type(args[0], defs)}
+        if origin is set:
+            schema['uniqueItems'] = True
+        return schema
+    return None
+
+# %% ../01_funccall.ipynb 21
+def _process_property(name, obj, props, req, defs):
+    "Process a single property of the schema"
+    p = _param(name, obj)
+    props[name] = p
+    if obj.default is empty: req[name] = True
+    
+    if hasattr(obj.anno, '__origin__'):
+        p.update(_handle_container(obj.anno.__origin__, obj.anno.__args__, defs))
+    else:
+        p.update(_handle_type(obj.anno, defs))
+
+# %% ../01_funccall.ipynb 22
+def _get_nested_schema(obj):
+    "Generate nested JSON schema for a class or function"
+    d = docments(obj, full=True)
+    props, req, defs = {}, {}, {}
+    
+    for n, o in d.items():
+        if n != 'return' and n != 'self':
+            _process_property(n, o, props, req, defs)
+    
+    schema = dict(type='object', properties=props, title=obj.__name__ if isinstance(obj, type) else None)
+    if req: schema['required'] = list(req)
+    if defs: schema['$defs'] = defs
+    return schema
+
+# %% ../01_funccall.ipynb 26
 def get_schema(f:callable, pname='input_schema')->dict:
-    "Convert function `f` into a JSON schema `dict` for tool use."
-    d = docments(f, full=True)
-    ret = d.pop('return')
-    d.pop('self', None) # Ignore `self` for methods
-    paramd = {
-        'type': "object",
-        'properties': {n:_param(n,o) for n,o in d.items() if n[0]!='_'},
-        'required': [n for n,o in d.items() if o.default is empty and n[0]!='_']
-    }
+    "Generate JSON schema for a class, function, or method"
+    schema = _get_nested_schema(f)
     desc = f.__doc__
     assert desc, "Docstring missing!"
+    d = docments(f, full=True)
+    ret = d.pop('return')
     if ret.anno is not empty: desc += f'\n\nReturns:\n- type: {_types(ret.anno)[0]}'
-    if ret.docment: desc += f'\n- description: {ret.docment}'
-    return {'name':f.__name__, 'description':desc, pname:paramd}
+    return {"name": f.__name__, "description": desc, pname: schema}
 
-# %% ../01_funccall.ipynb 24
+# %% ../01_funccall.ipynb 39
 import ast, time, signal, traceback
 from fastcore.utils import *
 
-# %% ../01_funccall.ipynb 25
+# %% ../01_funccall.ipynb 40
 def _copy_loc(new, orig):
     "Copy location information from original node to new node and all children."
     new = ast.copy_location(new, orig)
@@ -59,7 +109,7 @@ def _copy_loc(new, orig):
         elif isinstance(o, list): setattr(new, field, [_copy_loc(value, orig) for value in o])
     return new
 
-# %% ../01_funccall.ipynb 27
+# %% ../01_funccall.ipynb 42
 def _run(code:str ):
     "Run `code`, returning final expression (similar to IPython)"
     tree = ast.parse(code)
@@ -82,7 +132,7 @@ def _run(code:str ):
     if _result is not None: return _result
     return stdout_buffer.getvalue().strip()
 
-# %% ../01_funccall.ipynb 32
+# %% ../01_funccall.ipynb 47
 def python(code, # Code to execute
            timeout=5 # Maximum run time in seconds before a `TimeoutError` is raised
           ): # Result of last node, if it's an expression, or `None` otherwise
@@ -94,3 +144,19 @@ def python(code, # Code to execute
     try: return _run(code)
     except Exception as e: return traceback.format_exc()
     finally: signal.alarm(0)
+
+# %% ../01_funccall.ipynb 54
+def mk_ns(*funcs_or_objs):
+    merged = {}
+    for o in funcs_or_objs:
+        if isinstance(o, type): merged |= {n:getattr(o,n) for n,m in o.__dict__.items() if isinstance(m, (staticmethod, classmethod))}
+        if isinstance(o, object): merged |= {n:getattr(o,n) for n, m in inspect.getmembers(o, inspect.ismethod)} | {n:m for n,m in o.__class__.__dict__.items() if isinstance(m, staticmethod)}
+        if callable(o) and hasattr(o, '__name__'): merged |= {o.__name__: o}
+    return merged
+
+# %% ../01_funccall.ipynb 63
+def call_func(fc_name, fc_inputs, ns):
+    "Call the function `fc_name` with the given `fc_inputs` using namespace `ns`."
+    if not isinstance(ns, abc.Mapping): ns = mk_ns(*ns)
+    func = ns[fc_name]
+    return func(**fc_inputs)

toolslm/md_hier.py

@@ -0,0 +1,136 @@
+import re
+from fastcore.utils import *
+__all__ = ['markdown_to_dict', 'create_heading_dict']
+
+def markdown_to_dict(markdown_content):
+    def clean_heading(text): return re.sub(r'[^A-Za-z0-9 ]+', '', text).strip()
+
+    lines = markdown_content.splitlines()
+    headings = []
+
+    # Parse headings with their levels and line numbers
+    for idx, line in enumerate(lines):
+        match = re.match(r'^(#{1,6})\s*(.*)', line)
+        if match:
+            level = len(match.group(1))
+            text = match.group(2).strip()
+            headings.append({'level': level, 'text': text, 'line': idx})
+
+    # Assign content to each heading, including subheadings
+    for i, h in enumerate(headings):
+        start = h['line']  # Include the heading line itself
+        # Find the end index: next heading of same or higher level
+        for j in range(i + 1, len(headings)):
+            if headings[j]['level'] <= h['level']:
+                end = headings[j]['line']
+                break
+        else: end = len(lines)
+        h['content'] = '\n'.join(lines[start:end]).strip()
+
+    # Build the dictionary with hierarchical keys
+    result,stack = {},[]
+    for h in headings:
+        stack = stack[:h['level'] - 1] + [clean_heading(h['text'])]
+        key = '.'.join(stack)
+        result[key] = h['content']
+    return dict2obj(result)
+
+def create_heading_dict(text):
+    headings = re.findall(r'^#+.*', text, flags=re.MULTILINE)
+    result = {}
+    stack = [result]
+    prev_level = 0
+
+    for heading in headings:
+        level = heading.count('#')
+        title = heading.strip('#').strip()
+        while level <= prev_level:
+            stack.pop()
+            prev_level -= 1
+        new_dict = {}
+        stack[-1][title] = new_dict
+        stack.append(new_dict)
+        prev_level = level
+    return dict2obj(result)
+
+
+if __name__=='__main__':
+    md_content = """
+# User
+
+This is the User section.
+
+## Tokens
+
+Details about tokens.
+
+### Value
+
+The value of tokens.
+
+Some more details.
+
+## Settings
+
+User settings information.
+
+# Admin
+
+Admin section.
+
+## Users
+
+Admin users management.
+"""
+
+    result = markdown_to_dict(md_content)
+    #for key, value in result.items(): print(f'Key: {key}\nValue:\n{value}\n{"-"*40}')
+
+    def test_empty_content():
+        md_content = "# Empty Heading"
+        result = markdown_to_dict(md_content)
+        assert result['Empty Heading'] == '# Empty Heading'
+
+    def test_special_characters():
+        md_content = "# Heading *With* Special _Characters_!\nContent under heading."
+        result = markdown_to_dict(md_content)
+        assert 'Heading With Special Characters' in result
+        assert result['Heading With Special Characters'] == '# Heading *With* Special _Characters_!\nContent under heading.'
+
+    def test_duplicate_headings():
+        md_content = "# Duplicate\n## Duplicate\n### Duplicate\nContent under duplicate headings."
+        result = markdown_to_dict(md_content)
+        assert 'Duplicate' in result
+        assert 'Duplicate.Duplicate' in result
+        assert 'Duplicate.Duplicate.Duplicate' in result
+        assert result['Duplicate.Duplicate.Duplicate'] == '### Duplicate\nContent under duplicate headings.'
+
+    def test_no_content():
+        md_content = "# No Content Heading\n## Subheading"
+        result = markdown_to_dict(md_content)
+        assert result['No Content Heading'] == '# No Content Heading\n## Subheading'
+        assert result['No Content Heading.Subheading'] == '## Subheading'
+
+    def test_different_levels():
+        md_content = "### Level 3 Heading\nContent at level 3.\n# Level 1 Heading\nContent at level 1."
+        result = markdown_to_dict(md_content)
+        assert 'Level 3 Heading' in result
+        assert 'Level 1 Heading' in result
+        assert result['Level 3 Heading'] == '### Level 3 Heading\nContent at level 3.'
+        assert result['Level 1 Heading'] == '# Level 1 Heading\nContent at level 1.'
+
+    def test_parent_includes_subheadings():
+        md_content = "# Parent\nParent content.\n## Child\nChild content.\n### Grandchild\nGrandchild content."
+        result = markdown_to_dict(md_content)
+        assert result['Parent'] == '# Parent\nParent content.\n## Child\nChild content.\n### Grandchild\nGrandchild content.'
+        assert result['Parent.Child'] == '## Child\nChild content.\n### Grandchild\nGrandchild content.'
+        assert result['Parent.Child.Grandchild'] == '### Grandchild\nGrandchild content.'
+
+    test_empty_content()
+    test_special_characters()
+    test_duplicate_headings()
+    test_no_content()
+    test_different_levels()
+    test_parent_includes_subheadings()
+    print('tests passed')
+

{toolslm-0.0.6.dist-info → toolslm-0.0.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: toolslm
-Version: 0.0.6
+Version: 0.0.7
 Summary: Tools to make language models a bit easier to use
 Home-page: https://github.com/AnswerDotAI/toolslm
 Author: Jeremy Howard

toolslm-0.0.7.dist-info/RECORD

@@ -0,0 +1,13 @@
+toolslm/__init__.py,sha256=R9xOYoYrWKcfO5zvTeGC3m_eDNOvxMd8CocQs2tLufo,22
+toolslm/_modidx.py,sha256=EIl2FBWhcZUS46r1AU0wURYg2O6Z3aXTPUr3p8Smrqk,3882
+toolslm/download.py,sha256=tf0TGFzJ6qbxCjjuG9iRC2i6lutcF9GviWY0fJc_lSU,4378
+toolslm/funccall.py,sha256=hSvBvfMv-YcBSUUs4-NrYu1f8jg4gfu2s82cPyIHVkU,6534
+toolslm/md_hier.py,sha256=hkCjuOfIFWuMEiM2_XCoD9QIBjy9huLOSvpX_bMdn0Y,4645
+toolslm/shell.py,sha256=GVqfL74NHw66zzZ7jvGVLjE55ZNJGBPvEb8kLz4aoYc,1576
+toolslm/xml.py,sha256=Alcd96KfNO8LklVefyc51LbXBoVLRSgifrpMVZPqYsc,4120
+toolslm-0.0.7.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+toolslm-0.0.7.dist-info/METADATA,sha256=sdRs3kCMl1xI8Z1if4xsGWuGaX9hbYGB0zs0BbRhQp0,3882
+toolslm-0.0.7.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+toolslm-0.0.7.dist-info/entry_points.txt,sha256=xFz0Eymlo5X7BGpaO6DI9gMxvN5A7faebzrlr8ctp5I,95
+toolslm-0.0.7.dist-info/top_level.txt,sha256=4hRTrFWayz_Kz5221XjvlpCwVFrW3WPi1P0fllkTq9s,8
+toolslm-0.0.7.dist-info/RECORD,,

toolslm-0.0.6.dist-info/RECORD

@@ -1,12 +0,0 @@
-toolslm/__init__.py,sha256=QiiYsv0kcJaB8wCWyT-FnI2b6be87HA-CrrIUn8LQhg,22
-toolslm/_modidx.py,sha256=FiHwMAAjvPdu7kN0pA1OJTJbUg0ddo0o12_C9JUFPDc,3103
-toolslm/download.py,sha256=BIhmbDSxM__57tukac63iwPx5sXIfbjYp7gh_fhp4Gw,3621
-toolslm/funccall.py,sha256=ZXfzhP0N5cex7n8QHuxDfUb0BJX1iI1inFb064LAGlc,3914
-toolslm/shell.py,sha256=GVqfL74NHw66zzZ7jvGVLjE55ZNJGBPvEb8kLz4aoYc,1576
-toolslm/xml.py,sha256=Alcd96KfNO8LklVefyc51LbXBoVLRSgifrpMVZPqYsc,4120
-toolslm-0.0.6.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-toolslm-0.0.6.dist-info/METADATA,sha256=_J7uXsh_qRX_wVKxssQjFpRH7zZjAwC0av1J5UIfTdk,3882
-toolslm-0.0.6.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-toolslm-0.0.6.dist-info/entry_points.txt,sha256=xFz0Eymlo5X7BGpaO6DI9gMxvN5A7faebzrlr8ctp5I,95
-toolslm-0.0.6.dist-info/top_level.txt,sha256=4hRTrFWayz_Kz5221XjvlpCwVFrW3WPi1P0fllkTq9s,8
-toolslm-0.0.6.dist-info/RECORD,,