toolslm 0.0.4__py3-none-any.whl → 0.0.6__py3-none-any.whl

toolslm/__init__.py

@@ -1 +1 @@
-__version__ = "0.0.4"
+__version__ = "0.0.6"

toolslm/_modidx.py

@@ -5,7 +5,16 @@ d = { 'settings': { 'branch': 'main',
                 'doc_host': 'https://AnswerDotAI.github.io',
                 'git_url': 'https://github.com/AnswerDotAI/toolslm',
                 'lib_path': 'toolslm'},
-  'syms': { 'toolslm.funccall': { 'toolslm.funccall._copy_loc': ('funccall.html#_copy_loc', 'toolslm/funccall.py'),
+  'syms': { 'toolslm.download': { 'toolslm.download._tryget': ('download.html#_tryget', 'toolslm/download.py'),
+                                  'toolslm.download.clean_md': ('download.html#clean_md', 'toolslm/download.py'),
+                                  'toolslm.download.find_docs': ('download.html#find_docs', 'toolslm/download.py'),
+                                  'toolslm.download.get_llmstxt': ('download.html#get_llmstxt', 'toolslm/download.py'),
+                                  'toolslm.download.html2md': ('download.html#html2md', 'toolslm/download.py'),
+                                  'toolslm.download.read_docs': ('download.html#read_docs', 'toolslm/download.py'),
+                                  'toolslm.download.read_html': ('download.html#read_html', 'toolslm/download.py'),
+                                  'toolslm.download.read_md': ('download.html#read_md', 'toolslm/download.py'),
+                                  'toolslm.download.split_url': ('download.html#split_url', 'toolslm/download.py')},
+            'toolslm.funccall': { 'toolslm.funccall._copy_loc': ('funccall.html#_copy_loc', 'toolslm/funccall.py'),
                                   'toolslm.funccall._param': ('funccall.html#_param', 'toolslm/funccall.py'),
                                   'toolslm.funccall._run': ('funccall.html#_run', 'toolslm/funccall.py'),
                                   'toolslm.funccall._types': ('funccall.html#_types', 'toolslm/funccall.py'),

toolslm/download.py

@@ -0,0 +1,97 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../03_download.ipynb.
+
+# %% auto 0
+__all__ = ['clean_md', 'read_md', 'html2md', 'read_html', 'get_llmstxt', 'split_url', 'find_docs', 'read_docs']
+
+# %% ../03_download.ipynb 2
+from fastcore.utils import *
+from httpx import get
+from fastcore.meta import delegates
+from llms_txt import *
+
+from html2text import HTML2Text
+from bs4 import BeautifulSoup
+from urllib.parse import urlparse, urljoin
+
+# %% ../03_download.ipynb 4
+def clean_md(text, rm_comments=True, rm_details=True):
+    "Remove comments and `<details>` sections from `text`"
+    if rm_comments: text = re.sub(r'\n?<!--.*?-->\n?', '', text, flags=re.DOTALL)
+    if rm_details: text = re.sub(r'\n?<details>.*?</details>\n?', '', text, flags=re.DOTALL)
+    return text
+
+# %% ../03_download.ipynb 5
+@delegates(get)
+def read_md(url, rm_comments=True, rm_details=True, **kwargs):
+    "Read text from `url` and clean with `clean_docs`"
+    return clean_md(get(url, **kwargs).text, rm_comments=rm_comments, rm_details=rm_details)
+
+# %% ../03_download.ipynb 7
+def html2md(s:str):
+    "Convert `s` from HTML to markdown"
+    o = HTML2Text(bodywidth=5000)
+    o.ignore_links = True
+    o.mark_code = True
+    o.ignore_images = True
+    return o.handle(s)
+
+# %% ../03_download.ipynb 8
+def read_html(url, sel=None, rm_comments=True, rm_details=True):
+    "Get `url`, optionally selecting CSS selector `sel`, and convert to clean markdown"
+    page = get(url).text
+    if sel:
+        soup = BeautifulSoup(page, 'html.parser')
+        page = str(soup.find(sel))
+    md = html2md(page)
+    return clean_md(md, rm_comments, rm_details=rm_details)
+
+# %% ../03_download.ipynb 10
+def get_llmstxt(url, optional=False, n_workers=None):
+    "Get llms.txt file from and expand it with `llms_txt.create_ctx()`"
+    if not url.endswith('llms.txt'): return None
+    resp = get(url)
+    if resp.status_code!=200: return None
+    return create_ctx(resp.text, optional=optional, n_workers=n_workers)
+
+# %% ../03_download.ipynb 12
+def split_url(url):
+    "Split `url` into base, path, and file name, normalising name to '/' if empty"
+    parsed = urlparse(url.strip('/'))
+    base = f"{parsed.scheme}://{parsed.netloc}"
+    path,spl,fname = parsed.path.rpartition('/')
+    fname = spl+fname
+    if not path and not fname: path='/'
+    return base,path,fname
+
+# %% ../03_download.ipynb 14
+def _tryget(url):
+    "Return response from `url` if `status_code!=404`, otherwise `None`"
+    res = get(url)
+    return None if res.status_code==404 else url
+
+# %% ../03_download.ipynb 15
+def find_docs(url):
+    "If available, return LLM-friendly llms.txt context or markdown file location from `url`"
+    base,path,fname = split_url(url)
+    url = (base+path+fname).strip('/')
+    if fname=='/llms.txt': return url
+    if Path(fname).suffix in('.md', '.txt', '.rst'): return _tryget(url)
+    if '.' in fname: return _tryget(url+'.md')
+    res = _tryget(url+'/llms.txt')
+    if res: return res
+    res = _tryget(url+'/index.md')
+    if res: return res
+    res = _tryget(url+'/index.html.md')
+    if res: return res
+    res = _tryget(url+'/index-commonmark.md')
+    if res: return res
+    return None
+
+# %% ../03_download.ipynb 19
+def read_docs(url, optional=False, n_workers=None, rm_comments=True, rm_details=True):
+    "If available, return LLM-friendly llms.txt context or markdown file response for `url`"
+    url = find_docs(url)
+    if not url: return
+    if url.endswith('/llms.txt'): res = get_llmstxt(url, optional=optional, n_workers=n_workers)
+    else: res = get(url).text
+    return clean_md(res, rm_comments=rm_comments, rm_details=rm_details)

toolslm/funccall.py

@@ -16,6 +16,7 @@ def _types(t:type)->tuple[str,Optional[str]]:
     "Tuple of json schema type name and (if appropriate) array item name."
     if t is empty: raise TypeError('Missing type')
     tmap = {int:"integer", float:"number", str:"string", bool:"boolean", list:"array", dict:"object"}
+    tmap.update({k.__name__: v for k, v in tmap.items()})
     if getattr(t, '__origin__', None) in  (list,tuple): return "array", tmap.get(t.__args__[0], "object")
     else: return tmap[t], None
 
@@ -45,11 +46,11 @@ def get_schema(f:callable, pname='input_schema')->dict:
     if ret.docment: desc += f'\n- description: {ret.docment}'
     return {'name':f.__name__, 'description':desc, pname:paramd}
 
-# %% ../01_funccall.ipynb 22
+# %% ../01_funccall.ipynb 24
 import ast, time, signal, traceback
 from fastcore.utils import *
 
-# %% ../01_funccall.ipynb 23
+# %% ../01_funccall.ipynb 25
 def _copy_loc(new, orig):
     "Copy location information from original node to new node and all children."
     new = ast.copy_location(new, orig)
@@ -58,7 +59,7 @@ def _copy_loc(new, orig):
         elif isinstance(o, list): setattr(new, field, [_copy_loc(value, orig) for value in o])
     return new
 
-# %% ../01_funccall.ipynb 25
+# %% ../01_funccall.ipynb 27
 def _run(code:str ):
     "Run `code`, returning final expression (similar to IPython)"
     tree = ast.parse(code)
@@ -81,7 +82,7 @@ def _run(code:str ):
     if _result is not None: return _result
     return stdout_buffer.getvalue().strip()
 
-# %% ../01_funccall.ipynb 30
+# %% ../01_funccall.ipynb 32
 def python(code, # Code to execute
            timeout=5 # Maximum run time in seconds before a `TimeoutError` is raised
           ): # Result of last node, if it's an expression, or `None` otherwise

toolslm/xml.py

@@ -37,6 +37,7 @@ doctype = namedtuple('doctype', ['source', 'content'])
 # %% ../00_xml.ipynb 11
 def _add_nls(s):
     "Add newlines to start and end of `s` if missing"
+    if not s: return s
     if s[ 0]!='\n': s = '\n'+s
     if s[-1]!='\n': s = s+'\n'
     return s
@@ -49,40 +50,32 @@ def mk_doctype(content:str,  # The document content
     if source is None: source = hashlib.md5(content.encode()).hexdigest()[:8]
     return doctype(_add_nls(str(source).strip()), _add_nls(content.strip()))
 
-# %% ../00_xml.ipynb 17
+# %% ../00_xml.ipynb 16
 def mk_doc(index:int,  # The document index
            content:str,  # The document content
-           source:Optional[str]=None # URL, filename, etc; defaults to `md5(content)` if not provided
-          ) -> tuple:
-    "Create an `ft` format tuple for a single doc in Anthropic's recommended format"
-    dt = mk_doctype(content, source)
-    content = ft('document_content', dt.content)
-    source =  ft('source', dt.source)
-    return ft('document', source, content, index=index)
-
-# %% ../00_xml.ipynb 18
-def mk_doc(index:int,  # The document index
-           content:str,  # The document content
-           source:Optional[str]=None # URL, filename, etc; defaults to `md5(content)` if not provided
+           source:Optional[str]=None, # URL, filename, etc; defaults to `md5(content)` if not provided
+           **kwargs
           ) -> tuple:
     "Create an `ft` format tuple for a single doc in Anthropic's recommended format"
     dt = mk_doctype(content, source)
     content = Document_content(dt.content)
     source = Source(dt.source)
-    return Document(source, content, index=index)
+    return Document(source, content, index=index, **kwargs)
 
-# %% ../00_xml.ipynb 22
+# %% ../00_xml.ipynb 19
 def docs_xml(docs:list[str],  # The content of each document
              sources:Optional[list]=None,  # URLs, filenames, etc; each one defaults to `md5(content)` if not provided
-             prefix:bool=True # Include Anthropic's suggested prose intro?
+             prefix:bool=True, # Include Anthropic's suggested prose intro?
+             details:Optional[list]=None # Optional list of dicts with additional attrs for each doc
             )->str:
     "Create an XML string containing `docs` in Anthropic's recommended format"
     pre = 'Here are some documents for you to reference for your task:\n\n' if prefix else ''
     if sources is None: sources = [None]*len(docs)
-    docs = (mk_doc(i+1, *o) for i,o in enumerate(zip(docs,sources)))
+    if details is None: details = [{}]*len(docs)
+    docs = (mk_doc(i+1, d, s, **kw) for i,(d,s,kw) in enumerate(zip(docs,sources,details)))
     return pre + to_xml(Documents(docs))
 
-# %% ../00_xml.ipynb 29
+# %% ../00_xml.ipynb 26
 def files2ctx(
     fnames:list[Union[str,Path]], # List of file names to add to context
     prefix:bool=True # Include Anthropic's suggested prose intro?
@@ -91,7 +84,7 @@ def files2ctx(
     contents = [o.read_text() for o in fnames]
     return docs_xml(contents, fnames, prefix=prefix)
 
-# %% ../00_xml.ipynb 32
+# %% ../00_xml.ipynb 29
 @delegates(globtastic)
 def folder2ctx(
     folder:Union[str,Path], # Folder name containing files to add to context
@@ -101,7 +94,7 @@ def folder2ctx(
     fnames = globtastic(folder, **kwargs)
     return files2ctx(fnames, prefix=prefix)
 
-# %% ../00_xml.ipynb 34
+# %% ../00_xml.ipynb 31
 @call_parse
 @delegates(folder2ctx)
 def folder2ctx_cli(

{toolslm-0.0.4.dist-info → toolslm-0.0.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: toolslm
-Version: 0.0.4
+Version: 0.0.6
 Summary: Tools to make language models a bit easier to use
 Home-page: https://github.com/AnswerDotAI/toolslm
 Author: Jeremy Howard
@@ -17,6 +17,10 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: fastcore >=1.5.47
+Requires-Dist: beautifulsoup4
+Requires-Dist: html2text
+Requires-Dist: httpx
+Requires-Dist: llms-txt
 Provides-Extra: dev
 
 # toolslm

toolslm-0.0.6.dist-info/RECORD

@@ -0,0 +1,12 @@
+toolslm/__init__.py,sha256=QiiYsv0kcJaB8wCWyT-FnI2b6be87HA-CrrIUn8LQhg,22
+toolslm/_modidx.py,sha256=FiHwMAAjvPdu7kN0pA1OJTJbUg0ddo0o12_C9JUFPDc,3103
+toolslm/download.py,sha256=BIhmbDSxM__57tukac63iwPx5sXIfbjYp7gh_fhp4Gw,3621
+toolslm/funccall.py,sha256=ZXfzhP0N5cex7n8QHuxDfUb0BJX1iI1inFb064LAGlc,3914
+toolslm/shell.py,sha256=GVqfL74NHw66zzZ7jvGVLjE55ZNJGBPvEb8kLz4aoYc,1576
+toolslm/xml.py,sha256=Alcd96KfNO8LklVefyc51LbXBoVLRSgifrpMVZPqYsc,4120
+toolslm-0.0.6.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+toolslm-0.0.6.dist-info/METADATA,sha256=_J7uXsh_qRX_wVKxssQjFpRH7zZjAwC0av1J5UIfTdk,3882
+toolslm-0.0.6.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+toolslm-0.0.6.dist-info/entry_points.txt,sha256=xFz0Eymlo5X7BGpaO6DI9gMxvN5A7faebzrlr8ctp5I,95
+toolslm-0.0.6.dist-info/top_level.txt,sha256=4hRTrFWayz_Kz5221XjvlpCwVFrW3WPi1P0fllkTq9s,8
+toolslm-0.0.6.dist-info/RECORD,,

toolslm-0.0.4.dist-info/RECORD

@@ -1,11 +0,0 @@
-toolslm/__init__.py,sha256=1mptEzQihbdyqqzMgdns_j5ZGK9gz7hR2bsgA_TnjO4,22
-toolslm/_modidx.py,sha256=6T36Q2cYKH0lp9Tt9Us8xpZV-Z0FYqrtZGu2ZykHDkg,2068
-toolslm/funccall.py,sha256=mzWNLdZY6cYk-I3O5noRiEB089mPwJhnRQFsS5_JYDs,3856
-toolslm/shell.py,sha256=GVqfL74NHw66zzZ7jvGVLjE55ZNJGBPvEb8kLz4aoYc,1576
-toolslm/xml.py,sha256=dSJOHqSWnZlMK1Qf3396ISSaBHf5miNlLSYCixYB9ng,4398
-toolslm-0.0.4.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-toolslm-0.0.4.dist-info/METADATA,sha256=9Ni6CdLgvxTCx7LqIrnNyUgrfu0t2Wsdabp9jZlFNvw,3782
-toolslm-0.0.4.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-toolslm-0.0.4.dist-info/entry_points.txt,sha256=xFz0Eymlo5X7BGpaO6DI9gMxvN5A7faebzrlr8ctp5I,95
-toolslm-0.0.4.dist-info/top_level.txt,sha256=4hRTrFWayz_Kz5221XjvlpCwVFrW3WPi1P0fllkTq9s,8
-toolslm-0.0.4.dist-info/RECORD,,