toolslm 0.3.6__tar.gz → 0.3.8__tar.gz

{toolslm-0.3.6/toolslm.egg-info → toolslm-0.3.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: toolslm
-Version: 0.3.6
+Version: 0.3.8
 Summary: Tools to make language models a bit easier to use
 Home-page: https://github.com/AnswerDotAI/toolslm
 Author: Jeremy Howard
@@ -16,7 +16,7 @@ Classifier: License :: OSI Approved :: Apache Software License
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: fastcore>=1.8.11
+Requires-Dist: fastcore>=1.9.6
 Requires-Dist: httpx
 Provides-Extra: dev
 Dynamic: author

{toolslm-0.3.6 → toolslm-0.3.8}/settings.ini

@@ -1,11 +1,11 @@
 [DEFAULT]
 repo = toolslm
 lib_name = toolslm
-version = 0.3.6
+version = 0.3.8
 min_python = 3.9
 license = apache2
 black_formatting = False
-requirements = fastcore>=1.8.11 httpx
+requirements = fastcore>=1.9.6 httpx
 doc_path = _docs
 lib_path = toolslm
 nbs_path = .

toolslm-0.3.8/toolslm/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.3.8"

{toolslm-0.3.6 → toolslm-0.3.8}/toolslm/_modidx.py

@@ -49,4 +49,5 @@ d = { 'settings': { 'branch': 'main',
                              'toolslm.xml.mk_doc': ('xml.html#mk_doc', 'toolslm/xml.py'),
                              'toolslm.xml.mk_doctype': ('xml.html#mk_doctype', 'toolslm/xml.py'),
                              'toolslm.xml.nb2xml': ('xml.html#nb2xml', 'toolslm/xml.py'),
-                             'toolslm.xml.read_file': ('xml.html#read_file', 'toolslm/xml.py')}}}
+                             'toolslm.xml.read_file': ('xml.html#read_file', 'toolslm/xml.py'),
+                             'toolslm.xml.repo2ctx': ('xml.html#repo2ctx', 'toolslm/xml.py')}}}

{toolslm-0.3.6 → toolslm-0.3.8}/toolslm/xml.py

@@ -2,19 +2,18 @@
 
 # %% auto 0
 __all__ = ['doctype', 'json_to_xml', 'get_mime_text', 'cell2out', 'cell2xml', 'nb2xml', 'mk_doctype', 'mk_doc', 'docs_xml',
-           'read_file', 'files2ctx', 'folder2ctx', 'folder2ctx_cli']
+           'read_file', 'files2ctx', 'folder2ctx', 'repo2ctx', 'folder2ctx_cli']
 
 # %% ../00_xml.ipynb
 import hashlib,xml.etree.ElementTree as ET
 from collections import namedtuple
+from ghapi.all import GhApi
 
 from fastcore.utils import *
 from fastcore.meta import delegates
 from fastcore.xtras import hl_md
-from fastcore.xml import to_xml, Document, Documents, Document_content, Src, Source,Out,Outs,Cell
+from fastcore.xml import to_xml, Document, Documents, Document_content, Src, Source,Out,Outs,Cell,Notebook,Md,Code
 from fastcore.script import call_parse
-try: from IPython import display
-except: display=None
 
 # %% ../00_xml.ipynb
 def json_to_xml(d:dict, # JSON dictionary to convert
@@ -50,20 +49,23 @@ def cell2out(o):
     if hasattr(o, 'ename'): return Out(f"{o.ename}: {o.evalue}", type='error')
 
 # %% ../00_xml.ipynb
-def cell2xml(cell):
+def cell2xml(cell, out=True):
     "Convert notebook cell to concise XML format"
-    cts = Source(''.join(cell.source)) if hasattr(cell, 'source') and cell.source else None
+    src = ''.join(getattr(cell, 'source', ''))
+    f = Code if cell.cell_type=='code' else Md
+    if not out: return f(src)
+    parts = [Source(src)]
     out_items = L(getattr(cell,'outputs',[])).map(cell2out).filter()
-    outs = []
-    if out_items: outs = Outs(*out_items)
-    parts = [p for p in [cts, outs] if p]
-    return Cell(*parts, type=cell.cell_type)
+    if out_items: parts.append(Outs(*out_items))
+    return f(*parts)
 
 # %% ../00_xml.ipynb
-def nb2xml(fname):
-    nb = dict2obj(fname.read_json())
-    cells_xml = [to_xml(cell2xml(c), do_escape=False) for c in nb.cells if c.cell_type in ('code','markdown')]
-    return '\n'.join(cells_xml)
+def nb2xml(fname=None, nb=None, out=True):
+    "Convert notebook to XML format"
+    assert bool(fname)^bool(nb), "Pass either `fname` or `nb`"
+    if not nb: nb = dict2obj(fname.read_json())
+    cells_xml = [to_xml(cell2xml(c, out=out), do_escape=False) for c in nb.cells if c.cell_type in ('code','markdown')]
+    return to_xml(Notebook(*cells_xml), do_escape=False)
 
 # %% ../00_xml.ipynb
 doctype = namedtuple('doctype', ['src', 'content'])
@@ -110,37 +112,65 @@ def docs_xml(docs:list[str],  # The content of each document
     return pre + to_xml(Documents(docs), do_escape=False)
 
 # %% ../00_xml.ipynb
-def read_file(fname):
+def read_file(fname, out=True):
     "Read file content, converting notebooks to XML if needed"
     fname = Path(fname)
-    if fname.suffix == '.ipynb': return nb2xml(fname)
+    if fname.suffix == '.ipynb': return nb2xml(fname, out=out)
     return fname.read_text()
 
 # %% ../00_xml.ipynb
 def files2ctx(
     fnames:list[Union[str,Path]], # List of file names to add to context
-    prefix:bool=True # Include Anthropic's suggested prose intro?
+    prefix:bool=True, # Include Anthropic's suggested prose intro?
+    out:bool=True, # Include notebook cell outputs?
+    srcs:Optional[list]=None # Use the labels instead of `fnames`
 )->str: # XML for LM context
     "Convert files to XML context, handling notebooks"
     fnames = [Path(o) for o in fnames]
-    contents = [read_file(o) for o in fnames]
-    return docs_xml(contents, fnames, prefix=prefix)
+    contents = [read_file(o, out=out) for o in fnames]
+    return docs_xml(contents, srcs or fnames, prefix=prefix)
 
 # %% ../00_xml.ipynb
 @delegates(globtastic)
 def folder2ctx(
-    folder:Union[str,Path], # Folder name containing files to add to context
-    prefix:bool=True, # Include Anthropic's suggested prose intro?
-    **kwargs # Passed to `globtastic`
-)->str: # XML for Claude context
+    folder:Union[str,Path],
+    prefix:bool=True,
+    out:bool=True,
+    include_base:bool=True,
+    **kwargs
+)->str:
+    "Convert folder contents to XML context, handling notebooks"
+    folder = Path(folder)
     fnames = globtastic(folder, **kwargs)
-    return files2ctx(fnames, prefix=prefix)
+    srcs = fnames if include_base else [Path(f).relative_to(folder) for f in fnames]
+    return files2ctx(fnames, prefix=prefix, out=out, srcs=srcs)
+
+# %% ../00_xml.ipynb
+@delegates(folder2ctx)
+def repo2ctx(
+    owner:str,  # GitHub repo owner
+    repo:str,   # GitHub repo name
+    ref:str=None,  # Git ref (branch/tag/sha); defaults to repo's default branch
+    **kwargs  # Passed to `folder2ctx`
+)->str:  # XML for LM context
+    "Convert GitHub repo to XML context without cloning"
+    import tempfile, tarfile, io
+    api = GhApi()
+    if ref is None: ref = api.repos.get(owner, repo).default_branch
+    data = api.repos.download_tarball_archive(owner, repo, ref)
+    tf = tarfile.open(fileobj=io.BytesIO(data))
+    with tempfile.TemporaryDirectory() as tmp:
+        tf.extractall(tmp, filter='data')
+        subdir = Path(tmp) / tf.getmembers()[0].name.split('/')[0]
+        return folder2ctx(subdir, include_base=False, **kwargs)
 
 # %% ../00_xml.ipynb
 @call_parse
 @delegates(folder2ctx)
 def folder2ctx_cli(
     folder:str, # Folder name containing files to add to context
+    out:bool=True, # Include notebook cell outputs?
     **kwargs # Passed to `folder2ctx`
 )->str: # XML for Claude context
-    print(folder2ctx(folder, **kwargs))
+    "CLI to convert folder contents to XML context, handling notebooks"
+    print(folder2ctx(folder, out=out, **kwargs))

{toolslm-0.3.6 → toolslm-0.3.8/toolslm.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: toolslm
-Version: 0.3.6
+Version: 0.3.8
 Summary: Tools to make language models a bit easier to use
 Home-page: https://github.com/AnswerDotAI/toolslm
 Author: Jeremy Howard
@@ -16,7 +16,7 @@ Classifier: License :: OSI Approved :: Apache Software License
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: fastcore>=1.8.11
+Requires-Dist: fastcore>=1.9.6
 Requires-Dist: httpx
 Provides-Extra: dev
 Dynamic: author

toolslm-0.3.8/toolslm.egg-info/requires.txt

@@ -0,0 +1,4 @@
+fastcore>=1.9.6
+httpx
+
+[dev]

toolslm-0.3.6/toolslm/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.3.6"

toolslm-0.3.6/toolslm.egg-info/requires.txt

@@ -1,4 +0,0 @@
-fastcore>=1.8.11
-httpx
-
-[dev]