ipykernel-helper 0.0.14__py3-none-any.whl → 0.0.20__py3-none-any.whl

ipykernel_helper/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.0.14"
+__version__ = "0.0.20"
 from .core import *

ipykernel_helper/_modidx.py

@@ -21,13 +21,23 @@ d = { 'settings': { 'branch': 'main',
                                                                                               'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core.InteractiveShell.xpush': ( 'core.html#interactiveshell.xpush',
                                                                                          'ipykernel_helper/core.py'),
+                                       'ipykernel_helper.core._absolutify_imgs': ('core.html#_absolutify_imgs', 'ipykernel_helper/core.py'),
+                                       'ipykernel_helper.core._aify_imgs': ('core.html#_aify_imgs', 'ipykernel_helper/core.py'),
+                                       'ipykernel_helper.core._convert_math': ('core.html#_convert_math', 'ipykernel_helper/core.py'),
+                                       'ipykernel_helper.core._extract_section': ('core.html#_extract_section', 'ipykernel_helper/core.py'),
+                                       'ipykernel_helper.core._get_math_mode': ('core.html#_get_math_mode', 'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core._get_schema': ('core.html#_get_schema', 'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core._rank': ('core.html#_rank', 'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core._safe_repr': ('core.html#_safe_repr', 'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core._signatures': ('core.html#_signatures', 'ipykernel_helper/core.py'),
+                                       'ipykernel_helper.core.fix_editable_priority': ( 'core.html#fix_editable_priority',
+                                                                                        'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core.get_md': ('core.html#get_md', 'ipykernel_helper/core.py'),
+                                       'ipykernel_helper.core.gh_blob_to_raw': ('core.html#gh_blob_to_raw', 'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core.load_ipython_extension': ( 'core.html#load_ipython_extension',
                                                                                          'ipykernel_helper/core.py'),
+                                       'ipykernel_helper.core.parse_gh_url': ('core.html#parse_gh_url', 'ipykernel_helper/core.py'),
+                                       'ipykernel_helper.core.read_gh_repo': ('core.html#read_gh_repo', 'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core.read_url': ('core.html#read_url', 'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core.run_cmd': ('core.html#run_cmd', 'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core.scrape_url': ('core.html#scrape_url', 'ipykernel_helper/core.py'),

ipykernel_helper/core.py

@@ -3,7 +3,8 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_core.ipynb.
 
 # %% auto 0
-__all__ = ['transient', 'run_cmd', 'get_md', 'scrape_url', 'read_url', 'load_ipython_extension']
+__all__ = ['transient', 'run_cmd', 'get_md', 'scrape_url', 'parse_gh_url', 'gh_blob_to_raw', 'read_gh_repo', 'read_url',
+           'fix_editable_priority', 'load_ipython_extension']
 
 # %% ../nbs/00_core.ipynb
 from fastcore.meta import delegates
@@ -16,8 +17,11 @@ from textwrap import dedent
 from cloudscraper import create_scraper
 from toolslm.funccall import *
 from ast import literal_eval
+from urllib.parse import urlparse, urljoin
+from fastcore.net import HTTP404NotFoundError
+from ghapi.all import GhApi
 
-import typing,warnings,re
+import typing,warnings,re,os,html2text,base64
 
 from IPython.core.interactiveshell import InteractiveShell
 from IPython.core.completer import ProvisionalCompleterWarning
@@ -115,7 +119,7 @@ def get_vars(self:InteractiveShell, vs:list, literal=True):
 def _get_schema(ns: dict, t):
     "Check if tool `t` has errors."
     if t not in ns: return f"`{t}` not found. Did you run it?"
-    try: return get_schema(ns[t])
+    try: return {'type':'function', 'function':get_schema(ns[t], pname='parameters', evalable=True, skip_hidden=True)}
     except Exception as e: return f"`{t}`: {e}."
 
 @patch
@@ -146,56 +150,122 @@ def run_cmd(cmd, data='', meta=None, update=False, **kw):
     transient(data, meta=meta, update=update, cmd=cmd, **kw)
 
 # %% ../nbs/00_core.ipynb
-def get_md(cts):
-    from html2text import HTML2Text
-    h2t = HTML2Text(bodywidth=5000)
-    h2t.ignore_links = False
-    h2t.mark_code = True
-    h2t.ignore_images = False
-    res = h2t.handle(cts)
-    def _f(m): return f'```\n{dedent(m.group(1))}\n```'
-    return re.sub(r'\[code]\s*\n(.*?)\n\[/code]', _f, res or '', flags=re.DOTALL).strip()
+def _absolutify_imgs(md, base_url):
+    def fix(m):
+        alt,img_url = m.group(1),m.group(2)
+        if not img_url.startswith('http'): img_url = urljoin(base_url, img_url)
+        alt = alt.replace('\\','')
+        return f'![{alt}]({img_url})'
+    return re.sub(r'!\[(.*?)\]\((.*?)\)', fix, md)
+
+# %% ../nbs/00_core.ipynb
+def get_md(html, url='', mmode=None, ignore_links=False, ignore_images=False, mark_code=True):
+    "Convert HTML to markdown with absolute image URLs and optional math mode"
+    h = html2text.HTML2Text()
+    h.body_width = 0
+    h.ignore_links, h.ignore_images, h.mark_code = ignore_links, ignore_images, mark_code
+    res = _absolutify_imgs(h.handle(str(html)), url)
+    if mmode == 'safe': res = res.replace(r'\\(',r'\(').replace(r'\\)',r'\)')
+    return re.sub(r'\[code]\s*\n(.*?)\n\[/code]', lambda m: f'```\n{dedent(m.group(1))}\n```', res, flags=re.DOTALL).strip()
 
 # %% ../nbs/00_core.ipynb
 def scrape_url(url): return create_scraper().get(url)
 
 # %% ../nbs/00_core.ipynb
-def read_url(
-    url:str, # URL to read
-    as_md:bool=True, # Convert HTML to Markdown?
-    extract_section:bool=True, # If url has an anchor, return only that section
-    selector:str=None # Select section(s) using BeautifulSoup.select (overrides extract_section)
-):
-    "Read URL and return contents"
-    from urllib.parse import urlparse
+def _get_math_mode():
+    v = os.getenv('USE_KATEX', '')
+    if v.lower() in {'0', 'false', 'none', ''}: return None
+    return 'dollar' if v.lower().startswith('d') else 'safe'
+
+# %% ../nbs/00_core.ipynb
+def _aify_imgs(md): return re.sub(r'!\[(.*?)\]\((.*?)\)', r'![\1](\2#ai)', md)
+
+# %% ../nbs/00_core.ipynb
+def parse_gh_url(url):
+    "Parse GitHub URL into (owner, repo, type, ref, path) or None"
+    m = re.match(r'https?://(?:www\.)?github\.com/([^/]+)/([^/]+)(?:/([^/]+)(?:/([^/]+)(?:/(.+))?)?)?', url)
+    return dict(zip('owner repo typ ref path'.split(), m.groups())) if m else None
+
+# %% ../nbs/00_core.ipynb
+def gh_blob_to_raw(url):
+    "Convert github.com/user/repo/blob/... URL to raw.githubusercontent.com URL"
+    m = re.match(r'https?://(?:www\.)?github\.com/([^/]+)/([^/]+)/blob/([^/]+)/(.+)', url)
+    if not m: return url
+    owner, repo, ref, path = m.groups()
+    return f'https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path}'
+
+# %% ../nbs/00_core.ipynb
+def _extract_section(soup, url, selector=None):
+    "Extract a specific section from soup, or the whole thing"
+    if selector: return '\n\n'.join(str(s) for s in soup.select(selector))
+    parsed = urlparse(url)
+    if not parsed.fragment: return str(soup)
+    section = soup.find(id=parsed.fragment)
+    if not section: return ''
+    elements = [section]
+    current = section.next_sibling
+    while current:
+        if hasattr(current, 'name') and current.name == section.name: break
+        elements.append(current)
+        current = current.next_sibling
+    return ''.join(str(el) for el in elements)
+
+# %% ../nbs/00_core.ipynb
+def _convert_math(soup, mode):
+    for math in soup.find_all('math'):
+        annot = math.find('annotation', {'encoding': 'application/x-tex'})
+        if not annot: continue
+        tex,display = annot.text.strip(), math.get('display') == 'block'
+        if mode == 'dollar': wrap = f'$${tex}$$' if display else f'${tex}$'
+        else: wrap = f'$${tex}$$' if display else f'\({tex}\)'
+        math.replace_with(wrap)
+
+# %% ../nbs/00_core.ipynb
+def read_gh_repo(owner, repo, ref=None, path=''):
+    "Read GitHub repo info: description, file list, and README"
+    api = GhApi()
+    info = api.repos.get(owner, repo)
+    res = [f"# {info.full_name}", info.description or '']
+    ref = ref or info.default_branch
+    contents = api.repos.get_content(owner, repo, path or '', ref=ref)
+    files = [f"- {'📁 ' if c.type=='dir' else ''}{c.name}" for c in contents]
+    res.append(f'\n## /{path or ""} Files\n' + '\n'.join(files))
+    if not path:
+        try:
+            readme = api.repos.get_readme(owner, repo, ref=ref)
+            res.append('\n## README\n' + base64.b64decode(readme.content).decode())
+        except HTTP404NotFoundError: pass
+    return '\n'.join(res)
+
+# %% ../nbs/00_core.ipynb
+def read_url(url:str, as_md:bool=True, extract_section:bool=True, selector:str=None, ai_img:bool=False):
+    "Read url from web"
     from bs4 import BeautifulSoup
-    
+    gh = parse_gh_url(url)
+    if gh:
+        if gh['typ']=='blob': url = gh_blob_to_raw(url)
+        elif gh['typ'] in (None, 'tree'): return read_gh_repo(gh['owner'], gh['repo'], gh['ref'], gh['path'])
     o = scrape_url(url)
-    res, ctype = o.text, o.headers.get('content-type').split(';')[0]
-
-    soup = BeautifulSoup(res, "html.parser")
-    
-    if selector:
-        sections = soup.select(selector)
-        if sections: res = '\n\n'.join(str(section) for section in sections)
-        else: res = ''
-    elif extract_section:
-        parsed = urlparse(url)
-        if parsed.fragment:
-            section = soup.find(id=parsed.fragment)
-            if section:
-                tag_name = section.name
-                elements = [section]
-                current = section.next_sibling
-                while current:
-                    if hasattr(current, 'name') and current.name == tag_name: break
-                    elements.append(current)
-                    current = current.next_sibling
-                res = ''.join(str(el) for el in elements)
-            else: res = ''
-    if as_md and ctype == 'text/html': return get_md(res)
+    ctype = (o.headers.get('content-type') or 'text/plain').split(';')[0]
+    res = o.text
+    if ctype == 'text/html':
+        soup = BeautifulSoup(res, 'lxml')
+        if ('#' in url and extract_section) or selector: soup = BeautifulSoup(_extract_section(soup, url, selector), 'lxml')
+        mmode = _get_math_mode()
+        if mmode: _convert_math(soup, mmode)
+        base = soup.find('base')
+        base_url = urljoin(url, base['href'] if base else '')
+        res = get_md(soup, base_url, mmode) if as_md else str(soup)
+    if ai_img: res = _aify_imgs(res)
     return res
 
+# %% ../nbs/00_core.ipynb
+def fix_editable_priority():
+    import sys
+    from importlib.machinery import PathFinder
+    try: sys.meta_path.append(sys.meta_path.pop(sys.meta_path.index(PathFinder)))
+    except ValueError: pass
+
 # %% ../nbs/00_core.ipynb
 @patch
 def _get_info(self:Inspector, obj, oname='', formatter=None, info=None, detail_level=0, omit_sections=()):

{ipykernel_helper-0.0.14.dist-info → ipykernel_helper-0.0.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ipykernel-helper
-Version: 0.0.14
+Version: 0.0.20
 Summary: Helpers for ipykernel and friends
 Home-page: https://github.com/AnswerDotAI/ipykernel-helper
 Author: Jeremy Howard
@@ -19,13 +19,15 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: fastcore
-Requires-Dist: toolslm>=0.2.0
+Requires-Dist: toolslm>=0.3.11
 Requires-Dist: jedi
 Requires-Dist: ipython
 Requires-Dist: ipykernel
 Requires-Dist: beautifulsoup4
+Requires-Dist: lxml
 Requires-Dist: html2text
 Requires-Dist: cloudscraper
+Requires-Dist: ghapi
 Provides-Extra: dev
 Dynamic: author
 Dynamic: author-email

ipykernel_helper-0.0.20.dist-info/RECORD

@@ -0,0 +1,9 @@
+ipykernel_helper/__init__.py,sha256=mkmmPV_a7e3EgojncsapZLZfAn0jyTfnZIQiO1GWA0M,43
+ipykernel_helper/_modidx.py,sha256=QxDoY2rsZgiO12nSsD7YkAqIri_1_Y38uxVGxLJkd2Y,5032
+ipykernel_helper/core.py,sha256=0BEYTAdT1Wohip_glNJFmpumhCcOI22vfuCoOeheQE4,11961
+ipykernel_helper-0.0.20.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+ipykernel_helper-0.0.20.dist-info/METADATA,sha256=dIl05mep_Des63g4tVx0cD9ZP7VzPFDbDy4SzaVT2ts,2751
+ipykernel_helper-0.0.20.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ipykernel_helper-0.0.20.dist-info/entry_points.txt,sha256=HWiK9xz75QtZUaPaYrwpyH5B8MbW0Ea_vi11UmwBImM,54
+ipykernel_helper-0.0.20.dist-info/top_level.txt,sha256=_diD--64d9MauLE0pTxzZ58lkI8DvCrVc1hVAJsyc_Q,17
+ipykernel_helper-0.0.20.dist-info/RECORD,,

ipykernel_helper-0.0.14.dist-info/RECORD

@@ -1,9 +0,0 @@
-ipykernel_helper/__init__.py,sha256=z-0Rh9e-quhc6YCgBCQPxO9WBChy0UBbaCcAmIPEqfE,43
-ipykernel_helper/_modidx.py,sha256=AEG4MccZfb25rQi8UWrOpUrs1VIYHbexKEQGWnwYkRg,3706
-ipykernel_helper/core.py,sha256=yPDNnVGPURLVcNb7NcikqKvbdifxybduIdkyJdzHz1w,8420
-ipykernel_helper-0.0.14.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-ipykernel_helper-0.0.14.dist-info/METADATA,sha256=9kObKdTx58-Kw5Ecll9WelZw8LfYBViyzJ0rhAfLGb4,2709
-ipykernel_helper-0.0.14.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ipykernel_helper-0.0.14.dist-info/entry_points.txt,sha256=HWiK9xz75QtZUaPaYrwpyH5B8MbW0Ea_vi11UmwBImM,54
-ipykernel_helper-0.0.14.dist-info/top_level.txt,sha256=_diD--64d9MauLE0pTxzZ58lkI8DvCrVc1hVAJsyc_Q,17
-ipykernel_helper-0.0.14.dist-info/RECORD,,