ipykernel-helper 0.0.14__tar.gz → 0.0.25__tar.gz

{ipykernel_helper-0.0.14/ipykernel_helper.egg-info → ipykernel_helper-0.0.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ipykernel-helper
-Version: 0.0.14
+Version: 0.0.25
 Summary: Helpers for ipykernel and friends
 Home-page: https://github.com/AnswerDotAI/ipykernel-helper
 Author: Jeremy Howard
@@ -18,14 +18,16 @@ Classifier: License :: OSI Approved :: Apache Software License
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: fastcore
-Requires-Dist: toolslm>=0.2.0
+Requires-Dist: fastcore>=1.11.0
+Requires-Dist: toolslm>=0.3.13
 Requires-Dist: jedi
 Requires-Dist: ipython
 Requires-Dist: ipykernel
 Requires-Dist: beautifulsoup4
+Requires-Dist: lxml
 Requires-Dist: html2text
 Requires-Dist: cloudscraper
+Requires-Dist: ghapi
 Provides-Extra: dev
 Dynamic: author
 Dynamic: author-email

ipykernel_helper-0.0.25/ipykernel_helper/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "0.0.25"
+from .core import *

{ipykernel_helper-0.0.14 → ipykernel_helper-0.0.25}/ipykernel_helper/_modidx.py

@@ -21,13 +21,22 @@ d = { 'settings': { 'branch': 'main',
                                                                                               'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core.InteractiveShell.xpush': ( 'core.html#interactiveshell.xpush',
                                                                                          'ipykernel_helper/core.py'),
+                                       'ipykernel_helper.core._absolutify_imgs': ('core.html#_absolutify_imgs', 'ipykernel_helper/core.py'),
+                                       'ipykernel_helper.core._aify_imgs': ('core.html#_aify_imgs', 'ipykernel_helper/core.py'),
+                                       'ipykernel_helper.core._convert_math': ('core.html#_convert_math', 'ipykernel_helper/core.py'),
+                                       'ipykernel_helper.core._extract_section': ('core.html#_extract_section', 'ipykernel_helper/core.py'),
+                                       'ipykernel_helper.core._get_math_mode': ('core.html#_get_math_mode', 'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core._get_schema': ('core.html#_get_schema', 'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core._rank': ('core.html#_rank', 'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core._safe_repr': ('core.html#_safe_repr', 'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core._signatures': ('core.html#_signatures', 'ipykernel_helper/core.py'),
+                                       'ipykernel_helper.core.fix_editable_priority': ( 'core.html#fix_editable_priority',
+                                                                                        'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core.get_md': ('core.html#get_md', 'ipykernel_helper/core.py'),
+                                       'ipykernel_helper.core.gh_blob_to_raw': ('core.html#gh_blob_to_raw', 'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core.load_ipython_extension': ( 'core.html#load_ipython_extension',
                                                                                          'ipykernel_helper/core.py'),
+                                       'ipykernel_helper.core.read_gh_repo': ('core.html#read_gh_repo', 'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core.read_url': ('core.html#read_url', 'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core.run_cmd': ('core.html#run_cmd', 'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core.scrape_url': ('core.html#scrape_url', 'ipykernel_helper/core.py'),

{ipykernel_helper-0.0.14 → ipykernel_helper-0.0.25}/ipykernel_helper/core.py

@@ -3,11 +3,14 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_core.ipynb.
 
 # %% auto 0
-__all__ = ['transient', 'run_cmd', 'get_md', 'scrape_url', 'read_url', 'load_ipython_extension']
+__all__ = ['transient', 'run_cmd', 'get_md', 'scrape_url', 'gh_blob_to_raw', 'read_gh_repo', 'read_url', 'fix_editable_priority',
+           'load_ipython_extension']
 
 # %% ../nbs/00_core.ipynb
 from fastcore.meta import delegates
 from fastcore.utils import patch,dict2obj
+from fastcore.docments import sig_source,DocmentText
+from fastcore.net import HTTP404NotFoundError
 from types import ModuleType, FunctionType, MethodType, BuiltinFunctionType
 from inspect import signature, currentframe
 from functools import cmp_to_key,partial
@@ -15,9 +18,12 @@ from collections.abc import Mapping
 from textwrap import dedent
 from cloudscraper import create_scraper
 from toolslm.funccall import *
+from toolslm.xml import *
 from ast import literal_eval
+from urllib.parse import urlparse, urljoin
+from ghapi.all import GhApi
 
-import typing,warnings,re
+import typing,warnings,re,os,html2text,base64
 
 from IPython.core.interactiveshell import InteractiveShell
 from IPython.core.completer import ProvisionalCompleterWarning
@@ -115,7 +121,7 @@ def get_vars(self:InteractiveShell, vs:list, literal=True):
 def _get_schema(ns: dict, t):
     "Check if tool `t` has errors."
     if t not in ns: return f"`{t}` not found. Did you run it?"
-    try: return get_schema(ns[t])
+    try: return {'type':'function', 'function':get_schema(ns[t], pname='parameters', evalable=True, skip_hidden=True)}
     except Exception as e: return f"`{t}`: {e}."
 
 @patch
@@ -146,65 +152,130 @@ def run_cmd(cmd, data='', meta=None, update=False, **kw):
     transient(data, meta=meta, update=update, cmd=cmd, **kw)
 
 # %% ../nbs/00_core.ipynb
-def get_md(cts):
-    from html2text import HTML2Text
-    h2t = HTML2Text(bodywidth=5000)
-    h2t.ignore_links = False
-    h2t.mark_code = True
-    h2t.ignore_images = False
-    res = h2t.handle(cts)
-    def _f(m): return f'```\n{dedent(m.group(1))}\n```'
-    return re.sub(r'\[code]\s*\n(.*?)\n\[/code]', _f, res or '', flags=re.DOTALL).strip()
+def _absolutify_imgs(md, base_url):
+    def fix(m):
+        alt,img_url = m.group(1),m.group(2)
+        if not img_url.startswith('http'): img_url = urljoin(base_url, img_url)
+        alt = alt.replace('\\','')
+        return f'![{alt}]({img_url})'
+    return re.sub(r'!\[(.*?)\]\((.*?)\)', fix, md)
+
+# %% ../nbs/00_core.ipynb
+def get_md(html, url='', mmode=None, ignore_links=False, ignore_images=False, mark_code=True):
+    "Convert HTML to markdown with absolute image URLs and optional math mode"
+    h = html2text.HTML2Text()
+    h.body_width = 0
+    h.ignore_links, h.ignore_images, h.mark_code = ignore_links, ignore_images, mark_code
+    res = _absolutify_imgs(h.handle(str(html)), url)
+    if mmode == 'safe': res = res.replace(r'\\(',r'\(').replace(r'\\)',r'\)')
+    return re.sub(r'\[code]\s*\n(.*?)\n\[/code]', lambda m: f'```\n{dedent(m.group(1))}\n```', res, flags=re.DOTALL).strip()
 
 # %% ../nbs/00_core.ipynb
 def scrape_url(url): return create_scraper().get(url)
 
 # %% ../nbs/00_core.ipynb
-def read_url(
-    url:str, # URL to read
-    as_md:bool=True, # Convert HTML to Markdown?
-    extract_section:bool=True, # If url has an anchor, return only that section
-    selector:str=None # Select section(s) using BeautifulSoup.select (overrides extract_section)
-):
-    "Read URL and return contents"
-    from urllib.parse import urlparse
+def _get_math_mode():
+    v = os.getenv('USE_KATEX', '')
+    if v.lower() in {'0', 'false', 'none', ''}: return None
+    return 'dollar' if v.lower().startswith('d') else 'safe'
+
+# %% ../nbs/00_core.ipynb
+def _aify_imgs(md): return re.sub(r'!\[(.*?)\]\((.*?)\)', r'![\1](\2#ai)', md)
+
+# %% ../nbs/00_core.ipynb
+def gh_blob_to_raw(url):
+    "Convert github.com/user/repo/blob/... URL to raw.githubusercontent.com URL"
+    m = re.match(r'https?://(?:www\.)?github\.com/([^/]+)/([^/]+)/blob/([^/]+)/(.+)', url)
+    if not m: return url
+    owner, repo, ref, path = m.groups()
+    return f'https://raw.githubusercontent.com/{owner}/{repo}/{ref}/{path}'
+
+# %% ../nbs/00_core.ipynb
+def _extract_section(soup, url, selector=None):
+    "Extract a specific section from soup, or the whole thing"
+    if selector: return '\n\n'.join(str(s) for s in soup.select(selector))
+    parsed = urlparse(url)
+    if not parsed.fragment: return str(soup)
+    section = soup.find(id=parsed.fragment)
+    if not section: return ''
+    elements = [section]
+    current = section.next_sibling
+    while current:
+        if hasattr(current, 'name') and current.name == section.name: break
+        elements.append(current)
+        current = current.next_sibling
+    return ''.join(str(el) for el in elements)
+
+# %% ../nbs/00_core.ipynb
+def _convert_math(soup, mode):
+    for math in soup.find_all('math'):
+        annot = math.find('annotation', {'encoding': 'application/x-tex'})
+        if not annot: continue
+        tex,display = annot.text.strip(), math.get('display') == 'block'
+        if mode == 'dollar': wrap = f'$${tex}$$' if display else f'${tex}$'
+        else: wrap = f'$${tex}$$' if display else fr'\({tex}\)'
+        math.replace_with(wrap)
+
+# %% ../nbs/00_core.ipynb
+def read_gh_repo(owner, repo, ref=None, path=''):
+    "Read GitHub repo info: description, file list, and README"
+    api = GhApi()
+    info = api.repos.get(owner, repo)
+    res = [f"# {info.full_name}", info.description or '']
+    ref = ref or info.default_branch
+    contents = api.repos.get_content(owner, repo, path or '', ref=ref)
+    files = [f"- {'📁 ' if c.type=='dir' else ''}{c.name}" for c in contents]
+    res.append(f'\n## /{path or ""} Files\n' + '\n'.join(files))
+    if not path:
+        try:
+            readme = api.repos.get_readme(owner, repo, ref=ref)
+            res.append('\n## README\n' + base64.b64decode(readme.content).decode())
+        except HTTP404NotFoundError: pass
+    return '\n'.join(res)
+
+# %% ../nbs/00_core.ipynb
+def read_url(url:str, as_md:bool=True, extract_section:bool=True, selector:str=None, ai_img:bool=False):
+    "Read url from web"
     from bs4 import BeautifulSoup
-    
+    gh = parse_gh_url(url)
+    if gh:
+        if gh['typ']=='blob': url = gh_blob_to_raw(url)
+        elif gh['typ'] in (None, 'tree'): return read_gh_repo(gh['owner'], gh['repo'], gh['ref'], gh['path'])
     o = scrape_url(url)
-    res, ctype = o.text, o.headers.get('content-type').split(';')[0]
-
-    soup = BeautifulSoup(res, "html.parser")
-    
-    if selector:
-        sections = soup.select(selector)
-        if sections: res = '\n\n'.join(str(section) for section in sections)
-        else: res = ''
-    elif extract_section:
-        parsed = urlparse(url)
-        if parsed.fragment:
-            section = soup.find(id=parsed.fragment)
-            if section:
-                tag_name = section.name
-                elements = [section]
-                current = section.next_sibling
-                while current:
-                    if hasattr(current, 'name') and current.name == tag_name: break
-                    elements.append(current)
-                    current = current.next_sibling
-                res = ''.join(str(el) for el in elements)
-            else: res = ''
-    if as_md and ctype == 'text/html': return get_md(res)
+    ctype = (o.headers.get('content-type') or 'text/plain').split(';')[0]
+    res = o.text
+    if ctype == 'text/html':
+        soup = BeautifulSoup(res, 'lxml')
+        if ('#' in url and extract_section) or selector: soup = BeautifulSoup(_extract_section(soup, url, selector), 'lxml')
+        mmode = _get_math_mode()
+        if mmode: _convert_math(soup, mmode)
+        base = soup.find('base')
+        base_url = urljoin(url, base['href'] if base else '')
+        res = get_md(soup, base_url, mmode) if as_md else str(soup)
+    if ai_img: res = _aify_imgs(res)
     return res
 
+# %% ../nbs/00_core.ipynb
+def fix_editable_priority():
+    import sys
+    from importlib.machinery import PathFinder
+    try: sys.meta_path.append(sys.meta_path.pop(sys.meta_path.index(PathFinder)))
+    except ValueError: pass
+
 # %% ../nbs/00_core.ipynb
 @patch
 def _get_info(self:Inspector, obj, oname='', formatter=None, info=None, detail_level=0, omit_sections=()):
-    "Custom formatter for ?? output"
+    "Custom formatter for ? and ?? output"
     orig = self._orig__get_info(obj, oname=oname, formatter=formatter, info=info,
                                detail_level=detail_level, omit_sections=omit_sections)
-    if detail_level==0: return orig
-    info_dict = self.info(obj, oname=oname, info=info, detail_level=detail_level)
+    info_dict = self.info(obj, oname=oname, info=info, detail_level=2)
     out = []
+    if detail_level==0:
+        out.append(f"```python\n{DocmentText(obj)}\n```")
+        if c:=info_dict.get('docstring'): out.append(c)
+        if c:=info_dict.get('file'): out.append(f"**File:** `{c}`")
+        if c:=info_dict.get('type_name'): out.append(f"**Type:** {c}")
+        return {'text/markdown': '\n\n'.join(out), 'text/html': '', 'text/plain': orig['text/plain']}
     if c:=info_dict.get('source'): out.append(f"\n```python\n{dedent(c)}\n```")
     if c:=info_dict.get('file'): out.append(f"**File:** `{c}`")
     return {'text/markdown': '\n\n'.join(out), 'text/html': '', 'text/plain': orig['text/plain']}

{ipykernel_helper-0.0.14 → ipykernel_helper-0.0.25/ipykernel_helper.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ipykernel-helper
-Version: 0.0.14
+Version: 0.0.25
 Summary: Helpers for ipykernel and friends
 Home-page: https://github.com/AnswerDotAI/ipykernel-helper
 Author: Jeremy Howard
@@ -18,14 +18,16 @@ Classifier: License :: OSI Approved :: Apache Software License
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: fastcore
-Requires-Dist: toolslm>=0.2.0
+Requires-Dist: fastcore>=1.11.0
+Requires-Dist: toolslm>=0.3.13
 Requires-Dist: jedi
 Requires-Dist: ipython
 Requires-Dist: ipykernel
 Requires-Dist: beautifulsoup4
+Requires-Dist: lxml
 Requires-Dist: html2text
 Requires-Dist: cloudscraper
+Requires-Dist: ghapi
 Provides-Extra: dev
 Dynamic: author
 Dynamic: author-email

{ipykernel_helper-0.0.14 → ipykernel_helper-0.0.25}/ipykernel_helper.egg-info/requires.txt

@@ -1,10 +1,12 @@
-fastcore
-toolslm>=0.2.0
+fastcore>=1.11.0
+toolslm>=0.3.13
 jedi
 ipython
 ipykernel
 beautifulsoup4
+lxml
 html2text
 cloudscraper
+ghapi
 
 [dev]

{ipykernel_helper-0.0.14 → ipykernel_helper-0.0.25}/settings.ini

@@ -1,11 +1,11 @@
 [DEFAULT]
 repo = ipykernel-helper
 lib_name = ipykernel-helper
-version = 0.0.14
+version = 0.0.25
 min_python = 3.9
 license = apache2
 black_formatting = False
-requirements = fastcore toolslm>=0.2.0 jedi ipython ipykernel beautifulsoup4 html2text cloudscraper
+requirements = fastcore>=1.11.0 toolslm>=0.3.13 jedi ipython ipykernel beautifulsoup4 lxml html2text cloudscraper ghapi
 cell_number = False
 doc_path = _docs
 lib_path = ipykernel_helper

ipykernel_helper-0.0.14/ipykernel_helper/__init__.py

@@ -1,2 +0,0 @@
-__version__ = "0.0.14"
-from .core import *