ipykernel-helper 0.0.10__tar.gz → 0.0.14__tar.gz

{ipykernel_helper-0.0.10/ipykernel_helper.egg-info → ipykernel_helper-0.0.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ipykernel-helper
-Version: 0.0.10
+Version: 0.0.14
 Summary: Helpers for ipykernel and friends
 Home-page: https://github.com/AnswerDotAI/ipykernel-helper
 Author: Jeremy Howard
@@ -25,6 +25,7 @@ Requires-Dist: ipython
 Requires-Dist: ipykernel
 Requires-Dist: beautifulsoup4
 Requires-Dist: html2text
+Requires-Dist: cloudscraper
 Provides-Extra: dev
 Dynamic: author
 Dynamic: author-email

ipykernel_helper-0.0.14/ipykernel_helper/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "0.0.14"
+from .core import *

{ipykernel_helper-0.0.10 → ipykernel_helper-0.0.14}/ipykernel_helper/_modidx.py

@@ -30,4 +30,5 @@ d = { 'settings': { 'branch': 'main',
                                                                                          'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core.read_url': ('core.html#read_url', 'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core.run_cmd': ('core.html#run_cmd', 'ipykernel_helper/core.py'),
+                                       'ipykernel_helper.core.scrape_url': ('core.html#scrape_url', 'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core.transient': ('core.html#transient', 'ipykernel_helper/core.py')}}}

{ipykernel_helper-0.0.10 → ipykernel_helper-0.0.14}/ipykernel_helper/core.py

@@ -3,7 +3,7 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_core.ipynb.
 
 # %% auto 0
-__all__ = ['transient', 'run_cmd', 'get_md', 'read_url', 'load_ipython_extension']
+__all__ = ['transient', 'run_cmd', 'get_md', 'scrape_url', 'read_url', 'load_ipython_extension']
 
 # %% ../nbs/00_core.ipynb
 from fastcore.meta import delegates
@@ -13,7 +13,9 @@ from inspect import signature, currentframe
 from functools import cmp_to_key,partial
 from collections.abc import Mapping
 from textwrap import dedent
+from cloudscraper import create_scraper
 from toolslm.funccall import *
+from ast import literal_eval
 
 import typing,warnings,re
 
@@ -101,10 +103,13 @@ def sig_help(self:InteractiveShell, code, line_no=None, col_no=None):
 
 # %% ../nbs/00_core.ipynb
 @patch
-def get_vars(self:InteractiveShell, vs:list):
+def get_vars(self:InteractiveShell, vs:list, literal=True):
     "Get variables from namespace."
     ns = self.user_ns
-    return {v:ns[v] for v in vs if v in ns}
+    def _maybe_eval(o):
+        try: literal_eval(repr(o)); return o
+        except: return str(o)
+    return {v:_maybe_eval(ns[v]) if literal else str(ns[v]) for v in vs if v in ns}
 
 # %% ../nbs/00_core.ipynb
 def _get_schema(ns: dict, t):
@@ -151,6 +156,9 @@ def get_md(cts):
     def _f(m): return f'```\n{dedent(m.group(1))}\n```'
     return re.sub(r'\[code]\s*\n(.*?)\n\[/code]', _f, res or '', flags=re.DOTALL).strip()
 
+# %% ../nbs/00_core.ipynb
+def scrape_url(url): return create_scraper().get(url)
+
 # %% ../nbs/00_core.ipynb
 def read_url(
     url:str, # URL to read
@@ -159,21 +167,33 @@ def read_url(
     selector:str=None # Select section(s) using BeautifulSoup.select (overrides extract_section)
 ):
     "Read URL and return contents"
-    import httpx
     from urllib.parse import urlparse
     from bs4 import BeautifulSoup
     
-    res = httpx.get(url, follow_redirects=True).raise_for_status().text
+    o = scrape_url(url)
+    res, ctype = o.text, o.headers.get('content-type').split(';')[0]
+
+    soup = BeautifulSoup(res, "html.parser")
+    
     if selector:
-        sections = BeautifulSoup(res).select(selector)
+        sections = soup.select(selector)
         if sections: res = '\n\n'.join(str(section) for section in sections)
         else: res = ''
     elif extract_section:
         parsed = urlparse(url)
         if parsed.fragment:
-            section = BeautifulSoup(res).find(id=parsed.fragment)
-            if section: res = str(section)
-    if as_md: return get_md(res)
+            section = soup.find(id=parsed.fragment)
+            if section:
+                tag_name = section.name
+                elements = [section]
+                current = section.next_sibling
+                while current:
+                    if hasattr(current, 'name') and current.name == tag_name: break
+                    elements.append(current)
+                    current = current.next_sibling
+                res = ''.join(str(el) for el in elements)
+            else: res = ''
+    if as_md and ctype == 'text/html': return get_md(res)
     return res
 
 # %% ../nbs/00_core.ipynb

{ipykernel_helper-0.0.10 → ipykernel_helper-0.0.14/ipykernel_helper.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ipykernel-helper
-Version: 0.0.10
+Version: 0.0.14
 Summary: Helpers for ipykernel and friends
 Home-page: https://github.com/AnswerDotAI/ipykernel-helper
 Author: Jeremy Howard
@@ -25,6 +25,7 @@ Requires-Dist: ipython
 Requires-Dist: ipykernel
 Requires-Dist: beautifulsoup4
 Requires-Dist: html2text
+Requires-Dist: cloudscraper
 Provides-Extra: dev
 Dynamic: author
 Dynamic: author-email

{ipykernel_helper-0.0.10 → ipykernel_helper-0.0.14}/ipykernel_helper.egg-info/requires.txt

@@ -5,5 +5,6 @@ ipython
 ipykernel
 beautifulsoup4
 html2text
+cloudscraper
 
 [dev]

{ipykernel_helper-0.0.10 → ipykernel_helper-0.0.14}/settings.ini

@@ -1,11 +1,11 @@
 [DEFAULT]
 repo = ipykernel-helper
 lib_name = ipykernel-helper
-version = 0.0.10
+version = 0.0.14
 min_python = 3.9
 license = apache2
 black_formatting = False
-requirements = fastcore toolslm>=0.2.0 jedi ipython ipykernel beautifulsoup4 html2text
+requirements = fastcore toolslm>=0.2.0 jedi ipython ipykernel beautifulsoup4 html2text cloudscraper
 cell_number = False
 doc_path = _docs
 lib_path = ipykernel_helper

ipykernel_helper-0.0.10/ipykernel_helper/__init__.py

@@ -1,2 +0,0 @@
-__version__ = "0.0.10"
-from .core import *