ipykernel-helper 0.0.10__py3-none-any.whl → 0.0.12__py3-none-any.whl

ipykernel_helper/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.0.10"
+__version__ = "0.0.12"
 from .core import *

ipykernel_helper/_modidx.py

@@ -30,4 +30,5 @@ d = { 'settings': { 'branch': 'main',
                                                                                          'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core.read_url': ('core.html#read_url', 'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core.run_cmd': ('core.html#run_cmd', 'ipykernel_helper/core.py'),
+                                       'ipykernel_helper.core.scrape_url': ('core.html#scrape_url', 'ipykernel_helper/core.py'),
                                        'ipykernel_helper.core.transient': ('core.html#transient', 'ipykernel_helper/core.py')}}}

ipykernel_helper/core.py

@@ -3,7 +3,7 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_core.ipynb.
 
 # %% auto 0
-__all__ = ['transient', 'run_cmd', 'get_md', 'read_url', 'load_ipython_extension']
+__all__ = ['transient', 'run_cmd', 'get_md', 'scrape_url', 'read_url', 'load_ipython_extension']
 
 # %% ../nbs/00_core.ipynb
 from fastcore.meta import delegates
@@ -13,6 +13,7 @@ from inspect import signature, currentframe
 from functools import cmp_to_key,partial
 from collections.abc import Mapping
 from textwrap import dedent
+from cloudscraper import create_scraper
 from toolslm.funccall import *
 
 import typing,warnings,re
@@ -151,6 +152,9 @@ def get_md(cts):
     def _f(m): return f'```\n{dedent(m.group(1))}\n```'
     return re.sub(r'\[code]\s*\n(.*?)\n\[/code]', _f, res or '', flags=re.DOTALL).strip()
 
+# %% ../nbs/00_core.ipynb
+def scrape_url(url): return create_scraper().get(url)
+
 # %% ../nbs/00_core.ipynb
 def read_url(
     url:str, # URL to read
@@ -159,20 +163,30 @@ def read_url(
     selector:str=None # Select section(s) using BeautifulSoup.select (overrides extract_section)
 ):
     "Read URL and return contents"
-    import httpx
     from urllib.parse import urlparse
     from bs4 import BeautifulSoup
     
-    res = httpx.get(url, follow_redirects=True).raise_for_status().text
+    res = scrape_url(url).text
+    soup = BeautifulSoup(res, "html.parser")
+    
     if selector:
-        sections = BeautifulSoup(res).select(selector)
+        sections = soup.select(selector)
         if sections: res = '\n\n'.join(str(section) for section in sections)
         else: res = ''
     elif extract_section:
         parsed = urlparse(url)
         if parsed.fragment:
-            section = BeautifulSoup(res).find(id=parsed.fragment)
-            if section: res = str(section)
+            section = soup.find(id=parsed.fragment)
+            if section:
+                tag_name = section.name
+                elements = [section]
+                current = section.next_sibling
+                while current:
+                    if hasattr(current, 'name') and current.name == tag_name: break
+                    elements.append(current)
+                    current = current.next_sibling
+                res = ''.join(str(el) for el in elements)
+            else: res = ''
     if as_md: return get_md(res)
     return res
 

{ipykernel_helper-0.0.10.dist-info → ipykernel_helper-0.0.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ipykernel-helper
-Version: 0.0.10
+Version: 0.0.12
 Summary: Helpers for ipykernel and friends
 Home-page: https://github.com/AnswerDotAI/ipykernel-helper
 Author: Jeremy Howard
@@ -25,6 +25,7 @@ Requires-Dist: ipython
 Requires-Dist: ipykernel
 Requires-Dist: beautifulsoup4
 Requires-Dist: html2text
+Requires-Dist: cloudscraper
 Provides-Extra: dev
 Dynamic: author
 Dynamic: author-email

ipykernel_helper-0.0.12.dist-info/RECORD

@@ -0,0 +1,9 @@
+ipykernel_helper/__init__.py,sha256=NBQPBiYNpbwRh0tbd9ruOAJTKO2W4erG6F75QoavDZQ,43
+ipykernel_helper/_modidx.py,sha256=AEG4MccZfb25rQi8UWrOpUrs1VIYHbexKEQGWnwYkRg,3706
+ipykernel_helper/core.py,sha256=22E6FiiTI7Q8vPESVUHXyz8CoOWxDLkDe8tlSdJNEMQ,8150
+ipykernel_helper-0.0.12.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+ipykernel_helper-0.0.12.dist-info/METADATA,sha256=EYR5VZ2ZiIDN_Jn_UAKqFqoKYVnnknoj_bT_9Th-x60,2709
+ipykernel_helper-0.0.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+ipykernel_helper-0.0.12.dist-info/entry_points.txt,sha256=HWiK9xz75QtZUaPaYrwpyH5B8MbW0Ea_vi11UmwBImM,54
+ipykernel_helper-0.0.12.dist-info/top_level.txt,sha256=_diD--64d9MauLE0pTxzZ58lkI8DvCrVc1hVAJsyc_Q,17
+ipykernel_helper-0.0.12.dist-info/RECORD,,

ipykernel_helper-0.0.10.dist-info/RECORD

@@ -1,9 +0,0 @@
-ipykernel_helper/__init__.py,sha256=vGb5bWRnqikdeGjmhzb9R8htmRhc2CloD1a9bB7Ttvo,43
-ipykernel_helper/_modidx.py,sha256=INE9z1o4nb_NMRepxbiF5fBE5Il5BeKJZFTLmdxUGLw,3577
-ipykernel_helper/core.py,sha256=hf7ml7uXFjzllCvwuiRVfu8NQlq9AQ3Xdfd3trkY0sk,7650
-ipykernel_helper-0.0.10.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-ipykernel_helper-0.0.10.dist-info/METADATA,sha256=4ZPHVdmWsfST3aFKoJFXRbc3ODHNIDJRotcz9F02YDI,2681
-ipykernel_helper-0.0.10.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ipykernel_helper-0.0.10.dist-info/entry_points.txt,sha256=HWiK9xz75QtZUaPaYrwpyH5B8MbW0Ea_vi11UmwBImM,54
-ipykernel_helper-0.0.10.dist-info/top_level.txt,sha256=_diD--64d9MauLE0pTxzZ58lkI8DvCrVc1hVAJsyc_Q,17
-ipykernel_helper-0.0.10.dist-info/RECORD,,