blue-assistant 4.283.1__py3-none-any.whl → 4.288.1__py3-none-any.whl

blue_assistant/__init__.py

@@ -4,7 +4,7 @@ ICON = "🧠"
 
 DESCRIPTION = f"{ICON} An AI Assistant."
 
-VERSION = "4.283.1"
+VERSION = "4.288.1"
 
 REPO_NAME = "blue-assistant"
 

blue_assistant/script/actions/web_crawl.py

@@ -36,14 +36,14 @@ def web_crawl(
     seed_urls = script.vars[seed_url_var_name]
     log_list(logger, "using", seed_urls, "seed url(s)")
 
-    visited_urls = crawl_list_of_urls(
+    crawl_cache = crawl_list_of_urls(
         seed_urls=seed_urls,
         object_name=script.object_name,
         max_iterations=script.nodes[node_name]["max_iterations"],
         use_cache=use_cache,
+        cache_prefix=node_name,
     )
 
-    script.nodes[node_name]["visited_urls"] = visited_urls
-    script.nodes[node_name]["output"] = "TBA"
+    script.nodes[node_name]["output"] = crawl_cache
 
     return True

blue_assistant/web/crawl.py

@@ -20,6 +20,7 @@ def crawl_list_of_urls(
     max_iterations: int = 10,
     use_cache: bool = False,
     verbose: bool = False,
+    cache_prefix: str = "",
 ) -> Dict[str, str]:
     logger.info(
         "{}.crawl_list_of_urls({}): {} -{}> {}".format(
@@ -56,8 +57,10 @@ def crawl_list_of_urls(
         queue = queue[1:]
 
         logger.info(
-            "{} {} ...".format(
+            "{} [#{:,}/{:,}]: {} ".format(
                 "✅ " if url in crawl_cache else "🔗 ",
+                iteration,
+                len(queue),
                 url,
             )
         )
@@ -74,23 +77,26 @@ def crawl_list_of_urls(
             file.save_yaml(
                 filename=objects.path_of(
                     object_name=object_name,
-                    filename="crawl_summary_cache/{}.yaml".format(url_to_filename(url)),
+                    filename="{}-crawl_cache/{}.yaml".format(
+                        cache_prefix,
+                        url_to_filename(url),
+                    ),
                 ),
                 data=url_summary,
             )
 
         crawl_cache[url] = content_type
-        if "list_of_urls" in url_summary:
-            queue = list(
-                set(
-                    queue
-                    + [
-                        url
-                        for url in url_summary["list_of_urls"]
-                        if url not in crawl_cache.keys()
-                    ]
-                )
-            )
+
+        queue = (
+            queue
+            + url_summary.get("list_of_urls", [])
+            + [
+                url
+                for url in url_summary.get("list_of_ignored_urls", [])
+                if any(url.startswith(url_prefix) for url_prefix in seed_urls)
+            ]
+        )
+        queue = list({url for url in queue if url not in crawl_cache.keys()})
 
         iteration += 1
         if max_iterations != -1 and iteration >= max_iterations:

blue_assistant/web/fetch.py

@@ -49,6 +49,9 @@ def fetch_links_and_text(
         for a_tag in soup.find_all("a", href=True):
             a_url = urljoin(url, a_tag["href"])
 
+            if "#" in a_url:
+                a_url = a_url.split("#", 1)[0]
+
             if a_url.startswith(url):
                 if url not in list_of_urls:
                     logger.info(f"+= {a_url}")

{blue_assistant-4.283.1.dist-info → blue_assistant-4.288.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: blue_assistant
-Version: 4.283.1
+Version: 4.288.1
 Summary: 🧠 An AI Assistant.
 Home-page: https://github.com/kamangir/blue-assistant
 Author: Arash Abadpour (Kamangir)
@@ -124,4 +124,4 @@ Also home to [`@web`](https://raw.githubusercontent.com/kamangir/blue-assistant/
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.283.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.288.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.283.1.dist-info → blue_assistant-4.288.1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 blue_assistant/README.py,sha256=EJORj3I5pucJplI86lrFaZBN5C9IYNgKoG_V7h27NHw,2586
-blue_assistant/__init__.py,sha256=sPtxnf1X-ZphpOLGOeSuDHB2HL_PYWxMF_E6-7ERMEA,311
+blue_assistant/__init__.py,sha256=-afJ8TPucaY6ohmlQR93hpbM12zWRKiLRL0i_9jeUH4,311
 blue_assistant/__main__.py,sha256=URtal70XZc0--3FDTYWcLtnGOqBYjMX9gt-L1k8hDXI,361
 blue_assistant/config.env,sha256=npodyuuhkZUHUv9FnEiQQZkKxFbg8nQb1YpOCURqV3Y,301
 blue_assistant/env.py,sha256=FTSdJ8-J4jAyI0-h3MBgOweQBWd3YEFIibBHSXpClrY,760
@@ -43,7 +43,7 @@ blue_assistant/script/actions/__init__.py,sha256=GJJCaXSrfhwW9K5A2PHmrS7iGl-Ss9U
 blue_assistant/script/actions/generate_image.py,sha256=brg3u6e-cZvBLK8B7UAh-CXgmAwyvSN-jE00EXMSF3A,1357
 blue_assistant/script/actions/generate_text.py,sha256=GQ7sF1J-vBNgr-h01RZHMDQJOGDXx8PzQWEVNQwW_TU,1910
 blue_assistant/script/actions/generic.py,sha256=EjJkDj82ZFFQbjn-Uib-2Qs-nZG8kR2NzhsEVuOpgWQ,376
-blue_assistant/script/actions/web_crawl.py,sha256=LoZXEKOlSEVVVJaSobMqnOjbpumtScaCta3I8TQbV-A,1532
+blue_assistant/script/actions/web_crawl.py,sha256=UlieFTnXJSgnKm0ZWwPdGsdmYuFOYMcuGJ1BzsnKXAw,1510
 blue_assistant/script/repository/__init__.py,sha256=zVI3cubRqM9H6WgF0EUP9idILVLCumPFmJgKPM7iVlM,604
 blue_assistant/script/repository/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 blue_assistant/script/repository/base/classes.py,sha256=gM8OB1iKRKLa_dk7esAogAR9WaNZFgLhXkpd1iTtsuQ,5376
@@ -63,11 +63,11 @@ blue_assistant/script/repository/orbital_data_explorer/__init__.py,sha256=yy5FtC
 blue_assistant/script/repository/orbital_data_explorer/classes.py,sha256=ixYd_FHWYtp8Sfd6AiZkIqePjoUlT9iLg7TvuxHIDzA,204
 blue_assistant/web/__init__.py,sha256=70_JSpnfX1mLm8Xv3xHIujfr2FfGeHPRs6HraWDP1XA,114
 blue_assistant/web/__main__.py,sha256=4s2LNikNiT4UTbzOVQzV4j2jUWfmVIbE36WS1BxTWJY,1576
-blue_assistant/web/crawl.py,sha256=yeo_HJhX8Pp5E1BC7ZGBzNs_c6pvMYSC3olvr5K27hU,3118
-blue_assistant/web/fetch.py,sha256=0hbT246VzpYVCfJ8eflIZWGFMJoxml9vj-sYRCedCH4,2469
+blue_assistant/web/crawl.py,sha256=5RkAyUUU6QVRatJhar0TuvG9u8s3qS-wGu7Dp6MeFX0,3326
+blue_assistant/web/fetch.py,sha256=meso5ssN6OEk2xcPo3VMmFsXLqPlBVZ2FxureWoIYag,2546
 blue_assistant/web/functions.py,sha256=uJAC_kGOn2wA9AwOB_FB2f1dFYyNaEPPC42lN3klnFc,618
-blue_assistant-4.283.1.dist-info/LICENSE,sha256=ogEPNDSH0_dhiv_lT3ifVIdgIzHAqNA_SemnxUfPBJk,7048
-blue_assistant-4.283.1.dist-info/METADATA,sha256=SF7LXKUsui8jj3MOhUDTRA54gt2AT5HQ4y_dIsRuA0Q,5380
-blue_assistant-4.283.1.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
-blue_assistant-4.283.1.dist-info/top_level.txt,sha256=ud0BkBbdOVze13bNqHuhZj1rwCztaBtDf5ChEYzASOs,15
-blue_assistant-4.283.1.dist-info/RECORD,,
+blue_assistant-4.288.1.dist-info/LICENSE,sha256=ogEPNDSH0_dhiv_lT3ifVIdgIzHAqNA_SemnxUfPBJk,7048
+blue_assistant-4.288.1.dist-info/METADATA,sha256=yK58Vl9ASNwaBc1Bac4MGYyRtwc6R07SxMlM09DVJJQ,5380
+blue_assistant-4.288.1.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+blue_assistant-4.288.1.dist-info/top_level.txt,sha256=ud0BkBbdOVze13bNqHuhZj1rwCztaBtDf5ChEYzASOs,15
+blue_assistant-4.288.1.dist-info/RECORD,,