blue-assistant 4.283.1__tar.gz → 4.288.1__tar.gz

{blue_assistant-4.283.1/blue_assistant.egg-info → blue_assistant-4.288.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: blue_assistant
-Version: 4.283.1
+Version: 4.288.1
 Summary: 🧠 An AI Assistant.
 Home-page: https://github.com/kamangir/blue-assistant
 Author: Arash Abadpour (Kamangir)
@@ -124,4 +124,4 @@ Also home to [`@web`](https://raw.githubusercontent.com/kamangir/blue-assistant/
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.283.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.288.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.283.1 → blue_assistant-4.288.1}/README.md

@@ -79,4 +79,4 @@ Also home to [`@web`](./blue_assistant/web/)
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.283.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.288.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.283.1 → blue_assistant-4.288.1}/blue_assistant/__init__.py

@@ -4,7 +4,7 @@ ICON = "🧠"
 
 DESCRIPTION = f"{ICON} An AI Assistant."
 
-VERSION = "4.283.1"
+VERSION = "4.288.1"
 
 REPO_NAME = "blue-assistant"
 

{blue_assistant-4.283.1 → blue_assistant-4.288.1}/blue_assistant/script/actions/web_crawl.py

@@ -36,14 +36,14 @@ def web_crawl(
     seed_urls = script.vars[seed_url_var_name]
     log_list(logger, "using", seed_urls, "seed url(s)")
 
-    visited_urls = crawl_list_of_urls(
+    crawl_cache = crawl_list_of_urls(
         seed_urls=seed_urls,
         object_name=script.object_name,
         max_iterations=script.nodes[node_name]["max_iterations"],
         use_cache=use_cache,
+        cache_prefix=node_name,
     )
 
-    script.nodes[node_name]["visited_urls"] = visited_urls
-    script.nodes[node_name]["output"] = "TBA"
+    script.nodes[node_name]["output"] = crawl_cache
 
     return True

{blue_assistant-4.283.1 → blue_assistant-4.288.1}/blue_assistant/web/crawl.py

@@ -20,6 +20,7 @@ def crawl_list_of_urls(
     max_iterations: int = 10,
     use_cache: bool = False,
     verbose: bool = False,
+    cache_prefix: str = "",
 ) -> Dict[str, str]:
     logger.info(
         "{}.crawl_list_of_urls({}): {} -{}> {}".format(
@@ -56,8 +57,10 @@ def crawl_list_of_urls(
         queue = queue[1:]
 
         logger.info(
-            "{} {} ...".format(
+            "{} [#{:,}/{:,}]: {} ".format(
                 "✅ " if url in crawl_cache else "🔗 ",
+                iteration,
+                len(queue),
                 url,
             )
         )
@@ -74,23 +77,26 @@ def crawl_list_of_urls(
             file.save_yaml(
                 filename=objects.path_of(
                     object_name=object_name,
-                    filename="crawl_summary_cache/{}.yaml".format(url_to_filename(url)),
+                    filename="{}-crawl_cache/{}.yaml".format(
+                        cache_prefix,
+                        url_to_filename(url),
+                    ),
                 ),
                 data=url_summary,
             )
 
         crawl_cache[url] = content_type
-        if "list_of_urls" in url_summary:
-            queue = list(
-                set(
-                    queue
-                    + [
-                        url
-                        for url in url_summary["list_of_urls"]
-                        if url not in crawl_cache.keys()
-                    ]
-                )
-            )
+
+        queue = (
+            queue
+            + url_summary.get("list_of_urls", [])
+            + [
+                url
+                for url in url_summary.get("list_of_ignored_urls", [])
+                if any(url.startswith(url_prefix) for url_prefix in seed_urls)
+            ]
+        )
+        queue = list({url for url in queue if url not in crawl_cache.keys()})
 
         iteration += 1
         if max_iterations != -1 and iteration >= max_iterations:

{blue_assistant-4.283.1 → blue_assistant-4.288.1}/blue_assistant/web/fetch.py

@@ -49,6 +49,9 @@ def fetch_links_and_text(
         for a_tag in soup.find_all("a", href=True):
             a_url = urljoin(url, a_tag["href"])
 
+            if "#" in a_url:
+                a_url = a_url.split("#", 1)[0]
+
             if a_url.startswith(url):
                 if url not in list_of_urls:
                     logger.info(f"+= {a_url}")

{blue_assistant-4.283.1 → blue_assistant-4.288.1/blue_assistant.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: blue_assistant
-Version: 4.283.1
+Version: 4.288.1
 Summary: 🧠 An AI Assistant.
 Home-page: https://github.com/kamangir/blue-assistant
 Author: Arash Abadpour (Kamangir)
@@ -124,4 +124,4 @@ Also home to [`@web`](https://raw.githubusercontent.com/kamangir/blue-assistant/
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.283.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.288.1`](https://github.com/kamangir/blue-assistant).