blue-assistant 4.273.1__tar.gz → 4.288.1__tar.gz

{blue_assistant-4.273.1/blue_assistant.egg-info → blue_assistant-4.288.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: blue_assistant
-Version: 4.273.1
+Version: 4.288.1
 Summary: 🧠 An AI Assistant.
 Home-page: https://github.com/kamangir/blue-assistant
 Author: Arash Abadpour (Kamangir)
@@ -124,4 +124,4 @@ Also home to [`@web`](https://raw.githubusercontent.com/kamangir/blue-assistant/
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.236.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.273.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.288.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.273.1 → blue_assistant-4.288.1}/README.md

@@ -79,4 +79,4 @@ Also home to [`@web`](./blue_assistant/web/)
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.236.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.273.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.288.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.273.1 → blue_assistant-4.288.1}/blue_assistant/__init__.py

@@ -4,7 +4,7 @@ ICON = "🧠"
 
 DESCRIPTION = f"{ICON} An AI Assistant."
 
-VERSION = "4.273.1"
+VERSION = "4.288.1"
 
 REPO_NAME = "blue-assistant"
 

{blue_assistant-4.273.1 → blue_assistant-4.288.1}/blue_assistant/script/actions/web_crawl.py

@@ -34,16 +34,16 @@ def web_crawl(
         logger.error(f"{node_name}: {seed_url_var_name}: seed_urls not found in vars.")
         return False
     seed_urls = script.vars[seed_url_var_name]
-    log_list(logger, seed_urls, "seed url(s)")
+    log_list(logger, "using", seed_urls, "seed url(s)")
 
-    visited_urls = crawl_list_of_urls(
+    crawl_cache = crawl_list_of_urls(
         seed_urls=seed_urls,
         object_name=script.object_name,
         max_iterations=script.nodes[node_name]["max_iterations"],
         use_cache=use_cache,
+        cache_prefix=node_name,
     )
 
-    script.nodes[node_name]["visited_urls"] = visited_urls
-    script.nodes[node_name]["output"] = "TBA"
+    script.nodes[node_name]["output"] = crawl_cache
 
     return True

{blue_assistant-4.273.1 → blue_assistant-4.288.1}/blue_assistant/web/__main__.py

@@ -50,6 +50,7 @@ args = parser.parse_args()
 
 success = False
 if args.task == "crawl":
+    success = True
     dict_of_urls = crawl_list_of_urls(
         seed_urls=args.seed_urls.split("+"),
         object_name=args.object_name,
@@ -58,13 +59,7 @@ if args.task == "crawl":
     )
 
     if args.verbose == 1:
-        log_dict(logger, dict_of_urls, "url(s)")
-
-    success = post_to_object(
-        args.object_name,
-        NAME.replace(".", "-"),
-        dict_of_urls,
-    )
+        log_dict(logger, "crawled", dict_of_urls, "url(s)")
 elif args.task == "fetch":
     summary = fetch_links_and_text(
         url=args.url,

{blue_assistant-4.273.1 → blue_assistant-4.288.1}/blue_assistant/web/crawl.py

@@ -1,6 +1,7 @@
-from typing import List, Dict, Set
+from typing import List, Dict
 
 from blueness import module
+from blue_options.logger import log_dict, log_list
 from blue_objects import file
 from blue_objects import objects
 from blue_objects.metadata import get_from_object, post_to_object
@@ -19,6 +20,7 @@ def crawl_list_of_urls(
     max_iterations: int = 10,
     use_cache: bool = False,
     verbose: bool = False,
+    cache_prefix: str = "",
 ) -> Dict[str, str]:
     logger.info(
         "{}.crawl_list_of_urls({}): {} -{}> {}".format(
@@ -30,24 +32,35 @@ def crawl_list_of_urls(
         )
     )
 
-    crawl_cache: Dict[str, str] = (
-        get_from_object(
+    crawl_cache: Dict[str, str] = {}
+    queue: List[str] = [url for url in seed_urls]
+
+    if use_cache:
+        crawl_cache = get_from_object(
             object_name,
             "crawl_cache",
             {},
         )
-        if use_cache
-        else {}
-    )
+        log_dict(logger, "loaded cache:", crawl_cache, "url(s)")
 
-    queue: Set[str] = set(seed_urls)
+        queue += get_from_object(
+            object_name,
+            "crawl_queue",
+            [],
+        )
+
+    log_list(logger, "queue:", queue, "url(s)")
 
     iteration: int = 0
     while queue:
-        url = queue.pop()
+        url = queue[0]
+        queue = queue[1:]
+
         logger.info(
-            "{} {} ...".format(
-                "✅" if url in crawl_cache else "🔗",
+            "{} [#{:,}/{:,}]: {} ".format(
+                "✅ " if url in crawl_cache else "🔗 ",
+                iteration,
+                len(queue),
                 url,
             )
         )
@@ -64,14 +77,26 @@ def crawl_list_of_urls(
             file.save_yaml(
                 filename=objects.path_of(
                     object_name=object_name,
-                    filename="crawl_summary_cache/{}.yaml".format(url_to_filename(url)),
+                    filename="{}-crawl_cache/{}.yaml".format(
+                        cache_prefix,
+                        url_to_filename(url),
+                    ),
                 ),
                 data=url_summary,
             )
 
         crawl_cache[url] = content_type
-        if "links" in url_summary:
-            queue.update(url_summary["links"] - crawl_cache.keys())
+
+        queue = (
+            queue
+            + url_summary.get("list_of_urls", [])
+            + [
+                url
+                for url in url_summary.get("list_of_ignored_urls", [])
+                if any(url.startswith(url_prefix) for url_prefix in seed_urls)
+            ]
+        )
+        queue = list({url for url in queue if url not in crawl_cache.keys()})
 
         iteration += 1
         if max_iterations != -1 and iteration >= max_iterations:
@@ -88,4 +113,13 @@ def crawl_list_of_urls(
             crawl_cache,
         )
 
+        post_to_object(
+            object_name,
+            "crawl_queue",
+            queue,
+        )
+
+    log_dict(logger, "crawled", crawl_cache, "url(s)")
+    log_list(logger, "queue:", queue, "url(s)")
+
     return crawl_cache

{blue_assistant-4.273.1 → blue_assistant-4.288.1}/blue_assistant/web/fetch.py

@@ -1,4 +1,4 @@
-from typing import Dict, Any
+from typing import Dict, Any, List
 import requests
 from bs4 import BeautifulSoup, XMLParsedAsHTMLWarning
 from urllib.parse import urljoin
@@ -33,7 +33,8 @@ def fetch_links_and_text(
     content_type = response.headers.get("Content-Type", "")
     logger.info(f"content-type: {content_type}")
 
-    links = set()
+    list_of_urls: List[str] = []
+    list_of_ignored_urls: List[str] = []
     text = ""
 
     if not any(
@@ -48,12 +49,19 @@ def fetch_links_and_text(
         for a_tag in soup.find_all("a", href=True):
             a_url = urljoin(url, a_tag["href"])
 
+            if "#" in a_url:
+                a_url = a_url.split("#", 1)[0]
+
             if a_url.startswith(url):
-                logger.info(f"+= {a_url}")
-                links.add(a_url)
+                if url not in list_of_urls:
+                    logger.info(f"+= {a_url}")
+                    list_of_urls += [a_url]
                 continue
 
-            logger.info(f"ignored: {a_url}")
+            if a_url not in list_of_ignored_urls:
+                list_of_ignored_urls += [a_url]
+                if verbose:
+                    logger.info(f"ignored: {a_url}")
 
         text = soup.get_text(separator=" ", strip=True)
 
@@ -64,12 +72,22 @@ def fetch_links_and_text(
     text = re.sub(r"\s+", " ", text).strip()
 
     if verbose:
-        log_list(logger, list(links), "link(s)")
+        log_list(logger, "fetched", list_of_urls, "url(s)")
+        log_list(logger, "ignored", list_of_ignored_urls, "url(s)")
         log_long_text(logger, text)
+    else:
+        logger.info(
+            "{} url(s) collected, {} url(s) ignored, text: {:,} char(s).".format(
+                len(list_of_urls),
+                len(list_of_ignored_urls),
+                len(text),
+            )
+        )
 
     return {
         "url": url,
         "content_type": content_type,
-        "links": links,
+        "list_of_ignored_urls": list_of_ignored_urls,
+        "list_of_urls": list_of_urls,
         "text": text,
     }

{blue_assistant-4.273.1 → blue_assistant-4.288.1}/blue_assistant/web/functions.py

@@ -1,19 +1,4 @@
-from typing import List, Dict, Set, Tuple
-import requests
-from bs4 import BeautifulSoup, XMLParsedAsHTMLWarning
-from urllib.parse import urljoin
 import re
-import warnings
-
-from blueness import module
-from blue_options.logger import log_long_text, log_list
-
-from blue_assistant import NAME
-from blue_assistant.logger import logger
-
-warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
-
-NAME = module.name(__file__, NAME)
 
 
 def url_to_filename(

{blue_assistant-4.273.1 → blue_assistant-4.288.1/blue_assistant.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: blue_assistant
-Version: 4.273.1
+Version: 4.288.1
 Summary: 🧠 An AI Assistant.
 Home-page: https://github.com/kamangir/blue-assistant
 Author: Arash Abadpour (Kamangir)
@@ -124,4 +124,4 @@ Also home to [`@web`](https://raw.githubusercontent.com/kamangir/blue-assistant/
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.236.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.273.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.288.1`](https://github.com/kamangir/blue-assistant).