blue-assistant 4.268.1__tar.gz → 4.283.1__tar.gz

{blue_assistant-4.268.1/blue_assistant.egg-info → blue_assistant-4.283.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: blue_assistant
-Version: 4.268.1
+Version: 4.283.1
 Summary: 🧠 An AI Assistant.
 Home-page: https://github.com/kamangir/blue-assistant
 Author: Arash Abadpour (Kamangir)
@@ -124,4 +124,4 @@ Also home to [`@web`](https://raw.githubusercontent.com/kamangir/blue-assistant/
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.236.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.268.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.283.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.268.1 → blue_assistant-4.283.1}/README.md

@@ -79,4 +79,4 @@ Also home to [`@web`](./blue_assistant/web/)
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.236.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.268.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.283.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.268.1 → blue_assistant-4.283.1}/blue_assistant/.abcli/script/run.sh

@@ -3,7 +3,8 @@
 function blue_assistant_script_run() {
     local options=$1
     local do_dryrun=$(abcli_option_int "$options" dryrun 0)
-    local do_download=$(abcli_option_int "$options" download 0)
+    local use_cache=$(abcli_option_int "$options" cache 1)
+    local do_download=$(abcli_option_int "$options" download $use_cache)
     local do_upload=$(abcli_option_int "$options" upload $(abcli_not $do_dryrun))
 
     local script_options=$2
@@ -20,6 +21,7 @@ function blue_assistant_script_run() {
         run \
         --script_name $script_name \
         --object_name $object_name \
+        --use_cache $use_cache \
         "${@:4}"
     [[ $? -ne 0 ]] && return 1
 

{blue_assistant-4.268.1 → blue_assistant-4.283.1}/blue_assistant/.abcli/tests/script_run.sh

@@ -11,11 +11,13 @@ function test_blue_assistant_script_run() {
     for script_name in $(echo "$list_of_script_name" | tr + " "); do
         abcli_log "testing $script_name ..."
 
+        local object_name=test_blue_assistant_script_run-$(abcli_string_timestamp_short)
+
         abcli_eval ,$options \
             blue_assistant_script_run \
             ~upload,$options \
             script=$script_name \
-            test_blue_assistant_script_run-$(abcli_string_timestamp_short) \
+            $object_name \
             "${@:2}" \
             --test_mode 1 \
             --verbose 1

{blue_assistant-4.268.1 → blue_assistant-4.283.1}/blue_assistant/.abcli/web/crawl.sh

@@ -3,11 +3,15 @@
 function blue_assistant_web_crawl() {
     local options=$1
     local do_dryrun=$(abcli_option_int "$options" dryrun 0)
+    local use_cache=$(abcli_option_int "$options" cache 0)
+    local do_download=$(abcli_option_int "$options" download $use_cache)
     local do_upload=$(abcli_option_int "$options" upload $(abcli_not $do_dryrun))
 
     local seed_urls=${2:-void}
 
     local object_name=$(abcli_clarify_object $3 web-crawl-$(abcli_string_timestamp_short))
+    [[ "$do_download" == 1 ]] &&
+        abcli_download - $object_name
 
     abcli_log "crawling $seed_urls -> $object_name ..."
 
@@ -16,6 +20,7 @@ function blue_assistant_web_crawl() {
         crawl \
         --seed_urls $seed_urls \
         --object_name $object_name \
+        --use_cache $use_cache \
         "${@:4}"
     [[ $? -ne 0 ]] && return 1
 

{blue_assistant-4.268.1 → blue_assistant-4.283.1}/blue_assistant/__init__.py

@@ -4,7 +4,7 @@ ICON = "🧠"
 
 DESCRIPTION = f"{ICON} An AI Assistant."
 
-VERSION = "4.268.1"
+VERSION = "4.283.1"
 
 REPO_NAME = "blue-assistant"
 

{blue_assistant-4.268.1 → blue_assistant-4.283.1}/blue_assistant/help/script.py

@@ -30,7 +30,7 @@ def help_run(
     tokens: List[str],
     mono: bool,
 ) -> str:
-    options = xtra("download,dryrun,~upload", mono=mono)
+    options = xtra("~cache,download,dryrun,~upload", mono=mono)
 
     script_options = "script=<script>"
 

{blue_assistant-4.268.1 → blue_assistant-4.283.1}/blue_assistant/help/web.py

@@ -7,7 +7,7 @@ def help_crawl(
     tokens: List[str],
     mono: bool,
 ) -> str:
-    options = xtra("dryrun,~upload", mono=mono)
+    options = xtra("cache,~download,dryrun,~upload", mono=mono)
 
     args = [
         "[--max_iterations <100000>]",

{blue_assistant-4.268.1 → blue_assistant-4.283.1}/blue_assistant/script/__main__.py

@@ -48,6 +48,12 @@ parser.add_argument(
     default=1,
     help="0 | 1",
 )
+parser.add_argument(
+    "--use_cache",
+    type=int,
+    default=1,
+    help="0 | 1",
+)
 args = parser.parse_args()
 
 delim = " " if args.delim == "space" else args.delim
@@ -70,7 +76,9 @@ elif args.task == "run":
     )
 
     if success:
-        success = script.run()
+        success = script.run(
+            use_cache=args.use_cache == 1,
+        )
 else:
     success = None
 

{blue_assistant-4.268.1 → blue_assistant-4.283.1}/blue_assistant/script/actions/generate_image.py

@@ -17,6 +17,7 @@ NAME = module.name(__file__, NAME)
 def generate_image(
     script: BaseScript,
     node_name: str,
+    use_cache: bool,
 ) -> bool:
     logger.info(f"{NAME}: {script} @ {node_name} ...")
 

{blue_assistant-4.268.1 → blue_assistant-4.283.1}/blue_assistant/script/actions/generate_text.py

@@ -18,6 +18,7 @@ NAME = module.name(__file__, NAME)
 def generate_text(
     script: BaseScript,
     node_name: str,
+    use_cache: bool,
 ) -> bool:
     logger.info(f"{NAME}: {script} @ {node_name} ...")
 

{blue_assistant-4.268.1 → blue_assistant-4.283.1}/blue_assistant/script/actions/generic.py

@@ -10,6 +10,7 @@ NAME = module.name(__file__, NAME)
 def generic_action(
     script: BaseScript,
     node_name: str,
+    use_cache: bool,
 ) -> bool:
     logger.info(f"{NAME}: {script} @ {node_name} ...")
     return True

{blue_assistant-4.268.1 → blue_assistant-4.283.1}/blue_assistant/script/actions/web_crawl.py

@@ -1,12 +1,9 @@
-from typing import Dict
 from blueness import module
-from tqdm import tqdm
 
 from blue_options.logger import log_list
-from openai_commands.text_generation import api
 
 from blue_assistant import NAME
-from blue_assistant.web.functions import crawl_list_of_urls
+from blue_assistant.web.crawl import crawl_list_of_urls
 from blue_assistant.script.repository.base.classes import BaseScript
 from blue_assistant.logger import logger
 
@@ -17,6 +14,7 @@ NAME = module.name(__file__, NAME)
 def web_crawl(
     script: BaseScript,
     node_name: str,
+    use_cache: bool,
 ) -> bool:
     logger.info(f"{NAME}: {script} @ {node_name} ...")
 
@@ -24,37 +22,28 @@ def web_crawl(
     if not isinstance(seed_url_var_name, str):
         logger.error(f"{node_name}: seed_urls must be a string.")
         return False
-    # to allow both :::<var-name> and <var-name> - for convenience :)
-    if seed_url_var_name.startswith(":::"):
-        seed_url_var_name = seed_url_var_name[3:].strip()
     if not seed_url_var_name:
         logger.error(f"{node_name}: seed_urls not found.")
         return False
+
+    # to allow both :::<var-name> and <var-name> - for convenience :)
+    if seed_url_var_name.startswith(":::"):
+        seed_url_var_name = seed_url_var_name[3:].strip()
+
     if seed_url_var_name not in script.vars:
         logger.error(f"{node_name}: {seed_url_var_name}: seed_urls not found in vars.")
         return False
-
     seed_urls = script.vars[seed_url_var_name]
-    log_list(logger, seed_urls, "seed url(s)")
+    log_list(logger, "using", seed_urls, "seed url(s)")
 
     visited_urls = crawl_list_of_urls(
         seed_urls=seed_urls,
         object_name=script.object_name,
         max_iterations=script.nodes[node_name]["max_iterations"],
+        use_cache=use_cache,
     )
 
-    success, output, _ = api.generate_text(
-        prompt=script.nodes[node_name]["prompt"].replace(
-            ":::input", " ".join([content for content in visited_urls.values()])
-        ),
-        verbose=script.verbose,
-    )
-    if not success:
-        return success
-
-    logger.info(output)
-
     script.nodes[node_name]["visited_urls"] = visited_urls
-    script.nodes[node_name]["output"] = output
+    script.nodes[node_name]["output"] = "TBA"
 
     return True

{blue_assistant-4.268.1 → blue_assistant-4.283.1}/blue_assistant/script/repository/base/classes.py

@@ -145,12 +145,16 @@ class BaseScript:
             [node_name],
         )
 
-    def run(self) -> bool:
+    def run(
+        self,
+        use_cache: bool = True,
+    ) -> bool:
         logger.info(
-            "{}.run: {}:{} -> {}".format(
+            "{}.run: {}:{} -{}> {}".format(
                 NAME,
                 self.__class__.__name__,
                 self.name,
+                "use-cache-" if use_cache else "",
                 self.object_name,
             )
         )

{blue_assistant-4.268.1 → blue_assistant-4.283.1}/blue_assistant/script/repository/blue_amo/actions/slicing_into_frames.py

@@ -10,6 +10,7 @@ NAME = module.name(__file__, NAME)
 def slicing_into_frames(
     script: BaseScript,
     node_name: str,
+    use_cache: bool,
 ) -> bool:
     logger.info(f"{NAME}: processing the output...")
 

{blue_assistant-4.268.1 → blue_assistant-4.283.1}/blue_assistant/script/repository/blue_amo/actions/stitching_the_frames.py

@@ -18,6 +18,7 @@ NAME = module.name(__file__, NAME)
 def stitching_the_frames(
     script: BaseScript,
     node_name: str,
+    use_cache: bool,
 ) -> bool:
     list_of_frames_filenames: List[str] = [
         filename

{blue_assistant-4.268.1 → blue_assistant-4.283.1}/blue_assistant/script/repository/blue_amo/classes.py

@@ -66,14 +66,19 @@ class BlueAmoScript(GenericScript):
     def perform_action(
         self,
         node_name: str,
+        use_cache: bool,
     ) -> bool:
-        if not super().perform_action(node_name=node_name):
+        if not super().perform_action(
+            node_name=node_name,
+            use_cache=use_cache,
+        ):
             return False
 
         if node_name in dict_of_actions:
             return dict_of_actions[node_name](
                 script=self,
                 node_name=node_name,
+                use_cache=use_cache,
             )
 
         return True

{blue_assistant-4.268.1 → blue_assistant-4.283.1}/blue_assistant/script/repository/generic/classes.py

@@ -22,6 +22,7 @@ class GenericScript(BaseScript):
     def perform_action(
         self,
         node_name: str,
+        use_cache: bool,
     ) -> bool:
         action_name = self.nodes[node_name].get("action", "unknown")
         logger.info(f"---- node: {node_name} ---- ")
@@ -33,10 +34,14 @@ class GenericScript(BaseScript):
         return dict_of_actions[action_name](
             script=self,
             node_name=node_name,
+            use_cache=use_cache,
         )
 
-    def run(self) -> bool:
-        if not super().run():
+    def run(
+        self,
+        use_cache: bool = True,
+    ) -> bool:
+        if not super().run(use_cache=use_cache):
             return False
 
         success: bool = True
@@ -68,7 +73,10 @@ class GenericScript(BaseScript):
                     )
                     continue
 
-                if not self.perform_action(node_name=node_name):
+                if not self.perform_action(
+                    node_name=node_name,
+                    use_cache=use_cache,
+                ):
                     success = False
                     break
 

blue_assistant-4.283.1/blue_assistant/web/__init__.py

@@ -0,0 +1,2 @@
+from blue_assistant.web.crawl import crawl_list_of_urls
+from blue_assistant.web.fetch import fetch_links_and_text

{blue_assistant-4.268.1 → blue_assistant-4.283.1}/blue_assistant/web/__main__.py

@@ -6,7 +6,7 @@ from blue_options.logger import log_dict
 from blue_objects.metadata import post_to_object
 
 from blue_assistant import NAME
-from blue_assistant.web.functions import crawl_list_of_urls, fetch_links_and_text
+from blue_assistant.web import crawl_list_of_urls, fetch_links_and_text
 from blue_assistant.logger import logger
 
 NAME = module.name(__file__, NAME)
@@ -40,26 +40,28 @@ parser.add_argument(
     "--object_name",
     type=str,
 )
+parser.add_argument(
+    "--use_cache",
+    type=int,
+    default=0,
+    help="0 | 1",
+)
 args = parser.parse_args()
 
 success = False
 if args.task == "crawl":
+    success = True
     dict_of_urls = crawl_list_of_urls(
         seed_urls=args.seed_urls.split("+"),
         object_name=args.object_name,
         max_iterations=args.max_iterations,
+        use_cache=args.use_cache == 1,
     )
 
     if args.verbose == 1:
-        log_dict(logger, dict_of_urls, "url(s)")
-
-    success = post_to_object(
-        args.object_name,
-        NAME.replace(".", "-"),
-        dict_of_urls,
-    )
+        log_dict(logger, "crawled", dict_of_urls, "url(s)")
 elif args.task == "fetch":
-    links, text = fetch_links_and_text(
+    summary = fetch_links_and_text(
         url=args.url,
         verbose=True,
     )
@@ -67,10 +69,7 @@ elif args.task == "fetch":
     success = post_to_object(
         args.object_name,
         NAME.replace(".", "-"),
-        {
-            "links": list(links),
-            "text": text,
-        },
+        summary,
     )
 else:
     success = None

blue_assistant-4.283.1/blue_assistant/web/crawl.py

@@ -0,0 +1,119 @@
+from typing import List, Dict
+
+from blueness import module
+from blue_options.logger import log_dict, log_list
+from blue_objects import file
+from blue_objects import objects
+from blue_objects.metadata import get_from_object, post_to_object
+
+from blue_assistant import NAME
+from blue_assistant.web.fetch import fetch_links_and_text
+from blue_assistant.web.functions import url_to_filename
+from blue_assistant.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+def crawl_list_of_urls(
+    seed_urls: List[str],
+    object_name: str,
+    max_iterations: int = 10,
+    use_cache: bool = False,
+    verbose: bool = False,
+) -> Dict[str, str]:
+    logger.info(
+        "{}.crawl_list_of_urls({}): {} -{}> {}".format(
+            NAME,
+            len(seed_urls),
+            ", ".join(seed_urls),
+            "use-cache-" if use_cache else "",
+            object_name,
+        )
+    )
+
+    crawl_cache: Dict[str, str] = {}
+    queue: List[str] = [url for url in seed_urls]
+
+    if use_cache:
+        crawl_cache = get_from_object(
+            object_name,
+            "crawl_cache",
+            {},
+        )
+        log_dict(logger, "loaded cache:", crawl_cache, "url(s)")
+
+        queue += get_from_object(
+            object_name,
+            "crawl_queue",
+            [],
+        )
+
+    log_list(logger, "queue:", queue, "url(s)")
+
+    iteration: int = 0
+    while queue:
+        url = queue[0]
+        queue = queue[1:]
+
+        logger.info(
+            "{} {} ...".format(
+                "✅ " if url in crawl_cache else "🔗 ",
+                url,
+            )
+        )
+        if url in crawl_cache:
+            continue
+
+        url_summary = fetch_links_and_text(
+            url=url,
+            verbose=verbose,
+        )
+        content_type = url_summary.get("content_type", "unknown")
+
+        if use_cache and "html" in content_type:
+            file.save_yaml(
+                filename=objects.path_of(
+                    object_name=object_name,
+                    filename="crawl_summary_cache/{}.yaml".format(url_to_filename(url)),
+                ),
+                data=url_summary,
+            )
+
+        crawl_cache[url] = content_type
+        if "list_of_urls" in url_summary:
+            queue = list(
+                set(
+                    queue
+                    + [
+                        url
+                        for url in url_summary["list_of_urls"]
+                        if url not in crawl_cache.keys()
+                    ]
+                )
+            )
+
+        iteration += 1
+        if max_iterations != -1 and iteration >= max_iterations:
+            logger.warning(f"max iteration of {max_iterations} reached.")
+            break
+
+    if queue:
+        logger.warning(f"queue: {len(queue)}")
+
+    if use_cache:
+        post_to_object(
+            object_name,
+            "crawl_cache",
+            crawl_cache,
+        )
+
+        post_to_object(
+            object_name,
+            "crawl_queue",
+            queue,
+        )
+
+    log_dict(logger, "crawled", crawl_cache, "url(s)")
+    log_list(logger, "queue:", queue, "url(s)")
+
+    return crawl_cache

blue_assistant-4.283.1/blue_assistant/web/fetch.py

@@ -0,0 +1,90 @@
+from typing import Dict, Any, List
+import requests
+from bs4 import BeautifulSoup, XMLParsedAsHTMLWarning
+from urllib.parse import urljoin
+import re
+import warnings
+
+from blueness import module
+from blue_options.logger import log_long_text, log_list
+
+from blue_assistant import NAME
+from blue_assistant.logger import logger
+
+warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
+
+NAME = module.name(__file__, NAME)
+
+
+def fetch_links_and_text(
+    url: str,
+    verbose: bool = False,
+) -> Dict[str, Any]:
+    try:
+        response = requests.get(url, timeout=5)
+    except Exception as e:
+        logger.warning(e)
+        return {}
+
+    if response.status_code != 200:
+        logger.error(response)
+        return {}
+
+    content_type = response.headers.get("Content-Type", "")
+    logger.info(f"content-type: {content_type}")
+
+    list_of_urls: List[str] = []
+    list_of_ignored_urls: List[str] = []
+    text = ""
+
+    if not any(
+        thing in content_type
+        for thing in [
+            "pdf",
+            "xml",
+        ]
+    ):
+        soup = BeautifulSoup(response.text, "html.parser")
+
+        for a_tag in soup.find_all("a", href=True):
+            a_url = urljoin(url, a_tag["href"])
+
+            if a_url.startswith(url):
+                if url not in list_of_urls:
+                    logger.info(f"+= {a_url}")
+                    list_of_urls += [a_url]
+                continue
+
+            if a_url not in list_of_ignored_urls:
+                list_of_ignored_urls += [a_url]
+                if verbose:
+                    logger.info(f"ignored: {a_url}")
+
+        text = soup.get_text(separator=" ", strip=True)
+
+    # remove non-ASCII characters
+    text = re.sub(r"[^\x20-\x7E]+", "", text)
+    for thing in ["\r", "\n", "\t"]:
+        text = text.replace(thing, " ")
+    text = re.sub(r"\s+", " ", text).strip()
+
+    if verbose:
+        log_list(logger, "fetched", list_of_urls, "url(s)")
+        log_list(logger, "ignored", list_of_ignored_urls, "url(s)")
+        log_long_text(logger, text)
+    else:
+        logger.info(
+            "{} url(s) collected, {} url(s) ignored, text: {:,} char(s).".format(
+                len(list_of_urls),
+                len(list_of_ignored_urls),
+                len(text),
+            )
+        )
+
+    return {
+        "url": url,
+        "content_type": content_type,
+        "list_of_ignored_urls": list_of_ignored_urls,
+        "list_of_urls": list_of_urls,
+        "text": text,
+    }

blue_assistant-4.283.1/blue_assistant/web/functions.py

@@ -0,0 +1,24 @@
+import re
+
+
+def url_to_filename(
+    url: str,
+    max_length: int = 255,
+) -> str:
+    # Remove the URL scheme (http://, https://)
+    filename = re.sub(r"^https?://", "", url)
+
+    # Replace unwanted characters with an underscore
+    filename = re.sub(r"[^\w\s-]", "_", filename)
+
+    # Replace slashes with a hyphen to preserve some structure
+    filename = re.sub(r"\/", "-", filename)
+
+    # Replace spaces with underscores
+    filename = filename.replace(" ", "_")
+
+    # Ensure the filename length is not too long
+    if len(filename) > max_length:
+        filename = filename[:max_length]
+
+    return filename

{blue_assistant-4.268.1 → blue_assistant-4.283.1/blue_assistant.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: blue_assistant
-Version: 4.268.1
+Version: 4.283.1
 Summary: 🧠 An AI Assistant.
 Home-page: https://github.com/kamangir/blue-assistant
 Author: Arash Abadpour (Kamangir)
@@ -124,4 +124,4 @@ Also home to [`@web`](https://raw.githubusercontent.com/kamangir/blue-assistant/
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.236.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.268.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.283.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.268.1 → blue_assistant-4.283.1}/blue_assistant.egg-info/SOURCES.txt

@@ -74,4 +74,6 @@ blue_assistant/script/repository/orbital_data_explorer/__init__.py
 blue_assistant/script/repository/orbital_data_explorer/classes.py
 blue_assistant/web/__init__.py
 blue_assistant/web/__main__.py
+blue_assistant/web/crawl.py
+blue_assistant/web/fetch.py
 blue_assistant/web/functions.py

blue_assistant-4.268.1/blue_assistant/web/functions.py

@@ -1,97 +0,0 @@
-from typing import List, Dict, Set, Tuple
-import requests
-from bs4 import BeautifulSoup, XMLParsedAsHTMLWarning
-from urllib.parse import urljoin
-import re
-import warnings
-
-from blueness import module
-from blue_options.logger import log_long_text, log_list
-
-from blue_assistant import NAME
-from blue_assistant.logger import logger
-
-warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
-
-NAME = module.name(__file__, NAME)
-
-
-def fetch_links_and_text(
-    url: str,
-    verbose: bool = False,
-) -> Tuple[List[str], str]:
-    try:
-        response = requests.get(url, timeout=5)
-        response.raise_for_status()
-    except requests.RequestException:
-        return set(), ""
-
-    soup = BeautifulSoup(response.text, "html.parser")
-    links = set()
-
-    for a_tag in soup.find_all("a", href=True):
-        a_url = urljoin(url, a_tag["href"])
-
-        if a_url.startswith(url):
-            logger.info(f"+= {a_url}")
-            links.add(a_url)
-            continue
-
-        logger.info(f"ignored: {a_url}")
-
-    plain_text = soup.get_text(separator=" ", strip=True)
-
-    # remove non-ASCII characters
-    plain_text = re.sub(r"[^\x20-\x7E]+", "", plain_text)
-    for thing in ["\r", "\n", "\t"]:
-        plain_text = plain_text.replace(thing, " ")
-    plain_text = re.sub(r"\s+", " ", plain_text).strip()
-
-    if verbose:
-        log_list(logger, list(links), "link(s)")
-        log_long_text(logger, plain_text)
-
-    return links, plain_text
-
-
-def crawl_list_of_urls(
-    seed_urls: List[str],
-    object_name: str,
-    max_iterations: int = 10,
-    verbose: bool = False,
-) -> Dict[str, str]:
-    logger.info(
-        "{}.crawl_list_of_urls({}): {} -> {}".format(
-            NAME,
-            len(seed_urls),
-            ", ".join(seed_urls),
-            object_name,
-        )
-    )
-
-    visited: Dict[str, str] = {}
-    queue: Set[str] = set(seed_urls)
-
-    iteration: int = 0
-    while queue:
-        url = queue.pop()
-        if url in visited:
-            continue
-
-        logger.info(f"🔗  {url} ...")
-        url_links, url_text = fetch_links_and_text(
-            url=url,
-            verbose=verbose,
-        )
-        visited[url] = url_text
-        queue.update(url_links - visited.keys())
-
-        iteration += 1
-        if max_iterations != -1 and iteration >= max_iterations:
-            logger.warning(f"max iteration of {max_iterations} reached.")
-            break
-
-    if queue:
-        logger.warning(f"queue: {len(queue)}")
-
-    return visited