blue-assistant 4.283.1__tar.gz → 4.294.1__tar.gz

{blue_assistant-4.283.1/blue_assistant.egg-info → blue_assistant-4.294.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: blue_assistant
-Version: 4.283.1
+Version: 4.294.1
 Summary: 🧠 An AI Assistant.
 Home-page: https://github.com/kamangir/blue-assistant
 Author: Arash Abadpour (Kamangir)
@@ -124,4 +124,4 @@ Also home to [`@web`](https://raw.githubusercontent.com/kamangir/blue-assistant/
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.283.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.294.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.283.1 → blue_assistant-4.294.1}/README.md

@@ -79,4 +79,4 @@ Also home to [`@web`](./blue_assistant/web/)
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.283.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.294.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.283.1 → blue_assistant-4.294.1}/blue_assistant/.abcli/script/run.sh

@@ -3,8 +3,7 @@
 function blue_assistant_script_run() {
     local options=$1
     local do_dryrun=$(abcli_option_int "$options" dryrun 0)
-    local use_cache=$(abcli_option_int "$options" cache 1)
-    local do_download=$(abcli_option_int "$options" download $use_cache)
+    local do_download=$(abcli_option_int "$options" download $(abcli_not $do_dryrun))
     local do_upload=$(abcli_option_int "$options" upload $(abcli_not $do_dryrun))
 
     local script_options=$2
@@ -21,7 +20,6 @@ function blue_assistant_script_run() {
         run \
         --script_name $script_name \
         --object_name $object_name \
-        --use_cache $use_cache \
         "${@:4}"
     [[ $? -ne 0 ]] && return 1
 

{blue_assistant-4.283.1 → blue_assistant-4.294.1}/blue_assistant/.abcli/web/crawl.sh

@@ -3,8 +3,7 @@
 function blue_assistant_web_crawl() {
     local options=$1
     local do_dryrun=$(abcli_option_int "$options" dryrun 0)
-    local use_cache=$(abcli_option_int "$options" cache 0)
-    local do_download=$(abcli_option_int "$options" download $use_cache)
+    local do_download=$(abcli_option_int "$options" download $(abcli_not $do_dryrun))
     local do_upload=$(abcli_option_int "$options" upload $(abcli_not $do_dryrun))
 
     local seed_urls=${2:-void}
@@ -20,7 +19,6 @@ function blue_assistant_web_crawl() {
         crawl \
         --seed_urls $seed_urls \
         --object_name $object_name \
-        --use_cache $use_cache \
         "${@:4}"
     [[ $? -ne 0 ]] && return 1
 

{blue_assistant-4.283.1 → blue_assistant-4.294.1}/blue_assistant/__init__.py

@@ -4,7 +4,7 @@ ICON = "🧠"
 
 DESCRIPTION = f"{ICON} An AI Assistant."
 
-VERSION = "4.283.1"
+VERSION = "4.294.1"
 
 REPO_NAME = "blue-assistant"
 

{blue_assistant-4.283.1 → blue_assistant-4.294.1}/blue_assistant/help/script.py

@@ -30,7 +30,7 @@ def help_run(
     tokens: List[str],
     mono: bool,
 ) -> str:
-    options = xtra("~cache,download,dryrun,~upload", mono=mono)
+    options = xtra("~download,dryrun,~upload", mono=mono)
 
     script_options = "script=<script>"
 

{blue_assistant-4.283.1 → blue_assistant-4.294.1}/blue_assistant/help/web.py

@@ -7,7 +7,7 @@ def help_crawl(
     tokens: List[str],
     mono: bool,
 ) -> str:
-    options = xtra("cache,~download,dryrun,~upload", mono=mono)
+    options = xtra("~download,dryrun,~upload", mono=mono)
 
     args = [
         "[--max_iterations <100000>]",

{blue_assistant-4.283.1 → blue_assistant-4.294.1}/blue_assistant/script/__main__.py

@@ -48,12 +48,6 @@ parser.add_argument(
     default=1,
     help="0 | 1",
 )
-parser.add_argument(
-    "--use_cache",
-    type=int,
-    default=1,
-    help="0 | 1",
-)
 args = parser.parse_args()
 
 delim = " " if args.delim == "space" else args.delim
@@ -76,9 +70,7 @@ elif args.task == "run":
     )
 
     if success:
-        success = script.run(
-            use_cache=args.use_cache == 1,
-        )
+        success = script.run()
 else:
     success = None
 

{blue_assistant-4.283.1 → blue_assistant-4.294.1}/blue_assistant/script/actions/generate_image.py

@@ -17,7 +17,6 @@ NAME = module.name(__file__, NAME)
 def generate_image(
     script: BaseScript,
     node_name: str,
-    use_cache: bool,
 ) -> bool:
     logger.info(f"{NAME}: {script} @ {node_name} ...")
 

{blue_assistant-4.283.1 → blue_assistant-4.294.1}/blue_assistant/script/actions/generate_text.py

@@ -18,7 +18,6 @@ NAME = module.name(__file__, NAME)
 def generate_text(
     script: BaseScript,
     node_name: str,
-    use_cache: bool,
 ) -> bool:
     logger.info(f"{NAME}: {script} @ {node_name} ...")
 

{blue_assistant-4.283.1 → blue_assistant-4.294.1}/blue_assistant/script/actions/generic.py

@@ -10,7 +10,6 @@ NAME = module.name(__file__, NAME)
 def generic_action(
     script: BaseScript,
     node_name: str,
-    use_cache: bool,
 ) -> bool:
     logger.info(f"{NAME}: {script} @ {node_name} ...")
     return True

{blue_assistant-4.283.1 → blue_assistant-4.294.1}/blue_assistant/script/actions/web_crawl.py

@@ -14,7 +14,6 @@ NAME = module.name(__file__, NAME)
 def web_crawl(
     script: BaseScript,
     node_name: str,
-    use_cache: bool,
 ) -> bool:
     logger.info(f"{NAME}: {script} @ {node_name} ...")
 
@@ -36,14 +35,13 @@ def web_crawl(
     seed_urls = script.vars[seed_url_var_name]
     log_list(logger, "using", seed_urls, "seed url(s)")
 
-    visited_urls = crawl_list_of_urls(
+    success, _ = crawl_list_of_urls(
         seed_urls=seed_urls,
         object_name=script.object_name,
         max_iterations=script.nodes[node_name]["max_iterations"],
-        use_cache=use_cache,
+        cache_prefix=node_name,
     )
 
-    script.nodes[node_name]["visited_urls"] = visited_urls
-    script.nodes[node_name]["output"] = "TBA"
+    script.nodes[node_name]["output"] = success
 
-    return True
+    return success

{blue_assistant-4.283.1 → blue_assistant-4.294.1}/blue_assistant/script/repository/base/classes.py

@@ -147,14 +147,12 @@ class BaseScript:
 
     def run(
         self,
-        use_cache: bool = True,
     ) -> bool:
         logger.info(
-            "{}.run: {}:{} -{}> {}".format(
+            "{}.run: {}:{} -> {}".format(
                 NAME,
                 self.__class__.__name__,
                 self.name,
-                "use-cache-" if use_cache else "",
                 self.object_name,
             )
         )

{blue_assistant-4.283.1 → blue_assistant-4.294.1}/blue_assistant/script/repository/blue_amo/actions/slicing_into_frames.py

@@ -10,7 +10,6 @@ NAME = module.name(__file__, NAME)
 def slicing_into_frames(
     script: BaseScript,
     node_name: str,
-    use_cache: bool,
 ) -> bool:
     logger.info(f"{NAME}: processing the output...")
 

{blue_assistant-4.283.1 → blue_assistant-4.294.1}/blue_assistant/script/repository/blue_amo/actions/stitching_the_frames.py

@@ -18,7 +18,6 @@ NAME = module.name(__file__, NAME)
 def stitching_the_frames(
     script: BaseScript,
     node_name: str,
-    use_cache: bool,
 ) -> bool:
     list_of_frames_filenames: List[str] = [
         filename

{blue_assistant-4.283.1 → blue_assistant-4.294.1}/blue_assistant/script/repository/blue_amo/classes.py

@@ -66,11 +66,9 @@ class BlueAmoScript(GenericScript):
     def perform_action(
         self,
         node_name: str,
-        use_cache: bool,
     ) -> bool:
         if not super().perform_action(
             node_name=node_name,
-            use_cache=use_cache,
         ):
             return False
 
@@ -78,7 +76,6 @@ class BlueAmoScript(GenericScript):
             return dict_of_actions[node_name](
                 script=self,
                 node_name=node_name,
-                use_cache=use_cache,
             )
 
         return True

{blue_assistant-4.283.1 → blue_assistant-4.294.1}/blue_assistant/script/repository/generic/classes.py

@@ -22,7 +22,6 @@ class GenericScript(BaseScript):
     def perform_action(
         self,
         node_name: str,
-        use_cache: bool,
     ) -> bool:
         action_name = self.nodes[node_name].get("action", "unknown")
         logger.info(f"---- node: {node_name} ---- ")
@@ -34,14 +33,12 @@ class GenericScript(BaseScript):
         return dict_of_actions[action_name](
             script=self,
             node_name=node_name,
-            use_cache=use_cache,
         )
 
     def run(
         self,
-        use_cache: bool = True,
     ) -> bool:
-        if not super().run(use_cache=use_cache):
+        if not super().run():
             return False
 
         success: bool = True
@@ -75,7 +72,6 @@ class GenericScript(BaseScript):
 
                 if not self.perform_action(
                     node_name=node_name,
-                    use_cache=use_cache,
                 ):
                     success = False
                     break

{blue_assistant-4.283.1 → blue_assistant-4.294.1}/blue_assistant/web/__main__.py

@@ -40,26 +40,18 @@ parser.add_argument(
     "--object_name",
     type=str,
 )
-parser.add_argument(
-    "--use_cache",
-    type=int,
-    default=0,
-    help="0 | 1",
-)
 args = parser.parse_args()
 
 success = False
 if args.task == "crawl":
-    success = True
-    dict_of_urls = crawl_list_of_urls(
+    success, crawl_cache = crawl_list_of_urls(
         seed_urls=args.seed_urls.split("+"),
         object_name=args.object_name,
         max_iterations=args.max_iterations,
-        use_cache=args.use_cache == 1,
     )
 
     if args.verbose == 1:
-        log_dict(logger, "crawled", dict_of_urls, "url(s)")
+        log_dict(logger, "crawled", crawl_cache, "url(s)")
 elif args.task == "fetch":
     summary = fetch_links_and_text(
         url=args.url,

{blue_assistant-4.283.1 → blue_assistant-4.294.1}/blue_assistant/web/crawl.py

@@ -1,4 +1,4 @@
-from typing import List, Dict
+from typing import List, Dict, Tuple
 
 from blueness import module
 from blue_options.logger import log_dict, log_list
@@ -18,46 +18,47 @@ def crawl_list_of_urls(
     seed_urls: List[str],
     object_name: str,
     max_iterations: int = 10,
-    use_cache: bool = False,
     verbose: bool = False,
-) -> Dict[str, str]:
+    cache_prefix: str = "",
+) -> Tuple[bool, Dict[str, str]]:
     logger.info(
-        "{}.crawl_list_of_urls({}): {} -{}> {}".format(
+        "{}.crawl_list_of_urls({}): {} -> {}".format(
             NAME,
             len(seed_urls),
             ", ".join(seed_urls),
-            "use-cache-" if use_cache else "",
             object_name,
         )
     )
 
-    crawl_cache: Dict[str, str] = {}
-    queue: List[str] = [url for url in seed_urls]
-
-    if use_cache:
-        crawl_cache = get_from_object(
-            object_name,
-            "crawl_cache",
-            {},
-        )
-        log_dict(logger, "loaded cache:", crawl_cache, "url(s)")
-
-        queue += get_from_object(
-            object_name,
-            "crawl_queue",
-            [],
+    queue: List[str] = list(
+        set(
+            seed_urls
+            + get_from_object(
+                object_name,
+                f"{cache_prefix}_crawl_queue",
+                [],
+            )
         )
-
+    )
     log_list(logger, "queue:", queue, "url(s)")
 
+    crawl_cache: Dict[str, str] = get_from_object(
+        object_name,
+        f"{cache_prefix}_crawl_cache",
+        {},
+    )
+    log_dict(logger, "loaded cache:", crawl_cache, "url(s)")
+
     iteration: int = 0
     while queue:
         url = queue[0]
         queue = queue[1:]
 
         logger.info(
-            "{} {} ...".format(
+            "{} [#{:,}/{:,}]: {} ".format(
                 "✅ " if url in crawl_cache else "🔗 ",
+                iteration,
+                len(queue),
                 url,
             )
         )
@@ -70,27 +71,31 @@ def crawl_list_of_urls(
         )
         content_type = url_summary.get("content_type", "unknown")
 
-        if use_cache and "html" in content_type:
-            file.save_yaml(
+        if "html" in content_type:
+            if not file.save_yaml(
                 filename=objects.path_of(
                     object_name=object_name,
-                    filename="crawl_summary_cache/{}.yaml".format(url_to_filename(url)),
+                    filename="{}_crawl_cache/{}.yaml".format(
+                        cache_prefix,
+                        url_to_filename(url),
+                    ),
                 ),
                 data=url_summary,
-            )
+            ):
+                return False, {}
 
         crawl_cache[url] = content_type
-        if "list_of_urls" in url_summary:
-            queue = list(
-                set(
-                    queue
-                    + [
-                        url
-                        for url in url_summary["list_of_urls"]
-                        if url not in crawl_cache.keys()
-                    ]
-                )
-            )
+
+        queue = (
+            queue
+            + url_summary.get("list_of_urls", [])
+            + [
+                url
+                for url in url_summary.get("list_of_ignored_urls", [])
+                if any(url.startswith(url_prefix) for url_prefix in seed_urls)
+            ]
+        )
+        queue = list({url for url in queue if url not in crawl_cache.keys()})
 
         iteration += 1
         if max_iterations != -1 and iteration >= max_iterations:
@@ -100,20 +105,21 @@ def crawl_list_of_urls(
     if queue:
         logger.warning(f"queue: {len(queue)}")
 
-    if use_cache:
+    if not (
         post_to_object(
             object_name,
-            "crawl_cache",
+            f"{cache_prefix}_crawl_cache",
             crawl_cache,
         )
-
-        post_to_object(
+        and post_to_object(
             object_name,
-            "crawl_queue",
+            f"{cache_prefix}_crawl_queue",
             queue,
         )
+    ):
+        return False, {}
 
     log_dict(logger, "crawled", crawl_cache, "url(s)")
     log_list(logger, "queue:", queue, "url(s)")
 
-    return crawl_cache
+    return True, crawl_cache

{blue_assistant-4.283.1 → blue_assistant-4.294.1}/blue_assistant/web/fetch.py

@@ -49,6 +49,9 @@ def fetch_links_and_text(
         for a_tag in soup.find_all("a", href=True):
             a_url = urljoin(url, a_tag["href"])
 
+            if "#" in a_url:
+                a_url = a_url.split("#", 1)[0]
+
             if a_url.startswith(url):
                 if url not in list_of_urls:
                     logger.info(f"+= {a_url}")

{blue_assistant-4.283.1 → blue_assistant-4.294.1/blue_assistant.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: blue_assistant
-Version: 4.283.1
+Version: 4.294.1
 Summary: 🧠 An AI Assistant.
 Home-page: https://github.com/kamangir/blue-assistant
 Author: Arash Abadpour (Kamangir)
@@ -124,4 +124,4 @@ Also home to [`@web`](https://raw.githubusercontent.com/kamangir/blue-assistant/
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.283.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.294.1`](https://github.com/kamangir/blue-assistant).