blue-assistant 4.243.1__py3-none-any.whl → 4.273.1__py3-none-any.whl

blue_assistant/.abcli/script/run.sh

@@ -3,7 +3,8 @@
 function blue_assistant_script_run() {
     local options=$1
     local do_dryrun=$(abcli_option_int "$options" dryrun 0)
-    local do_download=$(abcli_option_int "$options" download 0)
+    local use_cache=$(abcli_option_int "$options" cache 1)
+    local do_download=$(abcli_option_int "$options" download $use_cache)
     local do_upload=$(abcli_option_int "$options" upload $(abcli_not $do_dryrun))
 
     local script_options=$2
@@ -20,6 +21,7 @@ function blue_assistant_script_run() {
         run \
         --script_name $script_name \
         --object_name $object_name \
+        --use_cache $use_cache \
         "${@:4}"
     [[ $? -ne 0 ]] && return 1
 

blue_assistant/.abcli/tests/script_run.sh

@@ -11,11 +11,13 @@ function test_blue_assistant_script_run() {
     for script_name in $(echo "$list_of_script_name" | tr + " "); do
         abcli_log "testing $script_name ..."
 
+        local object_name=test_blue_assistant_script_run-$(abcli_string_timestamp_short)
+
         abcli_eval ,$options \
             blue_assistant_script_run \
             ~upload,$options \
             script=$script_name \
-            test_blue_assistant_script_run-$(abcli_string_timestamp_short) \
+            $object_name \
             "${@:2}" \
             --test_mode 1 \
             --verbose 1

blue_assistant/.abcli/web/crawl.sh

@@ -3,11 +3,15 @@
 function blue_assistant_web_crawl() {
     local options=$1
     local do_dryrun=$(abcli_option_int "$options" dryrun 0)
+    local use_cache=$(abcli_option_int "$options" cache 0)
+    local do_download=$(abcli_option_int "$options" download $use_cache)
     local do_upload=$(abcli_option_int "$options" upload $(abcli_not $do_dryrun))
 
     local seed_urls=${2:-void}
 
     local object_name=$(abcli_clarify_object $3 web-crawl-$(abcli_string_timestamp_short))
+    [[ "$do_download" == 1 ]] &&
+        abcli_download - $object_name
 
     abcli_log "crawling $seed_urls -> $object_name ..."
 
@@ -16,6 +20,7 @@ function blue_assistant_web_crawl() {
         crawl \
         --seed_urls $seed_urls \
         --object_name $object_name \
+        --use_cache $use_cache \
         "${@:4}"
     [[ $? -ne 0 ]] && return 1
 

blue_assistant/README.py

@@ -1,14 +1,28 @@
 import os
 
+from blue_options.help.functions import get_help
 from blue_objects import file, README
 
 from blue_assistant import NAME, VERSION, ICON, REPO_NAME
+from blue_assistant.help.functions import help_functions
 
 
 items = README.Items(
     [
         {
-            "name": "hue",
+            "name": "orbital-data-explorer",
+            "url": "./blue_assistant/script/repository/orbital_data_explorer",
+            "marquee": "https://github.com/kamangir/assets/blob/main/blue-assistant/orbital-data-explorer.png?raw=true",
+            "description": "Access to the [Orbital Data Explorer](https://ode.rsl.wustl.edu/). 🔥",
+        },
+        {
+            "name": "🌀 blue script",
+            "marquee": "https://github.com/kamangir/assets/raw/main/blue-plugin/marquee.png?raw=true",
+            "description": "A minimal AI DAG interface.",
+            "url": "./blue_assistant/script/",
+        },
+        {
+            "name": "@hue",
             "url": "./blue_assistant/script/repository/hue",
             "marquee": "https://github.com/kamangir/assets/raw/main/blue-assistant/20250314_143702.jpg?raw=true",
             "description": '"send a color command to the Hue LED lights in my apartment."',
@@ -17,13 +31,7 @@ items = README.Items(
             "name": "blue-amo",
             "url": "./blue_assistant/script/repository/blue_amo/README.md",
             "marquee": "https://github.com/kamangir/assets/raw/main/blue-amo-2025-02-03-nswnx6/stitching_the_frames-2.png?raw=true",
-            "description": "A story developed and visualized, by AI.",
-        },
-        {
-            "name": "orbital-data-explorer",
-            "url": "./blue_assistant/script/repository/orbital_data_explorer",
-            "marquee": "https://github.com/kamangir/assets/blob/main/blue-assistant/orbital-data-explorer.png?raw=true",
-            "description": "Access to the [Orbital Data Explorer](https://ode.rsl.wustl.edu/), through AI. ⏸️",
+            "description": "Story development and visualization.",
         },
     ]
 )
@@ -39,6 +47,11 @@ def build():
             NAME=NAME,
             VERSION=VERSION,
             REPO_NAME=REPO_NAME,
+            help_function=lambda tokens: get_help(
+                tokens,
+                help_functions,
+                mono=True,
+            ),
         )
         for readme in [
             {
@@ -54,6 +67,7 @@ def build():
             {"path": "script/repository/hue/docs/round-1.md"},
             {"path": "script/repository/hue/docs"},
             #
+            {"path": "script/"},
             {"path": "web/"},
         ]
     )

blue_assistant/__init__.py

@@ -4,7 +4,7 @@ ICON = "🧠"
 
 DESCRIPTION = f"{ICON} An AI Assistant."
 
-VERSION = "4.243.1"
+VERSION = "4.273.1"
 
 REPO_NAME = "blue-assistant"
 

blue_assistant/help/script.py

@@ -30,7 +30,7 @@ def help_run(
     tokens: List[str],
     mono: bool,
 ) -> str:
-    options = xtra("download,dryrun,~upload", mono=mono)
+    options = xtra("~cache,download,dryrun,~upload", mono=mono)
 
     script_options = "script=<script>"
 

blue_assistant/help/web.py

@@ -7,7 +7,7 @@ def help_crawl(
     tokens: List[str],
     mono: bool,
 ) -> str:
-    options = xtra("dryrun,~upload", mono=mono)
+    options = xtra("cache,~download,dryrun,~upload", mono=mono)
 
     args = [
         "[--max_iterations <100000>]",

blue_assistant/script/__main__.py

@@ -48,6 +48,12 @@ parser.add_argument(
     default=1,
     help="0 | 1",
 )
+parser.add_argument(
+    "--use_cache",
+    type=int,
+    default=1,
+    help="0 | 1",
+)
 args = parser.parse_args()
 
 delim = " " if args.delim == "space" else args.delim
@@ -70,7 +76,9 @@ elif args.task == "run":
     )
 
     if success:
-        success = script.run()
+        success = script.run(
+            use_cache=args.use_cache == 1,
+        )
 else:
     success = None
 

blue_assistant/script/actions/__init__.py

@@ -1,14 +1,15 @@
-from typing import Dict, Callable, Tuple
+from typing import Dict, Callable
 
 from blue_assistant.script.repository.base.classes import BaseScript
 from blue_assistant.script.actions.generic import generic_action
 from blue_assistant.script.actions.generate_image import generate_image
 from blue_assistant.script.actions.generate_text import generate_text
-from blue_assistant.logger import logger
+from blue_assistant.script.actions.web_crawl import web_crawl
 
 
 dict_of_actions: Dict[str, Callable[[BaseScript, str], bool]] = {
     "generic": generic_action,
     "generate_image": generate_image,
     "generate_text": generate_text,
+    "web_crawl": web_crawl,
 }

blue_assistant/script/actions/generate_image.py

@@ -17,6 +17,7 @@ NAME = module.name(__file__, NAME)
 def generate_image(
     script: BaseScript,
     node_name: str,
+    use_cache: bool,
 ) -> bool:
     logger.info(f"{NAME}: {script} @ {node_name} ...")
 

blue_assistant/script/actions/generate_text.py

@@ -18,6 +18,7 @@ NAME = module.name(__file__, NAME)
 def generate_text(
     script: BaseScript,
     node_name: str,
+    use_cache: bool,
 ) -> bool:
     logger.info(f"{NAME}: {script} @ {node_name} ...")
 

blue_assistant/script/actions/generic.py

@@ -10,6 +10,7 @@ NAME = module.name(__file__, NAME)
 def generic_action(
     script: BaseScript,
     node_name: str,
+    use_cache: bool,
 ) -> bool:
     logger.info(f"{NAME}: {script} @ {node_name} ...")
     return True

blue_assistant/script/actions/web_crawl.py

@@ -0,0 +1,49 @@
+from blueness import module
+
+from blue_options.logger import log_list
+
+from blue_assistant import NAME
+from blue_assistant.web.crawl import crawl_list_of_urls
+from blue_assistant.script.repository.base.classes import BaseScript
+from blue_assistant.logger import logger
+
+
+NAME = module.name(__file__, NAME)
+
+
+def web_crawl(
+    script: BaseScript,
+    node_name: str,
+    use_cache: bool,
+) -> bool:
+    logger.info(f"{NAME}: {script} @ {node_name} ...")
+
+    seed_url_var_name = script.nodes[node_name].get("seed_urls", "")
+    if not isinstance(seed_url_var_name, str):
+        logger.error(f"{node_name}: seed_urls must be a string.")
+        return False
+    if not seed_url_var_name:
+        logger.error(f"{node_name}: seed_urls not found.")
+        return False
+
+    # to allow both :::<var-name> and <var-name> - for convenience :)
+    if seed_url_var_name.startswith(":::"):
+        seed_url_var_name = seed_url_var_name[3:].strip()
+
+    if seed_url_var_name not in script.vars:
+        logger.error(f"{node_name}: {seed_url_var_name}: seed_urls not found in vars.")
+        return False
+    seed_urls = script.vars[seed_url_var_name]
+    log_list(logger, seed_urls, "seed url(s)")
+
+    visited_urls = crawl_list_of_urls(
+        seed_urls=seed_urls,
+        object_name=script.object_name,
+        max_iterations=script.nodes[node_name]["max_iterations"],
+        use_cache=use_cache,
+    )
+
+    script.nodes[node_name]["visited_urls"] = visited_urls
+    script.nodes[node_name]["output"] = "TBA"
+
+    return True

blue_assistant/script/repository/base/classes.py

@@ -27,8 +27,6 @@ class BaseScript:
         self.object_name = object_name
 
         self.test_mode = test_mode
-        if self.test_mode:
-            logger.info("💰 test mode is on.")
 
         self.verbose = verbose
 
@@ -41,14 +39,70 @@ class BaseScript:
         success, self.metadata = file.load_yaml(metadata_filename)
         assert success, f"cannot load {self.name}/metadata.yaml"
 
-        logger.info("loaded {} node(s)".format(len(self.nodes)))
+        self.metadata.setdefault("script", {})
+        assert isinstance(
+            self.script,
+            dict,
+        ), "script: expected dict, received {}.".format(
+            self.script.__class__.__name__,
+        )
+
+        self.script.setdefault("nodes", {})
+        assert isinstance(
+            self.nodes,
+            dict,
+        ), "nodes: expected dict, received {}.".format(
+            self.nodes.__class__.__name__,
+        )
+
+        self.script.setdefault("vars", {})
+        assert isinstance(
+            self.vars,
+            dict,
+        ), "vars: expected dict, received {}.".format(
+            self.vars.__class__.__name__,
+        )
+
+        if self.test_mode:
+            logger.info("🧪  test mode is on.")
+
+            for node_name, node in self.nodes.items():
+                if "test_mode" in self.script:
+                    updates = self.script["test_mode"]
+                    logger.info(f"🧪  vars.update({updates})")
+                    self.vars.update(updates)
+
+                if "test_mode" in node:
+                    updates = node["test_mode"]
+                    logger.info(f"🧪  {node_name}.update({updates})")
+                    node.update(updates)
 
-        logger.info("loaded {} variable(s)".format(len(self.vars)))
+        logger.info(
+            "loaded {} node(s): {}".format(
+                len(self.nodes),
+                ", ".join(self.nodes.keys()),
+            )
+        )
+
+        logger.info(
+            "loaded {} var(s){}".format(
+                len(self.vars),
+                "" if verbose else ": {}".format(", ".join(self.vars.keys())),
+            )
+        )
         if verbose:
             for var_name, var_value in self.vars.items():
                 logger.info("{}: {}".format(var_name, var_value))
 
-        assert self.generate_graph(), "cannot generate graph"
+        assert self.generate_graph(), "cannot generate graph."
+
+    def __str__(self) -> str:
+        return "{}[{} var(s), {} node(s) -> {}]".format(
+            self.__class__.__name__,
+            len(self.vars),
+            len(self.nodes),
+            self.object_name,
+        )
 
     def apply_vars(self, text: str) -> str:
         for var_name, var_value in self.vars.items():
@@ -91,12 +145,16 @@ class BaseScript:
             [node_name],
         )
 
-    def run(self) -> bool:
+    def run(
+        self,
+        use_cache: bool = True,
+    ) -> bool:
         logger.info(
-            "{}.run: {}:{} -> {}".format(
+            "{}.run: {}:{} -{}> {}".format(
                 NAME,
                 self.__class__.__name__,
                 self.name,
+                "use-cache-" if use_cache else "",
                 self.object_name,
             )
         )
@@ -126,12 +184,12 @@ class BaseScript:
     # Aliases
     @property
     def script(self) -> Dict:
-        return self.metadata.get("script", {})
+        return self.metadata["script"]
 
     @property
     def nodes(self) -> Dict[str, Dict]:
-        return self.metadata.get("script", {}).get("nodes", {})
+        return self.metadata["script"]["nodes"]
 
     @property
     def vars(self) -> Dict:
-        return self.metadata.get("script", {}).get("vars", {})
+        return self.metadata["script"]["vars"]

blue_assistant/script/repository/blue_amo/actions/slicing_into_frames.py

@@ -10,6 +10,7 @@ NAME = module.name(__file__, NAME)
 def slicing_into_frames(
     script: BaseScript,
     node_name: str,
+    use_cache: bool,
 ) -> bool:
     logger.info(f"{NAME}: processing the output...")
 

blue_assistant/script/repository/blue_amo/actions/stitching_the_frames.py

@@ -18,6 +18,7 @@ NAME = module.name(__file__, NAME)
 def stitching_the_frames(
     script: BaseScript,
     node_name: str,
+    use_cache: bool,
 ) -> bool:
     list_of_frames_filenames: List[str] = [
         filename

blue_assistant/script/repository/blue_amo/classes.py

@@ -66,14 +66,19 @@ class BlueAmoScript(GenericScript):
     def perform_action(
         self,
         node_name: str,
+        use_cache: bool,
     ) -> bool:
-        if not super().perform_action(node_name=node_name):
+        if not super().perform_action(
+            node_name=node_name,
+            use_cache=use_cache,
+        ):
             return False
 
         if node_name in dict_of_actions:
             return dict_of_actions[node_name](
                 script=self,
                 node_name=node_name,
+                use_cache=use_cache,
             )
 
         return True

blue_assistant/script/repository/generic/classes.py

@@ -22,6 +22,7 @@ class GenericScript(BaseScript):
     def perform_action(
         self,
         node_name: str,
+        use_cache: bool,
     ) -> bool:
         action_name = self.nodes[node_name].get("action", "unknown")
         logger.info(f"---- node: {node_name} ---- ")
@@ -33,10 +34,14 @@ class GenericScript(BaseScript):
         return dict_of_actions[action_name](
             script=self,
             node_name=node_name,
+            use_cache=use_cache,
         )
 
-    def run(self) -> bool:
-        if not super().run():
+    def run(
+        self,
+        use_cache: bool = True,
+    ) -> bool:
+        if not super().run(use_cache=use_cache):
             return False
 
         success: bool = True
@@ -68,7 +73,10 @@ class GenericScript(BaseScript):
                     )
                     continue
 
-                if not self.perform_action(node_name=node_name):
+                if not self.perform_action(
+                    node_name=node_name,
+                    use_cache=use_cache,
+                ):
                     success = False
                     break
 

blue_assistant/script/repository/orbital_data_explorer/classes.py

@@ -1,40 +1,7 @@
 from blue_objects import file, path
 
 from blue_assistant.script.repository.generic.classes import GenericScript
-from blue_assistant.script.repository.orbital_data_explorer.actions import (
-    dict_of_actions,
-)
 
 
 class OrbitalDataExplorerScript(GenericScript):
     name = path.name(file.path(__file__))
-
-    def __init__(
-        self,
-        object_name: str,
-        test_mode: bool = False,
-        verbose: bool = False,
-    ):
-        super().__init__(
-            object_name=object_name,
-            test_mode=test_mode,
-            verbose=verbose,
-        )
-
-        if self.test_mode:
-            self.nodes["researching_the_questions"]["max_iterations"] = 3
-
-    def perform_action(
-        self,
-        node_name: str,
-    ) -> bool:
-        if not super().perform_action(node_name=node_name):
-            return False
-
-        if node_name in dict_of_actions:
-            return dict_of_actions[node_name](
-                script=self,
-                node_name=node_name,
-            )
-
-        return True

blue_assistant/web/__init__.py

@@ -0,0 +1,2 @@
+from blue_assistant.web.crawl import crawl_list_of_urls
+from blue_assistant.web.fetch import fetch_links_and_text

blue_assistant/web/__main__.py

@@ -6,7 +6,7 @@ from blue_options.logger import log_dict
 from blue_objects.metadata import post_to_object
 
 from blue_assistant import NAME
-from blue_assistant.web.functions import crawl_list_of_urls, fetch_links_and_text
+from blue_assistant.web import crawl_list_of_urls, fetch_links_and_text
 from blue_assistant.logger import logger
 
 NAME = module.name(__file__, NAME)
@@ -40,6 +40,12 @@ parser.add_argument(
     "--object_name",
     type=str,
 )
+parser.add_argument(
+    "--use_cache",
+    type=int,
+    default=0,
+    help="0 | 1",
+)
 args = parser.parse_args()
 
 success = False
@@ -48,6 +54,7 @@ if args.task == "crawl":
         seed_urls=args.seed_urls.split("+"),
         object_name=args.object_name,
         max_iterations=args.max_iterations,
+        use_cache=args.use_cache == 1,
     )
 
     if args.verbose == 1:
@@ -59,7 +66,7 @@ if args.task == "crawl":
         dict_of_urls,
     )
 elif args.task == "fetch":
-    links, text = fetch_links_and_text(
+    summary = fetch_links_and_text(
         url=args.url,
         verbose=True,
     )
@@ -67,10 +74,7 @@ elif args.task == "fetch":
     success = post_to_object(
         args.object_name,
         NAME.replace(".", "-"),
-        {
-            "links": list(links),
-            "text": text,
-        },
+        summary,
     )
 else:
     success = None

blue_assistant/web/crawl.py

@@ -0,0 +1,91 @@
+from typing import List, Dict, Set
+
+from blueness import module
+from blue_objects import file
+from blue_objects import objects
+from blue_objects.metadata import get_from_object, post_to_object
+
+from blue_assistant import NAME
+from blue_assistant.web.fetch import fetch_links_and_text
+from blue_assistant.web.functions import url_to_filename
+from blue_assistant.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+def crawl_list_of_urls(
+    seed_urls: List[str],
+    object_name: str,
+    max_iterations: int = 10,
+    use_cache: bool = False,
+    verbose: bool = False,
+) -> Dict[str, str]:
+    logger.info(
+        "{}.crawl_list_of_urls({}): {} -{}> {}".format(
+            NAME,
+            len(seed_urls),
+            ", ".join(seed_urls),
+            "use-cache-" if use_cache else "",
+            object_name,
+        )
+    )
+
+    crawl_cache: Dict[str, str] = (
+        get_from_object(
+            object_name,
+            "crawl_cache",
+            {},
+        )
+        if use_cache
+        else {}
+    )
+
+    queue: Set[str] = set(seed_urls)
+
+    iteration: int = 0
+    while queue:
+        url = queue.pop()
+        logger.info(
+            "{} {} ...".format(
+                "✅" if url in crawl_cache else "🔗",
+                url,
+            )
+        )
+        if url in crawl_cache:
+            continue
+
+        url_summary = fetch_links_and_text(
+            url=url,
+            verbose=verbose,
+        )
+        content_type = url_summary.get("content_type", "unknown")
+
+        if use_cache and "html" in content_type:
+            file.save_yaml(
+                filename=objects.path_of(
+                    object_name=object_name,
+                    filename="crawl_summary_cache/{}.yaml".format(url_to_filename(url)),
+                ),
+                data=url_summary,
+            )
+
+        crawl_cache[url] = content_type
+        if "links" in url_summary:
+            queue.update(url_summary["links"] - crawl_cache.keys())
+
+        iteration += 1
+        if max_iterations != -1 and iteration >= max_iterations:
+            logger.warning(f"max iteration of {max_iterations} reached.")
+            break
+
+    if queue:
+        logger.warning(f"queue: {len(queue)}")
+
+    if use_cache:
+        post_to_object(
+            object_name,
+            "crawl_cache",
+            crawl_cache,
+        )
+
+    return crawl_cache

blue_assistant/web/fetch.py

@@ -0,0 +1,75 @@
+from typing import Dict, Any
+import requests
+from bs4 import BeautifulSoup, XMLParsedAsHTMLWarning
+from urllib.parse import urljoin
+import re
+import warnings
+
+from blueness import module
+from blue_options.logger import log_long_text, log_list
+
+from blue_assistant import NAME
+from blue_assistant.logger import logger
+
+warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
+
+NAME = module.name(__file__, NAME)
+
+
+def fetch_links_and_text(
+    url: str,
+    verbose: bool = False,
+) -> Dict[str, Any]:
+    try:
+        response = requests.get(url, timeout=5)
+    except Exception as e:
+        logger.warning(e)
+        return {}
+
+    if response.status_code != 200:
+        logger.error(response)
+        return {}
+
+    content_type = response.headers.get("Content-Type", "")
+    logger.info(f"content-type: {content_type}")
+
+    links = set()
+    text = ""
+
+    if not any(
+        thing in content_type
+        for thing in [
+            "pdf",
+            "xml",
+        ]
+    ):
+        soup = BeautifulSoup(response.text, "html.parser")
+
+        for a_tag in soup.find_all("a", href=True):
+            a_url = urljoin(url, a_tag["href"])
+
+            if a_url.startswith(url):
+                logger.info(f"+= {a_url}")
+                links.add(a_url)
+                continue
+
+            logger.info(f"ignored: {a_url}")
+
+        text = soup.get_text(separator=" ", strip=True)
+
+    # remove non-ASCII characters
+    text = re.sub(r"[^\x20-\x7E]+", "", text)
+    for thing in ["\r", "\n", "\t"]:
+        text = text.replace(thing, " ")
+    text = re.sub(r"\s+", " ", text).strip()
+
+    if verbose:
+        log_list(logger, list(links), "link(s)")
+        log_long_text(logger, text)
+
+    return {
+        "url": url,
+        "content_type": content_type,
+        "links": links,
+        "text": text,
+    }

blue_assistant/web/functions.py

@@ -16,82 +16,24 @@ warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
 NAME = module.name(__file__, NAME)
 
 
-def fetch_links_and_text(
+def url_to_filename(
     url: str,
-    verbose: bool = False,
-) -> Tuple[List[str], str]:
-    try:
-        response = requests.get(url, timeout=5)
-        response.raise_for_status()
-    except requests.RequestException:
-        return set(), ""
+    max_length: int = 255,
+) -> str:
+    # Remove the URL scheme (http://, https://)
+    filename = re.sub(r"^https?://", "", url)
 
-    soup = BeautifulSoup(response.text, "html.parser")
-    links = set()
+    # Replace unwanted characters with an underscore
+    filename = re.sub(r"[^\w\s-]", "_", filename)
 
-    for a_tag in soup.find_all("a", href=True):
-        a_url = urljoin(url, a_tag["href"])
+    # Replace slashes with a hyphen to preserve some structure
+    filename = re.sub(r"\/", "-", filename)
 
-        if a_url.startswith(url):
-            logger.info(f"+= {a_url}")
-            links.add(a_url)
-            continue
+    # Replace spaces with underscores
+    filename = filename.replace(" ", "_")
 
-        logger.info(f"ignored: {a_url}")
+    # Ensure the filename length is not too long
+    if len(filename) > max_length:
+        filename = filename[:max_length]
 
-    plain_text = soup.get_text(separator=" ", strip=True)
-
-    # remove non-ASCII characters
-    plain_text = re.sub(r"[^\x20-\x7E]+", "", plain_text)
-    for thing in ["\r", "\n", "\t"]:
-        plain_text = plain_text.replace(thing, " ")
-    plain_text = re.sub(r"\s+", " ", plain_text).strip()
-
-    if verbose:
-        log_list(logger, list(links), "link(s)")
-        log_long_text(logger, plain_text)
-
-    return links, plain_text
-
-
-def crawl_list_of_urls(
-    seed_urls: List[str],
-    object_name: str,
-    max_iterations: int = 10,
-    verbose: bool = False,
-) -> Dict[str, str]:
-    logger.info(
-        "{}.crawl_list_of_urls({}): {} -> {}".format(
-            NAME,
-            len(seed_urls),
-            ", ".join(seed_urls),
-            object_name,
-        )
-    )
-
-    visited: Dict[str, str] = {}
-    queue: Set[str] = set(seed_urls)
-
-    iteration: int = 0
-    while queue:
-        url = queue.pop()
-        if url in visited:
-            continue
-
-        logger.info(f"🔗  {url} ...")
-        url_links, url_text = fetch_links_and_text(
-            url=url,
-            verbose=verbose,
-        )
-        visited[url] = url_text
-        queue.update(url_links - visited.keys())
-
-        iteration += 1
-        if max_iterations != -1 and iteration >= max_iterations:
-            logger.warning(f"max iteration of {max_iterations} reached.")
-            break
-
-    if queue:
-        logger.warning(f"queue: {len(queue)}")
-
-    return visited
+    return filename

{blue_assistant-4.243.1.dist-info → blue_assistant-4.273.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: blue_assistant
-Version: 4.243.1
+Version: 4.273.1
 Summary: 🧠 An AI Assistant.
 Home-page: https://github.com/kamangir/blue-assistant
 Author: Arash Abadpour (Kamangir)
@@ -112,8 +112,8 @@ graph LR
 
 |   |   |
 | --- | --- |
-| [`hue`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/hue) [![image](https://github.com/kamangir/assets/raw/main/blue-assistant/20250314_143702.jpg?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/hue) "send a color command to the Hue LED lights in my apartment." | [`blue-amo`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/blue_amo/README.md) [![image](https://github.com/kamangir/assets/raw/main/blue-amo-2025-02-03-nswnx6/stitching_the_frames-2.png?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/blue_amo/README.md) A story developed and visualized, by AI. |
-| [`orbital-data-explorer`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/orbital_data_explorer) [![image](https://github.com/kamangir/assets/blob/main/blue-assistant/orbital-data-explorer.png?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/orbital_data_explorer) Access to the [Orbital Data Explorer](https://ode.rsl.wustl.edu/), through AI. ⏸️ |  |
+| [`orbital-data-explorer`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/orbital_data_explorer) [![image](https://github.com/kamangir/assets/blob/main/blue-assistant/orbital-data-explorer.png?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/orbital_data_explorer) Access to the [Orbital Data Explorer](https://ode.rsl.wustl.edu/). 🔥 | [`🌀 blue script`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/) [![image](https://github.com/kamangir/assets/raw/main/blue-plugin/marquee.png?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/) A minimal AI DAG interface. |
+| [`@hue`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/hue) [![image](https://github.com/kamangir/assets/raw/main/blue-assistant/20250314_143702.jpg?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/hue) "send a color command to the Hue LED lights in my apartment." | [`blue-amo`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/blue_amo/README.md) [![image](https://github.com/kamangir/assets/raw/main/blue-amo-2025-02-03-nswnx6/stitching_the_frames-2.png?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/blue_amo/README.md) Story development and visualization. |
 
 ---
 
@@ -124,4 +124,4 @@ Also home to [`@web`](https://raw.githubusercontent.com/kamangir/blue-assistant/
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.236.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.243.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.236.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.273.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.243.1.dist-info → blue_assistant-4.273.1.dist-info}/RECORD

@@ -1,5 +1,5 @@
-blue_assistant/README.py,sha256=HIBukmtee6QW_oX-10mLX-0QrJoKECnEj79AXbqbDOs,2046
-blue_assistant/__init__.py,sha256=-mT913AX8fMIeCH6D0yVQhxEdzJMpO0SnmIcr1fH_iw,311
+blue_assistant/README.py,sha256=EJORj3I5pucJplI86lrFaZBN5C9IYNgKoG_V7h27NHw,2586
+blue_assistant/__init__.py,sha256=pPnU1NLSA8YbfAPdzhjwL5J0c1GSxruUlXume0IxvIE,311
 blue_assistant/__main__.py,sha256=URtal70XZc0--3FDTYWcLtnGOqBYjMX9gt-L1k8hDXI,361
 blue_assistant/config.env,sha256=npodyuuhkZUHUv9FnEiQQZkKxFbg8nQb1YpOCURqV3Y,301
 blue_assistant/env.py,sha256=FTSdJ8-J4jAyI0-h3MBgOweQBWd3YEFIibBHSXpClrY,760
@@ -20,53 +20,54 @@ blue_assistant/.abcli/hue/create_user.sh,sha256=Nh8FhnGweB2JZB7SVh-6jp8ud5YHeJSa
 blue_assistant/.abcli/hue/list.sh,sha256=ynptjPo6jZnwm-7wAVgGx-mZvyPKZ9b5JaJoY0xidCg,268
 blue_assistant/.abcli/hue/set.sh,sha256=VcADsfbjjbrxIMX9cVVHeK0MH649ZRY29V8YDTgflms,266
 blue_assistant/.abcli/script/list.sh,sha256=2lcVfqDfZP50NszF8o5YCo3TrJKeDc_qo7MTAF3XTGw,131
-blue_assistant/.abcli/script/run.sh,sha256=kSXmyM9NUj2X2orSGyu5t_P5frG-gyumbRq-xqF692c,911
+blue_assistant/.abcli/script/run.sh,sha256=gSobH_ct9o_9aC25S6CC-Io97EOA7vLbD1suOnFsr_A,1012
 blue_assistant/.abcli/tests/README.sh,sha256=Qs0YUxVB1OZZ70Nqw2kT1LKXeUnC5-XfQRMfqb8Cbwg,152
 blue_assistant/.abcli/tests/help.sh,sha256=7AAZzCEo5vZ1cBAMfj4virDClabaUMdOV-NqXSJQVUM,918
 blue_assistant/.abcli/tests/script_list.sh,sha256=OVOwWO9wR0eeDZTM6uub-eTKbz3eswU3vEUPWXcK-gQ,178
-blue_assistant/.abcli/tests/script_run.sh,sha256=vfmK8sjkMfSQPwCacQppiL6inMbvQP7nci7qLppFSL0,769
+blue_assistant/.abcli/tests/script_run.sh,sha256=5PFFNnWMpY55ZSYwIFsf9Ya0vHfncU1_8uBjY9Dke7I,809
 blue_assistant/.abcli/tests/version.sh,sha256=oR2rvYR8zi-0VDPIdPJsmsmWwYaamT8dmNTqUh3-8Gw,154
 blue_assistant/.abcli/tests/web_crawl.sh,sha256=sz3LbpidWvjG7kQoWxQBtdBe5yntm14ylAUsgPJWhko,372
 blue_assistant/.abcli/tests/web_fetch.sh,sha256=C8PFWlmRa9heNdP9yhshriCBKG1uUlps-oxhAM70AZI,312
-blue_assistant/.abcli/web/crawl.sh,sha256=M9YoKKJBKZT2OtmFPvRCSSKpiAq0zyacRAVZ6s7i3FM,698
+blue_assistant/.abcli/web/crawl.sh,sha256=GdDhp1MI1ZhGpH51Ie19Xb03xO8OM0cp9gGkgPjHCVE,934
 blue_assistant/.abcli/web/fetch.sh,sha256=9SggFZTtpff-gnCd987zP6UqzG4So5D4px2jMg2Vicc,674
 blue_assistant/help/__init__.py,sha256=ajz1GSNU9xYVrFEDSz6Xwg7amWQ_yvW75tQa1ZvRIWc,3
 blue_assistant/help/__main__.py,sha256=cVejR7OpoWPg0qLbm-PZf5TuJS27x49jzfiyCLyzEns,241
 blue_assistant/help/functions.py,sha256=O85zVEMtnm32O7KB6W6uQRoFXnE_4dW5pwYZtMakYDg,865
 blue_assistant/help/hue.py,sha256=ZElPG24ekiS7eIGLVrP2gB_womlGUuwln2cded4Li-c,2319
-blue_assistant/help/script.py,sha256=tofv49tIBqoH8ed9hDCFHqzWaXmyyPofvqElk2n976w,1121
-blue_assistant/help/web.py,sha256=-vi1P3p6zKQFbQzg9qhUMsjtZBLTjdJ0PZh7RokE9Wc,908
+blue_assistant/help/script.py,sha256=FmUZuGyVseFhytJxv7KW9Gr4gnm3U4ko9u6KSonjLvA,1128
+blue_assistant/help/web.py,sha256=87Ck9ofnBGuAn-akuYIMUsGFxLgEi1z0WRh-HF1156c,924
 blue_assistant/script/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-blue_assistant/script/__main__.py,sha256=eOSOo5yYTPMwIXZ0GkuWkmOcsDWrZtHvClyJizXSk2w,1657
+blue_assistant/script/__main__.py,sha256=7dHvHkfQCxRY0XsG2NGtDDT7h_mEJ6mvCOs7GeeTzxE,1798
 blue_assistant/script/load.py,sha256=JsDY9T3HTM9vXngvKsA0Mt_erxAnRR_jI62-JhrOBMU,831
-blue_assistant/script/actions/__init__.py,sha256=tiNL9HT1ql-Ut6CPYuj6B29PXQPuf-B-z8dJcksWACY,534
-blue_assistant/script/actions/generate_image.py,sha256=PgvOspDV8n2M7ZmgVOdZzJwQ1tnJNJ6V8gV94P74ksA,1336
-blue_assistant/script/actions/generate_text.py,sha256=SSyII0QPqiD538hveaN3RSyEHOt_lJ7Q-AVUg6fmlaM,1889
-blue_assistant/script/actions/generic.py,sha256=ET1RaKcUABM8HdIv8JecSpUFasYqmwHacL-5LjF-8NM,355
+blue_assistant/script/actions/__init__.py,sha256=GJJCaXSrfhwW9K5A2PHmrS7iGl-Ss9UUeYHUtxni4k0,576
+blue_assistant/script/actions/generate_image.py,sha256=brg3u6e-cZvBLK8B7UAh-CXgmAwyvSN-jE00EXMSF3A,1357
+blue_assistant/script/actions/generate_text.py,sha256=GQ7sF1J-vBNgr-h01RZHMDQJOGDXx8PzQWEVNQwW_TU,1910
+blue_assistant/script/actions/generic.py,sha256=EjJkDj82ZFFQbjn-Uib-2Qs-nZG8kR2NzhsEVuOpgWQ,376
+blue_assistant/script/actions/web_crawl.py,sha256=-EYbpXHbqPzKj-5AiDiXNgYIyUn-qv3mFz6RxhgBovo,1523
 blue_assistant/script/repository/__init__.py,sha256=zVI3cubRqM9H6WgF0EUP9idILVLCumPFmJgKPM7iVlM,604
 blue_assistant/script/repository/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-blue_assistant/script/repository/base/classes.py,sha256=-G81it3Beb3fbhE5H6g0BPJwvZKn3qELiBaqRWXP4OY,3717
+blue_assistant/script/repository/base/classes.py,sha256=gM8OB1iKRKLa_dk7esAogAR9WaNZFgLhXkpd1iTtsuQ,5376
 blue_assistant/script/repository/blue_amo/__init__.py,sha256=WjL9GIlN-DBnbUMJ8O_FxTp0rcVGlsIS3H9YtXEefTk,76
-blue_assistant/script/repository/blue_amo/classes.py,sha256=Rl_UX67AlfhVEhv8tfgdWKItOxI3cUfiGsN12-k8sWI,2085
+blue_assistant/script/repository/blue_amo/classes.py,sha256=8cOBClcLHXHKZK9HyOmN8GAB0lEdog93-CTfZwdavv8,2203
 blue_assistant/script/repository/blue_amo/actions/__init__.py,sha256=je2S21KvYB3QkbABs71parwUh8MCh2mdlNZfLx_QuDg,430
-blue_assistant/script/repository/blue_amo/actions/slicing_into_frames.py,sha256=79SI7_69FKKLeX_jHlfXnUWGtG4Sj7sBJOfeFwK8I9U,1201
-blue_assistant/script/repository/blue_amo/actions/stitching_the_frames.py,sha256=mbXriat6deEAmuo5Y1ValySnUXDENR7TZS_3nVPlQ6M,3622
+blue_assistant/script/repository/blue_amo/actions/slicing_into_frames.py,sha256=3PPdaJNbUyA0fvQ7Du6lFhRB5Uakax_CELPxhEuoGhI,1222
+blue_assistant/script/repository/blue_amo/actions/stitching_the_frames.py,sha256=n_UWcUqanEI6nn8AM0dlyjSUssk3xUrFmx-lZWUjOZE,3643
 blue_assistant/script/repository/generic/__init__.py,sha256=kLffGsQMQAFJTw6IZBE5eBxvshP1x9wwHHR4hsDJblo,75
-blue_assistant/script/repository/generic/classes.py,sha256=0_AE8TcSvtew5oM9BuJGJoi8jzRqImCmAwSpbn4dHuQ,2535
+blue_assistant/script/repository/generic/classes.py,sha256=5pP7jAZ42r2t4GEilK8uxCYlgydKJctO_lgsNr0BScs,2739
 blue_assistant/script/repository/hue/__init__.py,sha256=WjL9GIlN-DBnbUMJ8O_FxTp0rcVGlsIS3H9YtXEefTk,76
 blue_assistant/script/repository/hue/__main__.py,sha256=jaesrONQsrpVdg8A7NzzT8xpsdXs5gmrywOTE_TWD6c,2321
 blue_assistant/script/repository/hue/api.py,sha256=C3KzT_MG868gsznUXpwEbUleBjnJObWzZgzvN6wi3uo,4774
 blue_assistant/script/repository/hue/classes.py,sha256=YhifmcuylnZuI0_BjBPmwrSbsO-BOHDHNJ0pSLIExiE,188
 blue_assistant/script/repository/hue/colors.py,sha256=rUdtCroNAnzm1zUuVp8eVhvfIie1f7sd208ypsFAJ_w,625
 blue_assistant/script/repository/orbital_data_explorer/__init__.py,sha256=yy5FtCeHlr9dRfqxw4QYWr7_yRjnQpwVyuAY2vLrh4Q,110
-blue_assistant/script/repository/orbital_data_explorer/classes.py,sha256=NEJeud6UPXarnAIEvAX_ZKFoRnyK1tiEIORSsrixLxQ,1024
-blue_assistant/script/repository/orbital_data_explorer/actions/__init__.py,sha256=RcrFUAwnvhuwNh3gC65w9G26vd_cIa7LV1lFvGFcigk,370
-blue_assistant/script/repository/orbital_data_explorer/actions/researching_the_questions.py,sha256=rBJtrttp1LKFVkG5Lvntip6Dl3uO7CtlQRyd5PddVEU,1104
-blue_assistant/web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-blue_assistant/web/__main__.py,sha256=f6uPh7LnvIuVtrC9kZGbCr8nDFEJju-GcUgdRVsQvQE,1613
-blue_assistant/web/functions.py,sha256=AiezAvbw0eHG9XaYnXw8KQ_OjCfDvVPzsZxrzxwyPyg,2483
-blue_assistant-4.243.1.dist-info/LICENSE,sha256=ogEPNDSH0_dhiv_lT3ifVIdgIzHAqNA_SemnxUfPBJk,7048
-blue_assistant-4.243.1.dist-info/METADATA,sha256=VCI_zXnhWT1wAv9X-AQxh5fqgMXkoO9jh_L6MfHdXcY,5086
-blue_assistant-4.243.1.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
-blue_assistant-4.243.1.dist-info/top_level.txt,sha256=ud0BkBbdOVze13bNqHuhZj1rwCztaBtDf5ChEYzASOs,15
-blue_assistant-4.243.1.dist-info/RECORD,,
+blue_assistant/script/repository/orbital_data_explorer/classes.py,sha256=ixYd_FHWYtp8Sfd6AiZkIqePjoUlT9iLg7TvuxHIDzA,204
+blue_assistant/web/__init__.py,sha256=70_JSpnfX1mLm8Xv3xHIujfr2FfGeHPRs6HraWDP1XA,114
+blue_assistant/web/__main__.py,sha256=35RG-pqWEUcqNz0R59Efw4m00Azc46lu2r-42yX3Xqk,1663
+blue_assistant/web/crawl.py,sha256=K9EXWxhKBt6I9eViqSdpcx-z2aGsroafi2cmVOhgbOE,2368
+blue_assistant/web/fetch.py,sha256=b3EEMHoi-Tv2r2I5B2AVbDtHqrK0il42hvi2ZnBOKMY,1825
+blue_assistant/web/functions.py,sha256=CHGPM8RF-JtZlSL2vE0NFSZJVXMMMrXs8biwFk_JsSA,1042
+blue_assistant-4.273.1.dist-info/LICENSE,sha256=ogEPNDSH0_dhiv_lT3ifVIdgIzHAqNA_SemnxUfPBJk,7048
+blue_assistant-4.273.1.dist-info/METADATA,sha256=W6oaQcpfGcBWJfqHPU7K3gsZbljDiw82Y3Tn1tCKIrI,5380
+blue_assistant-4.273.1.dist-info/WHEEL,sha256=52BFRY2Up02UkjOa29eZOS2VxUrpPORXg1pkohGGUS8,91
+blue_assistant-4.273.1.dist-info/top_level.txt,sha256=ud0BkBbdOVze13bNqHuhZj1rwCztaBtDf5ChEYzASOs,15
+blue_assistant-4.273.1.dist-info/RECORD,,

blue_assistant/script/repository/orbital_data_explorer/actions/__init__.py

@@ -1,11 +0,0 @@
-from typing import Dict, Callable
-
-from blue_assistant.script.repository.base.classes import BaseScript
-from blue_assistant.script.repository.orbital_data_explorer.actions import (
-    researching_the_questions,
-)
-
-
-dict_of_actions: Dict[str, Callable[[BaseScript, str], bool]] = {
-    "researching_the_questions": researching_the_questions.researching_the_questions,
-}

blue_assistant/script/repository/orbital_data_explorer/actions/researching_the_questions.py

@@ -1,42 +0,0 @@
-from typing import Dict
-from blueness import module
-from tqdm import tqdm
-
-from openai_commands.text_generation import api
-
-from blue_assistant import NAME
-from blue_assistant.web.functions import crawl_list_of_urls
-from blue_assistant.script.repository.base.classes import BaseScript
-from blue_assistant.logger import logger
-
-
-NAME = module.name(__file__, NAME)
-
-
-def researching_the_questions(
-    script: BaseScript,
-    node_name: str,
-) -> bool:
-    logger.info(f"{NAME}: ...")
-
-    visited_urls = crawl_list_of_urls(
-        seed_urls=script.vars["seed_urls"],
-        object_name=script.object_name,
-        max_iterations=script.nodes[node_name]["max_iterations"],
-    )
-
-    success, output, _ = api.generate_text(
-        prompt=script.nodes[node_name]["prompt"].replace(
-            ":::input", " ".join([content for content in visited_urls.values()])
-        ),
-        verbose=script.verbose,
-    )
-    if not success:
-        return success
-
-    logger.info(output)
-
-    script.nodes[node_name]["visited_urls"] = visited_urls
-    script.nodes[node_name]["output"] = output
-
-    return True