blue-assistant 4.226.1__tar.gz → 4.268.1__tar.gz

{blue_assistant-4.226.1/blue_assistant.egg-info → blue_assistant-4.268.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: blue_assistant
-Version: 4.226.1
+Version: 4.268.1
 Summary: 🧠 An AI Assistant.
 Home-page: https://github.com/kamangir/blue-assistant
 Author: Arash Abadpour (Kamangir)
@@ -56,10 +56,14 @@ graph LR
     assistant_script_list["@assistant<br>script<br>list"]
     assistant_script_run["@assistant<br>script<br>run -<br>&lt;script&gt;<br>&lt;object-name&gt;"]
 
-    assistant_web_crawl["@assistant<br>web<br>crawl -<br>&lt;url-1&gt;+&lt;url-2&gt;+&lt;url-3&gt;<br>&lt;object-name&gt;"]
+    web_crawl["@web<br>crawl -<br>&lt;url-1&gt;+&lt;url-2&gt;<br>&lt;object-name&gt;"]
+
+    web_fetch["@web<br>fetch -<br>&lt;url&gt;<br>&lt;object-name&gt;"]
 
     script["📜 script"]:::folder
     url["🔗 url"]:::folder
+    url2["🔗 url"]:::folder
+    url3["🔗 url"]:::folder
     object["📂 object"]:::folder
 
 
@@ -69,9 +73,13 @@ graph LR
     object --> assistant_script_run
     assistant_script_run --> object
 
-    url --> assistant_web_crawl
-    assistant_web_crawl --> object
+    url --> web_crawl
+    url2 --> web_crawl
+    web_crawl --> url3
+    web_crawl --> object
 
+    url --> web_fetch
+    web_fetch --> object
 
     bridge_ip["🔗 bridge_ip"]:::folder
     hue_username["🔗 hue_username"]:::folder
@@ -104,12 +112,16 @@ graph LR
 
 |   |   |
 | --- | --- |
-| [`hue`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/hue) [![image](https://github.com/kamangir/assets/raw/main/blue-assistant/20250314_143702.jpg?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/hue) "send a color command to the Hue LED lights in my apartment." | [`blue-amo`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/blue_amo/README.md) [![image](https://github.com/kamangir/assets/raw/main/blue-amo-2025-02-03-nswnx6/stitching_the_frames-2.png?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/blue_amo/README.md) A story developed and visualized, by AI. |
-| [`orbital-data-explorer`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/orbital_data_explorer/README.md) [![image](https://github.com/kamangir/assets/blob/main/blue-assistant/orbital-data-explorer.png?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/orbital_data_explorer/README.md) Access to the [Orbital Data Explorer](https://ode.rsl.wustl.edu/), through AI. ⏸️ |  |
+| [`orbital-data-explorer`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/orbital_data_explorer) [![image](https://github.com/kamangir/assets/blob/main/blue-assistant/orbital-data-explorer.png?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/orbital_data_explorer) Access to the [Orbital Data Explorer](https://ode.rsl.wustl.edu/). 🔥 | [`🌀 blue script`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/) [![image](https://github.com/kamangir/assets/raw/main/blue-plugin/marquee.png?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/) A minimal AI DAG interface. |
+| [`@hue`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/hue) [![image](https://github.com/kamangir/assets/raw/main/blue-assistant/20250314_143702.jpg?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/hue) "send a color command to the Hue LED lights in my apartment." | [`blue-amo`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/blue_amo/README.md) [![image](https://github.com/kamangir/assets/raw/main/blue-amo-2025-02-03-nswnx6/stitching_the_frames-2.png?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/blue_amo/README.md) Story development and visualization. |
+
+---
+
+Also home to [`@web`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/web/)
 
 ---
 
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.234.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.226.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.236.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.268.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.226.1 → blue_assistant-4.268.1}/README.md

@@ -11,10 +11,14 @@ graph LR
     assistant_script_list["@assistant<br>script<br>list"]
     assistant_script_run["@assistant<br>script<br>run -<br>&lt;script&gt;<br>&lt;object-name&gt;"]
 
-    assistant_web_crawl["@assistant<br>web<br>crawl -<br>&lt;url-1&gt;+&lt;url-2&gt;+&lt;url-3&gt;<br>&lt;object-name&gt;"]
+    web_crawl["@web<br>crawl -<br>&lt;url-1&gt;+&lt;url-2&gt;<br>&lt;object-name&gt;"]
+
+    web_fetch["@web<br>fetch -<br>&lt;url&gt;<br>&lt;object-name&gt;"]
 
     script["📜 script"]:::folder
     url["🔗 url"]:::folder
+    url2["🔗 url"]:::folder
+    url3["🔗 url"]:::folder
     object["📂 object"]:::folder
 
 
@@ -24,9 +28,13 @@ graph LR
     object --> assistant_script_run
     assistant_script_run --> object
 
-    url --> assistant_web_crawl
-    assistant_web_crawl --> object
+    url --> web_crawl
+    url2 --> web_crawl
+    web_crawl --> url3
+    web_crawl --> object
 
+    url --> web_fetch
+    web_fetch --> object
 
     bridge_ip["🔗 bridge_ip"]:::folder
     hue_username["🔗 hue_username"]:::folder
@@ -59,12 +67,16 @@ graph LR
 
 |   |   |
 | --- | --- |
-| [`hue`](./blue_assistant/script/repository/hue) [![image](https://github.com/kamangir/assets/raw/main/blue-assistant/20250314_143702.jpg?raw=true)](./blue_assistant/script/repository/hue) "send a color command to the Hue LED lights in my apartment." | [`blue-amo`](./blue_assistant/script/repository/blue_amo/README.md) [![image](https://github.com/kamangir/assets/raw/main/blue-amo-2025-02-03-nswnx6/stitching_the_frames-2.png?raw=true)](./blue_assistant/script/repository/blue_amo/README.md) A story developed and visualized, by AI. |
-| [`orbital-data-explorer`](./blue_assistant/script/repository/orbital_data_explorer/README.md) [![image](https://github.com/kamangir/assets/blob/main/blue-assistant/orbital-data-explorer.png?raw=true)](./blue_assistant/script/repository/orbital_data_explorer/README.md) Access to the [Orbital Data Explorer](https://ode.rsl.wustl.edu/), through AI. ⏸️ |  |
+| [`orbital-data-explorer`](./blue_assistant/script/repository/orbital_data_explorer) [![image](https://github.com/kamangir/assets/blob/main/blue-assistant/orbital-data-explorer.png?raw=true)](./blue_assistant/script/repository/orbital_data_explorer) Access to the [Orbital Data Explorer](https://ode.rsl.wustl.edu/). 🔥 | [`🌀 blue script`](./blue_assistant/script/) [![image](https://github.com/kamangir/assets/raw/main/blue-plugin/marquee.png?raw=true)](./blue_assistant/script/) A minimal AI DAG interface. |
+| [`@hue`](./blue_assistant/script/repository/hue) [![image](https://github.com/kamangir/assets/raw/main/blue-assistant/20250314_143702.jpg?raw=true)](./blue_assistant/script/repository/hue) "send a color command to the Hue LED lights in my apartment." | [`blue-amo`](./blue_assistant/script/repository/blue_amo/README.md) [![image](https://github.com/kamangir/assets/raw/main/blue-amo-2025-02-03-nswnx6/stitching_the_frames-2.png?raw=true)](./blue_assistant/script/repository/blue_amo/README.md) Story development and visualization. |
+
+---
+
+Also home to [`@web`](./blue_assistant/web/)
 
 ---
 
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.234.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.226.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.236.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.268.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.226.1 → blue_assistant-4.268.1}/blue_assistant/.abcli/alias.sh

@@ -3,3 +3,5 @@
 alias @assistant=blue_assistant
 
 alias @hue=blue_assistant_hue
+
+alias @web=blue_assistant_web

{blue_assistant-4.226.1 → blue_assistant-4.268.1}/blue_assistant/.abcli/tests/help.sh

@@ -23,15 +23,16 @@ function test_blue_assistant_help() {
         "@assistant script list" \
         "@assistant script run" \
         \
-        "@assistant web" \
-        "@assistant web crawl" \
-        \
         "@hue" \
         "@hue create_user" \
         "@hue list" \
         "@hue set" \
         "@hue test" \
         \
+        "@web" \
+        "@web crawl" \
+        "@web fetch" \
+        \
         "blue_assistant"; do
         abcli_eval ,$options \
             abcli_help $module

{blue_assistant-4.226.1 → blue_assistant-4.268.1}/blue_assistant/.abcli/tests/web_crawl.sh

@@ -3,10 +3,12 @@
 function test_blue_assistant_web_crawl() {
     local options=$1
 
+    local object_name=test_blue_assistant_web_crawl-$(abcli_string_timestamp_short)
+
     abcli_eval ,$options \
         blue_assistant_web_crawl \
         ~upload \
         https://ode.rsl.wustl.edu/+https://oderest.rsl.wustl.edu/ \
-        test_blue_assistant_web_crawl-$(abcli_string_timestamp_short) \
+        $object_name \
         --max_iterations 3
 }

blue_assistant-4.268.1/blue_assistant/.abcli/tests/web_fetch.sh

@@ -0,0 +1,13 @@
+#! /usr/bin/env bash
+
+function test_blue_assistant_web_fetch() {
+    local options=$1
+
+    local object_name=test_blue_assistant_web_fetch-$(abcli_string_timestamp_short)
+
+    abcli_eval ,$options \
+        blue_assistant_web_fetch \
+        ~upload \
+        https://ode.rsl.wustl.edu/ \
+        $object_name
+}

blue_assistant-4.268.1/blue_assistant/.abcli/web/fetch.sh

@@ -0,0 +1,26 @@
+#! /usr/bin/env bash
+
+function blue_assistant_web_fetch() {
+    local options=$1
+    local do_dryrun=$(abcli_option_int "$options" dryrun 0)
+    local do_upload=$(abcli_option_int "$options" upload $(abcli_not $do_dryrun))
+
+    local url=${2:-void}
+
+    local object_name=$(abcli_clarify_object $3 web-fetch-$(abcli_string_timestamp_short))
+
+    abcli_log "fetching $url -> $object_name ..."
+
+    abcli_eval dryrun=$do_dryrun \
+        python3 -m blue_assistant.web \
+        fetch \
+        --url $url \
+        --object_name $object_name \
+        "${@:4}"
+    [[ $? -ne 0 ]] && return 1
+
+    [[ "$do_upload" == 1 ]] &&
+        abcli_upload - $object_name
+
+    return 0
+}

{blue_assistant-4.226.1 → blue_assistant-4.268.1}/blue_assistant/README.py

@@ -1,14 +1,28 @@
 import os
 
+from blue_options.help.functions import get_help
 from blue_objects import file, README
 
 from blue_assistant import NAME, VERSION, ICON, REPO_NAME
+from blue_assistant.help.functions import help_functions
 
 
 items = README.Items(
     [
         {
-            "name": "hue",
+            "name": "orbital-data-explorer",
+            "url": "./blue_assistant/script/repository/orbital_data_explorer",
+            "marquee": "https://github.com/kamangir/assets/blob/main/blue-assistant/orbital-data-explorer.png?raw=true",
+            "description": "Access to the [Orbital Data Explorer](https://ode.rsl.wustl.edu/). 🔥",
+        },
+        {
+            "name": "🌀 blue script",
+            "marquee": "https://github.com/kamangir/assets/raw/main/blue-plugin/marquee.png?raw=true",
+            "description": "A minimal AI DAG interface.",
+            "url": "./blue_assistant/script/",
+        },
+        {
+            "name": "@hue",
             "url": "./blue_assistant/script/repository/hue",
             "marquee": "https://github.com/kamangir/assets/raw/main/blue-assistant/20250314_143702.jpg?raw=true",
             "description": '"send a color command to the Hue LED lights in my apartment."',
@@ -17,13 +31,7 @@ items = README.Items(
             "name": "blue-amo",
             "url": "./blue_assistant/script/repository/blue_amo/README.md",
             "marquee": "https://github.com/kamangir/assets/raw/main/blue-amo-2025-02-03-nswnx6/stitching_the_frames-2.png?raw=true",
-            "description": "A story developed and visualized, by AI.",
-        },
-        {
-            "name": "orbital-data-explorer",
-            "url": "./blue_assistant/script/repository/orbital_data_explorer/README.md",
-            "marquee": "https://github.com/kamangir/assets/blob/main/blue-assistant/orbital-data-explorer.png?raw=true",
-            "description": "Access to the [Orbital Data Explorer](https://ode.rsl.wustl.edu/), through AI. ⏸️",
+            "description": "Story development and visualization.",
         },
     ]
 )
@@ -39,6 +47,11 @@ def build():
             NAME=NAME,
             VERSION=VERSION,
             REPO_NAME=REPO_NAME,
+            help_function=lambda tokens: get_help(
+                tokens,
+                help_functions,
+                mono=True,
+            ),
         )
         for readme in [
             {
@@ -47,8 +60,14 @@ def build():
                 "path": "..",
             },
             {"path": "script/repository/blue_amo"},
-            {"path": "script/repository/orbital_data_explorer"},
+            #
+            {"path": "script/repository/orbital_data_explorer/docs/round-1.md"},
+            {"path": "script/repository/orbital_data_explorer/docs"},
+            #
             {"path": "script/repository/hue/docs/round-1.md"},
             {"path": "script/repository/hue/docs"},
+            #
+            {"path": "script/"},
+            {"path": "web/"},
         ]
     )

{blue_assistant-4.226.1 → blue_assistant-4.268.1}/blue_assistant/__init__.py

@@ -4,7 +4,7 @@ ICON = "🧠"
 
 DESCRIPTION = f"{ICON} An AI Assistant."
 
-VERSION = "4.226.1"
+VERSION = "4.268.1"
 
 REPO_NAME = "blue-assistant"
 

{blue_assistant-4.226.1 → blue_assistant-4.268.1}/blue_assistant/help/web.py

@@ -15,19 +15,38 @@ def help_crawl(
 
     return show_usage(
         [
-            "@assistant",
-            "web",
+            "@web",
             "crawl",
             f"[{options}]",
             "<url-1>+<url-2>+<url-3>",
             "[-|<object-name>]",
         ]
         + args,
-        "crawl the web.",
+        "crawl the urls.",
+        mono=mono,
+    )
+
+
+def help_fetch(
+    tokens: List[str],
+    mono: bool,
+) -> str:
+    options = xtra("dryrun,~upload", mono=mono)
+
+    return show_usage(
+        [
+            "@web",
+            "fetch",
+            f"[{options}]",
+            "<url>",
+            "[-|<object-name>]",
+        ],
+        "fetch <url>.",
         mono=mono,
     )
 
 
 help_functions = {
     "crawl": help_crawl,
+    "fetch": help_fetch,
 }

{blue_assistant-4.226.1 → blue_assistant-4.268.1}/blue_assistant/script/actions/__init__.py

@@ -1,14 +1,15 @@
-from typing import Dict, Callable, Tuple
+from typing import Dict, Callable
 
 from blue_assistant.script.repository.base.classes import BaseScript
 from blue_assistant.script.actions.generic import generic_action
 from blue_assistant.script.actions.generate_image import generate_image
 from blue_assistant.script.actions.generate_text import generate_text
-from blue_assistant.logger import logger
+from blue_assistant.script.actions.web_crawl import web_crawl
 
 
 dict_of_actions: Dict[str, Callable[[BaseScript, str], bool]] = {
     "generic": generic_action,
     "generate_image": generate_image,
     "generate_text": generate_text,
+    "web_crawl": web_crawl,
 }

blue_assistant-4.226.1/blue_assistant/script/repository/orbital_data_explorer/actions/researching_the_questions.py → blue_assistant-4.268.1/blue_assistant/script/actions/web_crawl.py

@@ -2,10 +2,11 @@ from typing import Dict
 from blueness import module
 from tqdm import tqdm
 
+from blue_options.logger import log_list
 from openai_commands.text_generation import api
 
 from blue_assistant import NAME
-from blue_assistant.web.crawl import crawl_list_of_urls
+from blue_assistant.web.functions import crawl_list_of_urls
 from blue_assistant.script.repository.base.classes import BaseScript
 from blue_assistant.logger import logger
 
@@ -13,14 +14,31 @@ from blue_assistant.logger import logger
 NAME = module.name(__file__, NAME)
 
 
-def researching_the_questions(
+def web_crawl(
     script: BaseScript,
     node_name: str,
 ) -> bool:
-    logger.info(f"{NAME}: ...")
+    logger.info(f"{NAME}: {script} @ {node_name} ...")
+
+    seed_url_var_name = script.nodes[node_name].get("seed_urls", "")
+    if not isinstance(seed_url_var_name, str):
+        logger.error(f"{node_name}: seed_urls must be a string.")
+        return False
+    # to allow both :::<var-name> and <var-name> - for convenience :)
+    if seed_url_var_name.startswith(":::"):
+        seed_url_var_name = seed_url_var_name[3:].strip()
+    if not seed_url_var_name:
+        logger.error(f"{node_name}: seed_urls not found.")
+        return False
+    if seed_url_var_name not in script.vars:
+        logger.error(f"{node_name}: {seed_url_var_name}: seed_urls not found in vars.")
+        return False
+
+    seed_urls = script.vars[seed_url_var_name]
+    log_list(logger, seed_urls, "seed url(s)")
 
     visited_urls = crawl_list_of_urls(
-        seed_urls=script.vars["seed_urls"],
+        seed_urls=seed_urls,
         object_name=script.object_name,
         max_iterations=script.nodes[node_name]["max_iterations"],
     )

{blue_assistant-4.226.1 → blue_assistant-4.268.1}/blue_assistant/script/repository/base/classes.py

@@ -27,8 +27,6 @@ class BaseScript:
         self.object_name = object_name
 
         self.test_mode = test_mode
-        if self.test_mode:
-            logger.info("💰 test mode is on.")
 
         self.verbose = verbose
 
@@ -41,14 +39,70 @@ class BaseScript:
         success, self.metadata = file.load_yaml(metadata_filename)
         assert success, f"cannot load {self.name}/metadata.yaml"
 
-        logger.info("loaded {} node(s)".format(len(self.nodes)))
+        self.metadata.setdefault("script", {})
+        assert isinstance(
+            self.script,
+            dict,
+        ), "script: expected dict, received {}.".format(
+            self.script.__class__.__name__,
+        )
+
+        self.script.setdefault("nodes", {})
+        assert isinstance(
+            self.nodes,
+            dict,
+        ), "nodes: expected dict, received {}.".format(
+            self.nodes.__class__.__name__,
+        )
+
+        self.script.setdefault("vars", {})
+        assert isinstance(
+            self.vars,
+            dict,
+        ), "vars: expected dict, received {}.".format(
+            self.vars.__class__.__name__,
+        )
+
+        if self.test_mode:
+            logger.info("🧪  test mode is on.")
+
+            for node_name, node in self.nodes.items():
+                if "test_mode" in self.script:
+                    updates = self.script["test_mode"]
+                    logger.info(f"🧪  vars.update({updates})")
+                    self.vars.update(updates)
+
+                if "test_mode" in node:
+                    updates = node["test_mode"]
+                    logger.info(f"🧪  {node_name}.update({updates})")
+                    node.update(updates)
 
-        logger.info("loaded {} variable(s)".format(len(self.vars)))
+        logger.info(
+            "loaded {} node(s): {}".format(
+                len(self.nodes),
+                ", ".join(self.nodes.keys()),
+            )
+        )
+
+        logger.info(
+            "loaded {} var(s){}".format(
+                len(self.vars),
+                "" if verbose else ": {}".format(", ".join(self.vars.keys())),
+            )
+        )
         if verbose:
             for var_name, var_value in self.vars.items():
                 logger.info("{}: {}".format(var_name, var_value))
 
-        assert self.generate_graph(), "cannot generate graph"
+        assert self.generate_graph(), "cannot generate graph."
+
+    def __str__(self) -> str:
+        return "{}[{} var(s), {} node(s) -> {}]".format(
+            self.__class__.__name__,
+            len(self.vars),
+            len(self.nodes),
+            self.object_name,
+        )
 
     def apply_vars(self, text: str) -> str:
         for var_name, var_value in self.vars.items():
@@ -126,12 +180,12 @@ class BaseScript:
     # Aliases
     @property
     def script(self) -> Dict:
-        return self.metadata.get("script", {})
+        return self.metadata["script"]
 
     @property
     def nodes(self) -> Dict[str, Dict]:
-        return self.metadata.get("script", {}).get("nodes", {})
+        return self.metadata["script"]["nodes"]
 
     @property
     def vars(self) -> Dict:
-        return self.metadata.get("script", {}).get("vars", {})
+        return self.metadata["script"]["vars"]

{blue_assistant-4.226.1 → blue_assistant-4.268.1}/blue_assistant/script/repository/generic/classes.py

@@ -35,9 +35,7 @@ class GenericScript(BaseScript):
             node_name=node_name,
         )
 
-    def run(
-        self,
-    ) -> bool:
+    def run(self) -> bool:
         if not super().run():
             return False
 
@@ -50,6 +48,11 @@ class GenericScript(BaseScript):
                 if self.nodes[node_name].get("completed", False):
                     continue
 
+                if not self.nodes[node_name].get("runnable", True):
+                    logger.info(f"Not runnable, skipped: {node_name}.")
+                    self.nodes[node_name]["completed"] = True
+                    continue
+
                 pending_dependencies = [
                     node_name_
                     for node_name_ in self.G.successors(node_name)

blue_assistant-4.268.1/blue_assistant/script/repository/orbital_data_explorer/classes.py

@@ -0,0 +1,7 @@
+from blue_objects import file, path
+
+from blue_assistant.script.repository.generic.classes import GenericScript
+
+
+class OrbitalDataExplorerScript(GenericScript):
+    name = path.name(file.path(__file__))

{blue_assistant-4.226.1 → blue_assistant-4.268.1}/blue_assistant/web/__main__.py

@@ -2,9 +2,11 @@ import argparse
 
 from blueness import module
 from blueness.argparse.generic import sys_exit
+from blue_options.logger import log_dict
+from blue_objects.metadata import post_to_object
 
 from blue_assistant import NAME
-from blue_assistant.web.crawl import crawl_list_of_urls
+from blue_assistant.web.functions import crawl_list_of_urls, fetch_links_and_text
 from blue_assistant.logger import logger
 
 NAME = module.name(__file__, NAME)
@@ -13,7 +15,7 @@ parser = argparse.ArgumentParser(NAME)
 parser.add_argument(
     "task",
     type=str,
-    help="crawl",
+    help="crawl | fetch",
 )
 parser.add_argument(
     "--max_iterations",
@@ -26,6 +28,10 @@ parser.add_argument(
     default=0,
     help="0 | 1",
 )
+parser.add_argument(
+    "--url",
+    type=str,
+)
 parser.add_argument(
     "--seed_urls",
     type=str,
@@ -38,22 +44,34 @@ args = parser.parse_args()
 
 success = False
 if args.task == "crawl":
-    success = True
-
-    output = crawl_list_of_urls(
+    dict_of_urls = crawl_list_of_urls(
         seed_urls=args.seed_urls.split("+"),
         object_name=args.object_name,
         max_iterations=args.max_iterations,
     )
 
     if args.verbose == 1:
-        logger.info(f"{len(output)} url(s)")
-        for index, (url, content) in enumerate(output.items()):
-            logger.info(f"#{index: 4} - {url}: {content[:200]}...\n")
-            if index > 10:
-                logger.info("...")
-                break
+        log_dict(logger, dict_of_urls, "url(s)")
 
+    success = post_to_object(
+        args.object_name,
+        NAME.replace(".", "-"),
+        dict_of_urls,
+    )
+elif args.task == "fetch":
+    links, text = fetch_links_and_text(
+        url=args.url,
+        verbose=True,
+    )
+
+    success = post_to_object(
+        args.object_name,
+        NAME.replace(".", "-"),
+        {
+            "links": list(links),
+            "text": text,
+        },
+    )
 else:
     success = None
 

blue_assistant-4.226.1/blue_assistant/web/crawl.py → blue_assistant-4.268.1/blue_assistant/web/functions.py

@@ -1,18 +1,25 @@
-from typing import List, Dict, Set
+from typing import List, Dict, Set, Tuple
 import requests
-from bs4 import BeautifulSoup
-from urllib.parse import urljoin, urlparse
+from bs4 import BeautifulSoup, XMLParsedAsHTMLWarning
+from urllib.parse import urljoin
+import re
+import warnings
 
 from blueness import module
-
+from blue_options.logger import log_long_text, log_list
 
 from blue_assistant import NAME
 from blue_assistant.logger import logger
 
+warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
+
 NAME = module.name(__file__, NAME)
 
 
-def fetch_links_and_content(url, base_url, original_path):
+def fetch_links_and_text(
+    url: str,
+    verbose: bool = False,
+) -> Tuple[List[str], str]:
     try:
         response = requests.get(url, timeout=5)
         response.raise_for_status()
@@ -23,17 +30,27 @@ def fetch_links_and_content(url, base_url, original_path):
     links = set()
 
     for a_tag in soup.find_all("a", href=True):
-        full_url = urljoin(base_url, a_tag["href"])
-        parsed_url = urlparse(full_url)
+        a_url = urljoin(url, a_tag["href"])
+
+        if a_url.startswith(url):
+            logger.info(f"+= {a_url}")
+            links.add(a_url)
+            continue
 
-        # Ensure link is from the same domain and in the same directory tree
-        if parsed_url.netloc == urlparse(
-            base_url
-        ).netloc and parsed_url.path.startswith(original_path):
-            links.add(full_url)
+        logger.info(f"ignored: {a_url}")
 
     plain_text = soup.get_text(separator=" ", strip=True)
 
+    # remove non-ASCII characters
+    plain_text = re.sub(r"[^\x20-\x7E]+", "", plain_text)
+    for thing in ["\r", "\n", "\t"]:
+        plain_text = plain_text.replace(thing, " ")
+    plain_text = re.sub(r"\s+", " ", plain_text).strip()
+
+    if verbose:
+        log_list(logger, list(links), "link(s)")
+        log_long_text(logger, plain_text)
+
     return links, plain_text
 
 
@@ -41,6 +58,7 @@ def crawl_list_of_urls(
     seed_urls: List[str],
     object_name: str,
     max_iterations: int = 10,
+    verbose: bool = False,
 ) -> Dict[str, str]:
     logger.info(
         "{}.crawl_list_of_urls({}): {} -> {}".format(
@@ -53,23 +71,20 @@ def crawl_list_of_urls(
 
     visited: Dict[str, str] = {}
     queue: Set[str] = set(seed_urls)
-    base_url = urlparse(seed_urls[0]).scheme + "://" + urlparse(seed_urls[0]).netloc
-    original_path = (
-        urlparse(seed_urls[0]).path.rsplit("/", 1)[0] + "/"
-    )  # Get base directory
 
     iteration: int = 0
     while queue:
-        current_url = queue.pop()
-        if current_url in visited:
+        url = queue.pop()
+        if url in visited:
             continue
 
-        logger.info(f"🔗  {current_url} ...")
-        new_links, content = fetch_links_and_content(
-            current_url, base_url, original_path
+        logger.info(f"🔗  {url} ...")
+        url_links, url_text = fetch_links_and_text(
+            url=url,
+            verbose=verbose,
         )
-        visited[current_url] = content
-        queue.update(new_links - visited.keys())
+        visited[url] = url_text
+        queue.update(url_links - visited.keys())
 
         iteration += 1
         if max_iterations != -1 and iteration >= max_iterations:

{blue_assistant-4.226.1 → blue_assistant-4.268.1/blue_assistant.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: blue_assistant
-Version: 4.226.1
+Version: 4.268.1
 Summary: 🧠 An AI Assistant.
 Home-page: https://github.com/kamangir/blue-assistant
 Author: Arash Abadpour (Kamangir)
@@ -56,10 +56,14 @@ graph LR
     assistant_script_list["@assistant<br>script<br>list"]
     assistant_script_run["@assistant<br>script<br>run -<br>&lt;script&gt;<br>&lt;object-name&gt;"]
 
-    assistant_web_crawl["@assistant<br>web<br>crawl -<br>&lt;url-1&gt;+&lt;url-2&gt;+&lt;url-3&gt;<br>&lt;object-name&gt;"]
+    web_crawl["@web<br>crawl -<br>&lt;url-1&gt;+&lt;url-2&gt;<br>&lt;object-name&gt;"]
+
+    web_fetch["@web<br>fetch -<br>&lt;url&gt;<br>&lt;object-name&gt;"]
 
     script["📜 script"]:::folder
     url["🔗 url"]:::folder
+    url2["🔗 url"]:::folder
+    url3["🔗 url"]:::folder
     object["📂 object"]:::folder
 
 
@@ -69,9 +73,13 @@ graph LR
     object --> assistant_script_run
     assistant_script_run --> object
 
-    url --> assistant_web_crawl
-    assistant_web_crawl --> object
+    url --> web_crawl
+    url2 --> web_crawl
+    web_crawl --> url3
+    web_crawl --> object
 
+    url --> web_fetch
+    web_fetch --> object
 
     bridge_ip["🔗 bridge_ip"]:::folder
     hue_username["🔗 hue_username"]:::folder
@@ -104,12 +112,16 @@ graph LR
 
 |   |   |
 | --- | --- |
-| [`hue`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/hue) [![image](https://github.com/kamangir/assets/raw/main/blue-assistant/20250314_143702.jpg?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/hue) "send a color command to the Hue LED lights in my apartment." | [`blue-amo`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/blue_amo/README.md) [![image](https://github.com/kamangir/assets/raw/main/blue-amo-2025-02-03-nswnx6/stitching_the_frames-2.png?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/blue_amo/README.md) A story developed and visualized, by AI. |
-| [`orbital-data-explorer`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/orbital_data_explorer/README.md) [![image](https://github.com/kamangir/assets/blob/main/blue-assistant/orbital-data-explorer.png?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/orbital_data_explorer/README.md) Access to the [Orbital Data Explorer](https://ode.rsl.wustl.edu/), through AI. ⏸️ |  |
+| [`orbital-data-explorer`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/orbital_data_explorer) [![image](https://github.com/kamangir/assets/blob/main/blue-assistant/orbital-data-explorer.png?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/orbital_data_explorer) Access to the [Orbital Data Explorer](https://ode.rsl.wustl.edu/). 🔥 | [`🌀 blue script`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/) [![image](https://github.com/kamangir/assets/raw/main/blue-plugin/marquee.png?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/) A minimal AI DAG interface. |
+| [`@hue`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/hue) [![image](https://github.com/kamangir/assets/raw/main/blue-assistant/20250314_143702.jpg?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/hue) "send a color command to the Hue LED lights in my apartment." | [`blue-amo`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/blue_amo/README.md) [![image](https://github.com/kamangir/assets/raw/main/blue-amo-2025-02-03-nswnx6/stitching_the_frames-2.png?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/blue_amo/README.md) Story development and visualization. |
+
+---
+
+Also home to [`@web`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/web/)
 
 ---
 
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.234.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.226.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.236.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.268.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.226.1 → blue_assistant-4.268.1}/blue_assistant.egg-info/SOURCES.txt

@@ -38,7 +38,9 @@ blue_assistant/.abcli/tests/script_list.sh
 blue_assistant/.abcli/tests/script_run.sh
 blue_assistant/.abcli/tests/version.sh
 blue_assistant/.abcli/tests/web_crawl.sh
+blue_assistant/.abcli/tests/web_fetch.sh
 blue_assistant/.abcli/web/crawl.sh
+blue_assistant/.abcli/web/fetch.sh
 blue_assistant/help/__init__.py
 blue_assistant/help/__main__.py
 blue_assistant/help/functions.py
@@ -52,6 +54,7 @@ blue_assistant/script/actions/__init__.py
 blue_assistant/script/actions/generate_image.py
 blue_assistant/script/actions/generate_text.py
 blue_assistant/script/actions/generic.py
+blue_assistant/script/actions/web_crawl.py
 blue_assistant/script/repository/__init__.py
 blue_assistant/script/repository/base/__init__.py
 blue_assistant/script/repository/base/classes.py
@@ -69,8 +72,6 @@ blue_assistant/script/repository/hue/classes.py
 blue_assistant/script/repository/hue/colors.py
 blue_assistant/script/repository/orbital_data_explorer/__init__.py
 blue_assistant/script/repository/orbital_data_explorer/classes.py
-blue_assistant/script/repository/orbital_data_explorer/actions/__init__.py
-blue_assistant/script/repository/orbital_data_explorer/actions/researching_the_questions.py
 blue_assistant/web/__init__.py
 blue_assistant/web/__main__.py
-blue_assistant/web/crawl.py
+blue_assistant/web/functions.py

{blue_assistant-4.226.1 → blue_assistant-4.268.1}/setup.py

@@ -19,7 +19,6 @@ setup(
         f"{NAME}.script.repository.generic",
         f"{NAME}.script.repository.hue",
         f"{NAME}.script.repository.orbital_data_explorer",
-        f"{NAME}.script.repository.orbital_data_explorer.actions",
         f"{NAME}.web",
     ],
     include_package_data=True,

blue_assistant-4.226.1/blue_assistant/script/repository/orbital_data_explorer/actions/__init__.py

@@ -1,11 +0,0 @@
-from typing import Dict, Callable
-
-from blue_assistant.script.repository.base.classes import BaseScript
-from blue_assistant.script.repository.orbital_data_explorer.actions import (
-    researching_the_questions,
-)
-
-
-dict_of_actions: Dict[str, Callable[[BaseScript, str], bool]] = {
-    "researching_the_questions": researching_the_questions.researching_the_questions,
-}

blue_assistant-4.226.1/blue_assistant/script/repository/orbital_data_explorer/classes.py

@@ -1,40 +0,0 @@
-from blue_objects import file, path
-
-from blue_assistant.script.repository.generic.classes import GenericScript
-from blue_assistant.script.repository.orbital_data_explorer.actions import (
-    dict_of_actions,
-)
-
-
-class OrbitalDataExplorerScript(GenericScript):
-    name = path.name(file.path(__file__))
-
-    def __init__(
-        self,
-        object_name: str,
-        test_mode: bool = False,
-        verbose: bool = False,
-    ):
-        super().__init__(
-            object_name=object_name,
-            test_mode=test_mode,
-            verbose=verbose,
-        )
-
-        if self.test_mode:
-            self.nodes["researching_the_questions"]["max_iterations"] = 3
-
-    def perform_action(
-        self,
-        node_name: str,
-    ) -> bool:
-        if not super().perform_action(node_name=node_name):
-            return False
-
-        if node_name in dict_of_actions:
-            return dict_of_actions[node_name](
-                script=self,
-                node_name=node_name,
-            )
-
-        return True