blue-assistant 4.221.1__tar.gz → 4.243.1__tar.gz

{blue_assistant-4.221.1/blue_assistant.egg-info → blue_assistant-4.243.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: blue_assistant
-Version: 4.221.1
+Version: 4.243.1
 Summary: 🧠 An AI Assistant.
 Home-page: https://github.com/kamangir/blue-assistant
 Author: Arash Abadpour (Kamangir)
@@ -56,10 +56,14 @@ graph LR
     assistant_script_list["@assistant<br>script<br>list"]
     assistant_script_run["@assistant<br>script<br>run -<br>&lt;script&gt;<br>&lt;object-name&gt;"]
 
-    assistant_web_crawl["@assistant<br>web<br>crawl -<br>&lt;url-1&gt;+&lt;url-2&gt;+&lt;url-3&gt;<br>&lt;object-name&gt;"]
+    web_crawl["@web<br>crawl -<br>&lt;url-1&gt;+&lt;url-2&gt;<br>&lt;object-name&gt;"]
+
+    web_fetch["@web<br>fetch -<br>&lt;url&gt;<br>&lt;object-name&gt;"]
 
     script["📜 script"]:::folder
     url["🔗 url"]:::folder
+    url2["🔗 url"]:::folder
+    url3["🔗 url"]:::folder
     object["📂 object"]:::folder
 
 
@@ -69,9 +73,13 @@ graph LR
     object --> assistant_script_run
     assistant_script_run --> object
 
-    url --> assistant_web_crawl
-    assistant_web_crawl --> object
+    url --> web_crawl
+    url2 --> web_crawl
+    web_crawl --> url3
+    web_crawl --> object
 
+    url --> web_fetch
+    web_fetch --> object
 
     bridge_ip["🔗 bridge_ip"]:::folder
     hue_username["🔗 hue_username"]:::folder
@@ -105,11 +113,15 @@ graph LR
 |   |   |
 | --- | --- |
 | [`hue`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/hue) [![image](https://github.com/kamangir/assets/raw/main/blue-assistant/20250314_143702.jpg?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/hue) "send a color command to the Hue LED lights in my apartment." | [`blue-amo`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/blue_amo/README.md) [![image](https://github.com/kamangir/assets/raw/main/blue-amo-2025-02-03-nswnx6/stitching_the_frames-2.png?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/blue_amo/README.md) A story developed and visualized, by AI. |
-| [`orbital-data-explorer`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/orbital_data_explorer/README.md) [![image](https://github.com/kamangir/assets/blob/main/blue-assistant/orbital-data-explorer.png?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/orbital_data_explorer/README.md) Access to the [Orbital Data Explorer](https://ode.rsl.wustl.edu/), through AI. ⏸️ |  |
+| [`orbital-data-explorer`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/orbital_data_explorer) [![image](https://github.com/kamangir/assets/blob/main/blue-assistant/orbital-data-explorer.png?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/orbital_data_explorer) Access to the [Orbital Data Explorer](https://ode.rsl.wustl.edu/), through AI. ⏸️ |  |
+
+---
+
+Also home to [`@web`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/web/)
 
 ---
 
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.234.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.221.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.236.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.243.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.221.1 → blue_assistant-4.243.1}/README.md

@@ -11,10 +11,14 @@ graph LR
     assistant_script_list["@assistant<br>script<br>list"]
     assistant_script_run["@assistant<br>script<br>run -<br>&lt;script&gt;<br>&lt;object-name&gt;"]
 
-    assistant_web_crawl["@assistant<br>web<br>crawl -<br>&lt;url-1&gt;+&lt;url-2&gt;+&lt;url-3&gt;<br>&lt;object-name&gt;"]
+    web_crawl["@web<br>crawl -<br>&lt;url-1&gt;+&lt;url-2&gt;<br>&lt;object-name&gt;"]
+
+    web_fetch["@web<br>fetch -<br>&lt;url&gt;<br>&lt;object-name&gt;"]
 
     script["📜 script"]:::folder
     url["🔗 url"]:::folder
+    url2["🔗 url"]:::folder
+    url3["🔗 url"]:::folder
     object["📂 object"]:::folder
 
 
@@ -24,9 +28,13 @@ graph LR
     object --> assistant_script_run
     assistant_script_run --> object
 
-    url --> assistant_web_crawl
-    assistant_web_crawl --> object
+    url --> web_crawl
+    url2 --> web_crawl
+    web_crawl --> url3
+    web_crawl --> object
 
+    url --> web_fetch
+    web_fetch --> object
 
     bridge_ip["🔗 bridge_ip"]:::folder
     hue_username["🔗 hue_username"]:::folder
@@ -60,11 +68,15 @@ graph LR
 |   |   |
 | --- | --- |
 | [`hue`](./blue_assistant/script/repository/hue) [![image](https://github.com/kamangir/assets/raw/main/blue-assistant/20250314_143702.jpg?raw=true)](./blue_assistant/script/repository/hue) "send a color command to the Hue LED lights in my apartment." | [`blue-amo`](./blue_assistant/script/repository/blue_amo/README.md) [![image](https://github.com/kamangir/assets/raw/main/blue-amo-2025-02-03-nswnx6/stitching_the_frames-2.png?raw=true)](./blue_assistant/script/repository/blue_amo/README.md) A story developed and visualized, by AI. |
-| [`orbital-data-explorer`](./blue_assistant/script/repository/orbital_data_explorer/README.md) [![image](https://github.com/kamangir/assets/blob/main/blue-assistant/orbital-data-explorer.png?raw=true)](./blue_assistant/script/repository/orbital_data_explorer/README.md) Access to the [Orbital Data Explorer](https://ode.rsl.wustl.edu/), through AI. ⏸️ |  |
+| [`orbital-data-explorer`](./blue_assistant/script/repository/orbital_data_explorer) [![image](https://github.com/kamangir/assets/blob/main/blue-assistant/orbital-data-explorer.png?raw=true)](./blue_assistant/script/repository/orbital_data_explorer) Access to the [Orbital Data Explorer](https://ode.rsl.wustl.edu/), through AI. ⏸️ |  |
+
+---
+
+Also home to [`@web`](./blue_assistant/web/)
 
 ---
 
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.234.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.221.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.236.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.243.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.221.1 → blue_assistant-4.243.1}/blue_assistant/.abcli/alias.sh

@@ -3,3 +3,5 @@
 alias @assistant=blue_assistant
 
 alias @hue=blue_assistant_hue
+
+alias @web=blue_assistant_web

{blue_assistant-4.221.1 → blue_assistant-4.243.1}/blue_assistant/.abcli/tests/help.sh

@@ -23,15 +23,16 @@ function test_blue_assistant_help() {
         "@assistant script list" \
         "@assistant script run" \
         \
-        "@assistant web" \
-        "@assistant web crawl" \
-        \
         "@hue" \
         "@hue create_user" \
         "@hue list" \
         "@hue set" \
         "@hue test" \
         \
+        "@web" \
+        "@web crawl" \
+        "@web fetch" \
+        \
         "blue_assistant"; do
         abcli_eval ,$options \
             abcli_help $module

{blue_assistant-4.221.1 → blue_assistant-4.243.1}/blue_assistant/.abcli/tests/web_crawl.sh

@@ -3,10 +3,12 @@
 function test_blue_assistant_web_crawl() {
     local options=$1
 
+    local object_name=test_blue_assistant_web_crawl-$(abcli_string_timestamp_short)
+
     abcli_eval ,$options \
         blue_assistant_web_crawl \
         ~upload \
         https://ode.rsl.wustl.edu/+https://oderest.rsl.wustl.edu/ \
-        test_blue_assistant_web_crawl-$(abcli_string_timestamp_short) \
+        $object_name \
         --max_iterations 3
 }

blue_assistant-4.243.1/blue_assistant/.abcli/tests/web_fetch.sh

@@ -0,0 +1,13 @@
+#! /usr/bin/env bash
+
+function test_blue_assistant_web_fetch() {
+    local options=$1
+
+    local object_name=test_blue_assistant_web_fetch-$(abcli_string_timestamp_short)
+
+    abcli_eval ,$options \
+        blue_assistant_web_fetch \
+        ~upload \
+        https://ode.rsl.wustl.edu/ \
+        $object_name
+}

blue_assistant-4.243.1/blue_assistant/.abcli/web/fetch.sh

@@ -0,0 +1,26 @@
+#! /usr/bin/env bash
+
+function blue_assistant_web_fetch() {
+    local options=$1
+    local do_dryrun=$(abcli_option_int "$options" dryrun 0)
+    local do_upload=$(abcli_option_int "$options" upload $(abcli_not $do_dryrun))
+
+    local url=${2:-void}
+
+    local object_name=$(abcli_clarify_object $3 web-fetch-$(abcli_string_timestamp_short))
+
+    abcli_log "fetching $url -> $object_name ..."
+
+    abcli_eval dryrun=$do_dryrun \
+        python3 -m blue_assistant.web \
+        fetch \
+        --url $url \
+        --object_name $object_name \
+        "${@:4}"
+    [[ $? -ne 0 ]] && return 1
+
+    [[ "$do_upload" == 1 ]] &&
+        abcli_upload - $object_name
+
+    return 0
+}

{blue_assistant-4.221.1 → blue_assistant-4.243.1}/blue_assistant/README.py

@@ -21,7 +21,7 @@ items = README.Items(
         },
         {
             "name": "orbital-data-explorer",
-            "url": "./blue_assistant/script/repository/orbital_data_explorer/README.md",
+            "url": "./blue_assistant/script/repository/orbital_data_explorer",
             "marquee": "https://github.com/kamangir/assets/blob/main/blue-assistant/orbital-data-explorer.png?raw=true",
             "description": "Access to the [Orbital Data Explorer](https://ode.rsl.wustl.edu/), through AI. ⏸️",
         },
@@ -47,8 +47,13 @@ def build():
                 "path": "..",
             },
             {"path": "script/repository/blue_amo"},
-            {"path": "script/repository/orbital_data_explorer"},
+            #
+            {"path": "script/repository/orbital_data_explorer/docs/round-1.md"},
+            {"path": "script/repository/orbital_data_explorer/docs"},
+            #
             {"path": "script/repository/hue/docs/round-1.md"},
             {"path": "script/repository/hue/docs"},
+            #
+            {"path": "web/"},
         ]
     )

{blue_assistant-4.221.1 → blue_assistant-4.243.1}/blue_assistant/__init__.py

@@ -4,7 +4,7 @@ ICON = "🧠"
 
 DESCRIPTION = f"{ICON} An AI Assistant."
 
-VERSION = "4.221.1"
+VERSION = "4.243.1"
 
 REPO_NAME = "blue-assistant"
 

{blue_assistant-4.221.1 → blue_assistant-4.243.1}/blue_assistant/config.env

@@ -5,4 +5,6 @@ BLUE_ASSISTANT_IMAGE_DEFAULT_MODEL=dall-e-3
 BLUE_ASSISTANT_IMAGE_DEFAULT_QUALITY=standard
 BLUE_ASSISTANT_IMAGE_DEFAULT_SIZE=1024x1024
 
-HUE_BRIDGE_IP_ADDRESS=192.168.1.95
+HUE_BRIDGE_IP_ADDRESS=192.168.1.95
+HUE_TEST_DEFAULT_INTERVAL=0.01
+HUE_MAX_SATURATION=254

{blue_assistant-4.221.1 → blue_assistant-4.243.1}/blue_assistant/env.py

@@ -17,3 +17,5 @@ BLUE_ASSISTANT_IMAGE_DEFAULT_SIZE = get_env("BLUE_ASSISTANT_IMAGE_DEFAULT_SIZE")
 HUE_BRIDGE_IP_ADDRESS = get_env("HUE_BRIDGE_IP_ADDRESS")
 
 HUE_BRIDGE_USERNAME = get_env("HUE_BRIDGE_USERNAME")
+HUE_TEST_DEFAULT_INTERVAL = get_env("HUE_TEST_DEFAULT_INTERVAL", 0.01)
+HUE_MAX_SATURATION = get_env("HUE_MAX_SATURATION", 254)

{blue_assistant-4.221.1 → blue_assistant-4.243.1}/blue_assistant/help/hue.py

@@ -1,4 +1,5 @@
 from typing import List
+import cv2
 
 from blue_options.terminal import show_usage, xtra
 
@@ -62,7 +63,7 @@ def help_set(
         f"[--username <{env.HUE_BRIDGE_USERNAME}>]",
         "[--light_id <light_id>]",
         "[--hue <65535>]",
-        "[--saturation <254>]",
+        f"[--saturation <{env.HUE_MAX_SATURATION}>]",
         "[--verbose 1]",
     ]
 
@@ -88,8 +89,8 @@ def help_test(
         f"[--bridge_ip <{env.HUE_BRIDGE_IP_ADDRESS}>]",
         f"[--username <{env.HUE_BRIDGE_USERNAME}>]",
         "[--light_id all | <light_id>]",
-        "[--interval <1>]",
-        "[--hue <65535>]",
+        f"[--interval <{env.HUE_TEST_DEFAULT_INTERVAL:.2f}>]",
+        f"[--colormap <{cv2.COLORMAP_HOT}>]",
         "[--verbose 1]",
     ]
 
@@ -101,6 +102,9 @@ def help_test(
         ]
         + args,
         "test hue.",
+        {
+            "colormap: https://docs.opencv.org/4.x/d3/d50/group__imgproc__colormap.html": [],
+        },
         mono=mono,
     )
 

{blue_assistant-4.221.1 → blue_assistant-4.243.1}/blue_assistant/help/web.py

@@ -15,19 +15,38 @@ def help_crawl(
 
     return show_usage(
         [
-            "@assistant",
-            "web",
+            "@web",
             "crawl",
             f"[{options}]",
             "<url-1>+<url-2>+<url-3>",
             "[-|<object-name>]",
         ]
         + args,
-        "crawl the web.",
+        "crawl the urls.",
+        mono=mono,
+    )
+
+
+def help_fetch(
+    tokens: List[str],
+    mono: bool,
+) -> str:
+    options = xtra("dryrun,~upload", mono=mono)
+
+    return show_usage(
+        [
+            "@web",
+            "fetch",
+            f"[{options}]",
+            "<url>",
+            "[-|<object-name>]",
+        ],
+        "fetch <url>.",
         mono=mono,
     )
 
 
 help_functions = {
     "crawl": help_crawl,
+    "fetch": help_fetch,
 }

{blue_assistant-4.221.1 → blue_assistant-4.243.1}/blue_assistant/script/repository/generic/classes.py

@@ -35,9 +35,7 @@ class GenericScript(BaseScript):
             node_name=node_name,
         )
 
-    def run(
-        self,
-    ) -> bool:
+    def run(self) -> bool:
         if not super().run():
             return False
 
@@ -50,6 +48,11 @@ class GenericScript(BaseScript):
                 if self.nodes[node_name].get("completed", False):
                     continue
 
+                if not self.nodes[node_name].get("runnable", True):
+                    logger.info(f"Not runnable, skipped: {node_name}.")
+                    self.nodes[node_name]["completed"] = True
+                    continue
+
                 pending_dependencies = [
                     node_name_
                     for node_name_ in self.G.successors(node_name)

{blue_assistant-4.221.1 → blue_assistant-4.243.1}/blue_assistant/script/repository/hue/__main__.py

@@ -1,11 +1,12 @@
 import argparse
+import cv2
 
 from blueness import module
 from blueness.argparse.generic import sys_exit
 
 from blue_assistant import NAME
 from blue_assistant import env
-from blue_assistant.script.repository.hue.functions import (
+from blue_assistant.script.repository.hue.api import (
     create_user,
     list_lights,
     set_light_color,
@@ -47,8 +48,8 @@ parser.add_argument(
 parser.add_argument(
     "--saturation",
     type=int,
-    default=254,
-    help="0 to 254",
+    default=env.HUE_MAX_SATURATION,
+    help=f"0 to {env.HUE_MAX_SATURATION}",
 )
 parser.add_argument(
     "--verbose",
@@ -59,9 +60,15 @@ parser.add_argument(
 parser.add_argument(
     "--interval",
     type=float,
-    default=0.1,
+    default=env.HUE_TEST_DEFAULT_INTERVAL,
     help="in seconds",
 )
+parser.add_argument(
+    "--colormap",
+    type=int,
+    default=cv2.COLORMAP_HOT,
+    help="//docs.opencv.org/4.x/d3/d50/group__imgproc__colormap.html",
+)
 args = parser.parse_args()
 
 success = False
@@ -92,7 +99,7 @@ elif args.task == "test":
         username=args.username,
         light_id=args.light_id,
         interval=args.interval,
-        hue=args.hue,
+        colormap=args.colormap,
         verbose=args.verbose == 1,
     )
 else:

blue_assistant-4.221.1/blue_assistant/script/repository/hue/functions.py → blue_assistant-4.243.1/blue_assistant/script/repository/hue/api.py

@@ -2,11 +2,14 @@ from typing import Tuple, Dict, List
 import requests
 from time import sleep
 from tqdm import tqdm
+import cv2
+import random
 
 from blueness import module
 
 from blue_assistant import NAME
 from blue_assistant import env
+from blue_assistant.script.repository.hue.colors import get_hue_values
 from blue_assistant.logger import logger
 
 NAME = module.name(__file__, NAME)
@@ -118,9 +121,9 @@ def set_light_color(
 
 
 def test(
-    hue: int,
+    colormap: int = cv2.COLORMAP_HOT,
     light_id: str = "all",
-    interval: float = 0.1,
+    interval: float = env.HUE_TEST_DEFAULT_INTERVAL,
     bridge_ip: str = env.HUE_BRIDGE_IP_ADDRESS,
     username: str = env.HUE_BRIDGE_USERNAME,
     verbose: bool = False,
@@ -129,12 +132,12 @@ def test(
         light_id = "all"
 
     logger.info(
-        "{}.test({}@{}:{}) @ hue=0x{:x}, interval={} s".format(
+        "{}.test({}@{}:{}) @ colormap #{}, interval={} s".format(
             NAME,
             username,
             bridge_ip,
             light_id,
-            hue,
+            colormap,
             interval,
         )
     )
@@ -152,20 +155,32 @@ def test(
     else:
         list_of_lights = [light_id]
 
-    saturation = 0
-    while True:
-        for light_id_ in tqdm(list_of_lights):
-            set_light_color(
-                light_id=light_id_,
-                hue=hue,
-                saturation=saturation,
-                bridge_ip=bridge_ip,
-                username=username,
-                verbose=verbose,
-            )
-
-            sleep(interval)
-
-        saturation = 254 - saturation
+    list_of_hue_values = get_hue_values(
+        colormap=colormap,
+        length=len(list_of_lights),
+    )
+    list_of_hue_values = list_of_hue_values + list_of_hue_values
+
+    hue_offset: int = 0
+    try:
+        while True:
+            logger.info(f"hue_offset={hue_offset}")
+
+            for light_index in tqdm(range(len(list_of_lights))):
+                set_light_color(
+                    light_id=list_of_lights[light_index],
+                    hue=list_of_hue_values[hue_offset + light_index],
+                    saturation=random.randint(1, env.HUE_MAX_SATURATION),
+                    bridge_ip=bridge_ip,
+                    username=username,
+                    verbose=verbose,
+                )
+
+                sleep(interval)
+
+            hue_offset = (hue_offset + 1) % len(list_of_lights)
+
+    except KeyboardInterrupt:
+        logger.info("Ctrl+C detected.")
 
     return True

blue_assistant-4.243.1/blue_assistant/script/repository/hue/colors.py

@@ -0,0 +1,21 @@
+from typing import List
+import numpy as np
+import cv2
+
+
+# https://chatgpt.com/c/67d4c06c-2168-8005-9d8b-4b6c9848957e
+def get_hue_values(
+    colormap: int = cv2.COLORMAP_HOT,
+    length: int = 255,
+) -> List[int]:
+    # Create a gradient from 0 to 255
+    gradient = np.linspace(0, 255, length).astype("uint8")
+    gradient = np.repeat(gradient[np.newaxis, :], 1, axis=0)
+
+    # Apply the colormap
+    color_mapped_image = cv2.applyColorMap(gradient, colormap)
+
+    # Convert BGR to HSV
+    hsv_image = cv2.cvtColor(color_mapped_image, cv2.COLOR_BGR2HSV)
+
+    return (hsv_image[0, :, 0] * (65535 / 179)).astype(int).tolist()

{blue_assistant-4.221.1 → blue_assistant-4.243.1}/blue_assistant/script/repository/orbital_data_explorer/actions/researching_the_questions.py

@@ -5,7 +5,7 @@ from tqdm import tqdm
 from openai_commands.text_generation import api
 
 from blue_assistant import NAME
-from blue_assistant.web.crawl import crawl_list_of_urls
+from blue_assistant.web.functions import crawl_list_of_urls
 from blue_assistant.script.repository.base.classes import BaseScript
 from blue_assistant.logger import logger
 

{blue_assistant-4.221.1 → blue_assistant-4.243.1}/blue_assistant/web/__main__.py

@@ -2,9 +2,11 @@ import argparse
 
 from blueness import module
 from blueness.argparse.generic import sys_exit
+from blue_options.logger import log_dict
+from blue_objects.metadata import post_to_object
 
 from blue_assistant import NAME
-from blue_assistant.web.crawl import crawl_list_of_urls
+from blue_assistant.web.functions import crawl_list_of_urls, fetch_links_and_text
 from blue_assistant.logger import logger
 
 NAME = module.name(__file__, NAME)
@@ -13,7 +15,7 @@ parser = argparse.ArgumentParser(NAME)
 parser.add_argument(
     "task",
     type=str,
-    help="crawl",
+    help="crawl | fetch",
 )
 parser.add_argument(
     "--max_iterations",
@@ -26,6 +28,10 @@ parser.add_argument(
     default=0,
     help="0 | 1",
 )
+parser.add_argument(
+    "--url",
+    type=str,
+)
 parser.add_argument(
     "--seed_urls",
     type=str,
@@ -38,22 +44,34 @@ args = parser.parse_args()
 
 success = False
 if args.task == "crawl":
-    success = True
-
-    output = crawl_list_of_urls(
+    dict_of_urls = crawl_list_of_urls(
         seed_urls=args.seed_urls.split("+"),
         object_name=args.object_name,
         max_iterations=args.max_iterations,
     )
 
     if args.verbose == 1:
-        logger.info(f"{len(output)} url(s)")
-        for index, (url, content) in enumerate(output.items()):
-            logger.info(f"#{index: 4} - {url}: {content[:200]}...\n")
-            if index > 10:
-                logger.info("...")
-                break
+        log_dict(logger, dict_of_urls, "url(s)")
 
+    success = post_to_object(
+        args.object_name,
+        NAME.replace(".", "-"),
+        dict_of_urls,
+    )
+elif args.task == "fetch":
+    links, text = fetch_links_and_text(
+        url=args.url,
+        verbose=True,
+    )
+
+    success = post_to_object(
+        args.object_name,
+        NAME.replace(".", "-"),
+        {
+            "links": list(links),
+            "text": text,
+        },
+    )
 else:
     success = None
 

blue_assistant-4.221.1/blue_assistant/web/crawl.py → blue_assistant-4.243.1/blue_assistant/web/functions.py

@@ -1,18 +1,25 @@
-from typing import List, Dict, Set
+from typing import List, Dict, Set, Tuple
 import requests
-from bs4 import BeautifulSoup
-from urllib.parse import urljoin, urlparse
+from bs4 import BeautifulSoup, XMLParsedAsHTMLWarning
+from urllib.parse import urljoin
+import re
+import warnings
 
 from blueness import module
-
+from blue_options.logger import log_long_text, log_list
 
 from blue_assistant import NAME
 from blue_assistant.logger import logger
 
+warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
+
 NAME = module.name(__file__, NAME)
 
 
-def fetch_links_and_content(url, base_url, original_path):
+def fetch_links_and_text(
+    url: str,
+    verbose: bool = False,
+) -> Tuple[List[str], str]:
     try:
         response = requests.get(url, timeout=5)
         response.raise_for_status()
@@ -23,17 +30,27 @@ def fetch_links_and_content(url, base_url, original_path):
     links = set()
 
     for a_tag in soup.find_all("a", href=True):
-        full_url = urljoin(base_url, a_tag["href"])
-        parsed_url = urlparse(full_url)
+        a_url = urljoin(url, a_tag["href"])
+
+        if a_url.startswith(url):
+            logger.info(f"+= {a_url}")
+            links.add(a_url)
+            continue
 
-        # Ensure link is from the same domain and in the same directory tree
-        if parsed_url.netloc == urlparse(
-            base_url
-        ).netloc and parsed_url.path.startswith(original_path):
-            links.add(full_url)
+        logger.info(f"ignored: {a_url}")
 
     plain_text = soup.get_text(separator=" ", strip=True)
 
+    # remove non-ASCII characters
+    plain_text = re.sub(r"[^\x20-\x7E]+", "", plain_text)
+    for thing in ["\r", "\n", "\t"]:
+        plain_text = plain_text.replace(thing, " ")
+    plain_text = re.sub(r"\s+", " ", plain_text).strip()
+
+    if verbose:
+        log_list(logger, list(links), "link(s)")
+        log_long_text(logger, plain_text)
+
     return links, plain_text
 
 
@@ -41,6 +58,7 @@ def crawl_list_of_urls(
     seed_urls: List[str],
     object_name: str,
     max_iterations: int = 10,
+    verbose: bool = False,
 ) -> Dict[str, str]:
     logger.info(
         "{}.crawl_list_of_urls({}): {} -> {}".format(
@@ -53,23 +71,20 @@ def crawl_list_of_urls(
 
     visited: Dict[str, str] = {}
     queue: Set[str] = set(seed_urls)
-    base_url = urlparse(seed_urls[0]).scheme + "://" + urlparse(seed_urls[0]).netloc
-    original_path = (
-        urlparse(seed_urls[0]).path.rsplit("/", 1)[0] + "/"
-    )  # Get base directory
 
     iteration: int = 0
     while queue:
-        current_url = queue.pop()
-        if current_url in visited:
+        url = queue.pop()
+        if url in visited:
             continue
 
-        logger.info(f"🔗  {current_url} ...")
-        new_links, content = fetch_links_and_content(
-            current_url, base_url, original_path
+        logger.info(f"🔗  {url} ...")
+        url_links, url_text = fetch_links_and_text(
+            url=url,
+            verbose=verbose,
         )
-        visited[current_url] = content
-        queue.update(new_links - visited.keys())
+        visited[url] = url_text
+        queue.update(url_links - visited.keys())
 
         iteration += 1
         if max_iterations != -1 and iteration >= max_iterations:

{blue_assistant-4.221.1 → blue_assistant-4.243.1/blue_assistant.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: blue_assistant
-Version: 4.221.1
+Version: 4.243.1
 Summary: 🧠 An AI Assistant.
 Home-page: https://github.com/kamangir/blue-assistant
 Author: Arash Abadpour (Kamangir)
@@ -56,10 +56,14 @@ graph LR
     assistant_script_list["@assistant<br>script<br>list"]
     assistant_script_run["@assistant<br>script<br>run -<br>&lt;script&gt;<br>&lt;object-name&gt;"]
 
-    assistant_web_crawl["@assistant<br>web<br>crawl -<br>&lt;url-1&gt;+&lt;url-2&gt;+&lt;url-3&gt;<br>&lt;object-name&gt;"]
+    web_crawl["@web<br>crawl -<br>&lt;url-1&gt;+&lt;url-2&gt;<br>&lt;object-name&gt;"]
+
+    web_fetch["@web<br>fetch -<br>&lt;url&gt;<br>&lt;object-name&gt;"]
 
     script["📜 script"]:::folder
     url["🔗 url"]:::folder
+    url2["🔗 url"]:::folder
+    url3["🔗 url"]:::folder
     object["📂 object"]:::folder
 
 
@@ -69,9 +73,13 @@ graph LR
     object --> assistant_script_run
     assistant_script_run --> object
 
-    url --> assistant_web_crawl
-    assistant_web_crawl --> object
+    url --> web_crawl
+    url2 --> web_crawl
+    web_crawl --> url3
+    web_crawl --> object
 
+    url --> web_fetch
+    web_fetch --> object
 
     bridge_ip["🔗 bridge_ip"]:::folder
     hue_username["🔗 hue_username"]:::folder
@@ -105,11 +113,15 @@ graph LR
 |   |   |
 | --- | --- |
 | [`hue`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/hue) [![image](https://github.com/kamangir/assets/raw/main/blue-assistant/20250314_143702.jpg?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/hue) "send a color command to the Hue LED lights in my apartment." | [`blue-amo`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/blue_amo/README.md) [![image](https://github.com/kamangir/assets/raw/main/blue-amo-2025-02-03-nswnx6/stitching_the_frames-2.png?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/blue_amo/README.md) A story developed and visualized, by AI. |
-| [`orbital-data-explorer`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/orbital_data_explorer/README.md) [![image](https://github.com/kamangir/assets/blob/main/blue-assistant/orbital-data-explorer.png?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/orbital_data_explorer/README.md) Access to the [Orbital Data Explorer](https://ode.rsl.wustl.edu/), through AI. ⏸️ |  |
+| [`orbital-data-explorer`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/orbital_data_explorer) [![image](https://github.com/kamangir/assets/blob/main/blue-assistant/orbital-data-explorer.png?raw=true)](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/script/repository/orbital_data_explorer) Access to the [Orbital Data Explorer](https://ode.rsl.wustl.edu/), through AI. ⏸️ |  |
+
+---
+
+Also home to [`@web`](https://raw.githubusercontent.com/kamangir/blue-assistant/main/blue_assistant/web/)
 
 ---
 
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.234.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.221.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.236.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.243.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.221.1 → blue_assistant-4.243.1}/blue_assistant.egg-info/SOURCES.txt

@@ -38,7 +38,9 @@ blue_assistant/.abcli/tests/script_list.sh
 blue_assistant/.abcli/tests/script_run.sh
 blue_assistant/.abcli/tests/version.sh
 blue_assistant/.abcli/tests/web_crawl.sh
+blue_assistant/.abcli/tests/web_fetch.sh
 blue_assistant/.abcli/web/crawl.sh
+blue_assistant/.abcli/web/fetch.sh
 blue_assistant/help/__init__.py
 blue_assistant/help/__main__.py
 blue_assistant/help/functions.py
@@ -64,12 +66,13 @@ blue_assistant/script/repository/generic/__init__.py
 blue_assistant/script/repository/generic/classes.py
 blue_assistant/script/repository/hue/__init__.py
 blue_assistant/script/repository/hue/__main__.py
+blue_assistant/script/repository/hue/api.py
 blue_assistant/script/repository/hue/classes.py
-blue_assistant/script/repository/hue/functions.py
+blue_assistant/script/repository/hue/colors.py
 blue_assistant/script/repository/orbital_data_explorer/__init__.py
 blue_assistant/script/repository/orbital_data_explorer/classes.py
 blue_assistant/script/repository/orbital_data_explorer/actions/__init__.py
 blue_assistant/script/repository/orbital_data_explorer/actions/researching_the_questions.py
 blue_assistant/web/__init__.py
 blue_assistant/web/__main__.py
-blue_assistant/web/crawl.py
+blue_assistant/web/functions.py