blue-assistant 4.142.1__tar.gz → 4.154.1__tar.gz

{blue_assistant-4.142.1/blue_assistant.egg-info → blue_assistant-4.154.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: blue_assistant
-Version: 4.142.1
+Version: 4.154.1
 Summary: 🧠 An AI Assistant.
 Home-page: https://github.com/kamangir/blue-assistant
 Author: Arash Abadpour (Kamangir)
@@ -30,6 +30,7 @@ Requires-Dist: pylint
 Requires-Dist: pytest
 Requires-Dist: python-dotenv[cli]
 Requires-Dist: tqdm
+Requires-Dist: beautifulsoup4
 Requires-Dist: networkx
 Requires-Dist: pydot
 Dynamic: author
@@ -44,7 +45,7 @@ Dynamic: summary
 
 # 🧠 blue-assistant
 
-🧠 `@assistant` runs AI scripts; DAGs that combine deterministic and AI operations, such as below,
+🧠 `@assistant` runs AI scripts; DAGs that combine deterministic and AI operations.
 
 |   |   |
 | --- | --- |
@@ -59,7 +60,10 @@ graph LR
     assistant_script_list["@assistant<br>script<br>list"]
     assistant_script_run["@assistant<br>script<br>run -<br>&lt;script&gt;<br>&lt;object-name&gt;"]
 
+    assistant_web_crawl["@assistant<br>web<br>crawl -<br>&lt;url-1&gt;+&lt;url-2&gt;+&lt;url-3&gt;<br>&lt;object-name&gt;"]
+
     script["📜 script"]:::folder
+    url["🔗 url"]:::folder
     object["📂 object"]:::folder
 
     script --> assistant_script_list
@@ -68,6 +72,9 @@ graph LR
     object --> assistant_script_run
     assistant_script_run --> object
 
+    url --> assistant_web_crawl
+    assistant_web_crawl --> object
+
     classDef folder fill:#999,stroke:#333,stroke-width:2px;
 ```
 
@@ -76,4 +83,4 @@ graph LR
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.207.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.142.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.207.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.154.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.142.1 → blue_assistant-4.154.1}/README.md

@@ -1,6 +1,6 @@
 # 🧠 blue-assistant
 
-🧠 `@assistant` runs AI scripts; DAGs that combine deterministic and AI operations, such as below,
+🧠 `@assistant` runs AI scripts; DAGs that combine deterministic and AI operations.
 
 |   |   |
 | --- | --- |
@@ -15,7 +15,10 @@ graph LR
     assistant_script_list["@assistant<br>script<br>list"]
     assistant_script_run["@assistant<br>script<br>run -<br>&lt;script&gt;<br>&lt;object-name&gt;"]
 
+    assistant_web_crawl["@assistant<br>web<br>crawl -<br>&lt;url-1&gt;+&lt;url-2&gt;+&lt;url-3&gt;<br>&lt;object-name&gt;"]
+
     script["📜 script"]:::folder
+    url["🔗 url"]:::folder
     object["📂 object"]:::folder
 
     script --> assistant_script_list
@@ -24,6 +27,9 @@ graph LR
     object --> assistant_script_run
     assistant_script_run --> object
 
+    url --> assistant_web_crawl
+    assistant_web_crawl --> object
+
     classDef folder fill:#999,stroke:#333,stroke-width:2px;
 ```
 
@@ -32,4 +38,4 @@ graph LR
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.207.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.142.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.207.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.154.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.142.1 → blue_assistant-4.154.1}/blue_assistant/.abcli/tests/help.sh

@@ -23,6 +23,9 @@ function test_blue_assistant_help() {
         "@assistant script list" \
         "@assistant script run" \
         \
+        "@assistant web" \
+        "@assistant web crawl" \
+        \
         "blue_assistant"; do
         abcli_eval ,$options \
             abcli_help $module

blue_assistant-4.154.1/blue_assistant/.abcli/tests/web_crawl.sh

@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+function test_blue_assistant_web_crawl() {
+    local options=$1
+
+    abcli_eval ,$options \
+        blue_assistant_web_crawl \
+        ~upload \
+        https://ode.rsl.wustl.edu/+https://oderest.rsl.wustl.edu/ \
+        test_blue_assistant_web_crawl-$(abcli_string_timestamp_short) \
+        --max_iterations 3
+}

blue_assistant-4.154.1/blue_assistant/.abcli/web/crawl.sh

@@ -0,0 +1,26 @@
+#! /usr/bin/env bash
+
+function blue_assistant_web_crawl() {
+    local options=$1
+    local do_dryrun=$(abcli_option_int "$options" dryrun 0)
+    local do_upload=$(abcli_option_int "$options" upload $(abcli_not $do_dryrun))
+
+    local seed_urls=${2:-void}
+
+    local object_name=$(abcli_clarify_object $3 web-crawl-$(abcli_string_timestamp_short))
+
+    abcli_log "crawling $seed_urls -> $object_name ..."
+
+    abcli_eval dryrun=$do_dryrun \
+        python3 -m blue_assistant.web \
+        crawl \
+        --seed_urls $seed_urls \
+        --object_name $object_name \
+        "${@:4}"
+    [[ $? -ne 0 ]] && return 1
+
+    [[ "$do_upload" == 1 ]] &&
+        abcli_upload - $object_name
+
+    return 0
+}

blue_assistant-4.154.1/blue_assistant/.abcli/web.sh

@@ -0,0 +1,15 @@
+#! /usr/bin/env bash
+
+function blue_assistant_web() {
+    local task=$(abcli_unpack_keyword $1 help)
+
+    local function_name=blue_assistant_web_$task
+    if [[ $(type -t $function_name) == "function" ]]; then
+        $function_name "${@:2}"
+        return
+    fi
+
+    python3 -m blue_assistant.web "$@"
+}
+
+abcli_source_caller_suffix_path /web

{blue_assistant-4.142.1 → blue_assistant-4.154.1}/blue_assistant/__init__.py

@@ -4,7 +4,7 @@ ICON = "🧠"
 
 DESCRIPTION = f"{ICON} An AI Assistant."
 
-VERSION = "4.142.1"
+VERSION = "4.154.1"
 
 REPO_NAME = "blue-assistant"
 

{blue_assistant-4.142.1 → blue_assistant-4.154.1}/blue_assistant/help/functions.py

@@ -5,6 +5,7 @@ from abcli.help.generic import help_functions as generic_help_functions
 
 from blue_assistant import ALIAS
 from blue_assistant.help.script import help_functions as help_script
+from blue_assistant.help.web import help_functions as help_web
 
 
 def help_browse(
@@ -30,5 +31,6 @@ help_functions.update(
     {
         "browse": help_browse,
         "script": help_script,
+        "web": help_web,
     }
 )

blue_assistant-4.154.1/blue_assistant/help/web.py

@@ -0,0 +1,33 @@
+from typing import List
+
+from blue_options.terminal import show_usage, xtra
+
+
+def help_crawl(
+    tokens: List[str],
+    mono: bool,
+) -> str:
+    options = xtra("dryrun,~upload", mono=mono)
+
+    args = [
+        "[--max_iterations <100000>]",
+    ]
+
+    return show_usage(
+        [
+            "@assistant",
+            "web",
+            "crawl",
+            f"[{options}]",
+            "<url-1>+<url-2>+<url-3>",
+            "[-|<object-name>]",
+        ]
+        + args,
+        "crawl the web.",
+        mono=mono,
+    )
+
+
+help_functions = {
+    "crawl": help_crawl,
+}

{blue_assistant-4.142.1 → blue_assistant-4.154.1}/blue_assistant/script/actions/__init__.py

@@ -4,13 +4,11 @@ from blue_assistant.script.repository.base.classes import BaseScript
 from blue_assistant.script.actions.generic import generic_action
 from blue_assistant.script.actions.generate_image import generate_image
 from blue_assistant.script.actions.generate_text import generate_text
-from blue_assistant.script.actions.skip import skip_action
 from blue_assistant.logger import logger
 
 
 dict_of_actions: Dict[str, Callable[[BaseScript, str], bool]] = {
-    "generic_action": generic_action,
+    "generic": generic_action,
     "generate_image": generate_image,
     "generate_text": generate_text,
-    "skip": skip_action,
 }

{blue_assistant-4.142.1 → blue_assistant-4.154.1}/blue_assistant/script/repository/blue_amo/classes.py

@@ -44,7 +44,7 @@ class BlueAmoScript(GenericScript):
 
         reduce_node = "stitching_the_frames"
         self.G.add_node(reduce_node)
-        self.nodes[reduce_node] = {"action": "skip"}
+        self.nodes[reduce_node] = {"action": "generic"}
 
         for index in range(self.vars["frame_count"]):
             node_name = f"generating_frame_{index+1:03d}"

blue_assistant-4.154.1/blue_assistant/web/__main__.py

@@ -0,0 +1,60 @@
+import argparse
+
+from blueness import module
+from blueness.argparse.generic import sys_exit
+
+from blue_assistant import NAME
+from blue_assistant.web.crawl import crawl_list_of_urls
+from blue_assistant.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+parser = argparse.ArgumentParser(NAME)
+parser.add_argument(
+    "task",
+    type=str,
+    help="crawl",
+)
+parser.add_argument(
+    "--max_iterations",
+    type=int,
+    default=10,
+)
+parser.add_argument(
+    "--verbose",
+    type=int,
+    default=0,
+    help="0 | 1",
+)
+parser.add_argument(
+    "--seed_urls",
+    type=str,
+)
+parser.add_argument(
+    "--object_name",
+    type=str,
+)
+args = parser.parse_args()
+
+success = False
+if args.task == "crawl":
+    success = True
+
+    output = crawl_list_of_urls(
+        seed_urls=args.seed_urls.split("+"),
+        object_name=args.object_name,
+        max_iterations=args.max_iterations,
+    )
+
+    if args.verbose == 1:
+        logger.info(f"{len(output)} url(s)")
+        for index, (url, content) in enumerate(output.items()):
+            logger.info(f"#{index: 4} - {url}: {content[:200]}...\n")
+            if index > 10:
+                logger.info("...")
+                break
+
+else:
+    success = None
+
+sys_exit(logger, NAME, args.task, success)

blue_assistant-4.154.1/blue_assistant/web/crawl.py

@@ -0,0 +1,82 @@
+from typing import List, Dict, Set
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+
+from blueness import module
+
+
+from blue_assistant import NAME
+from blue_assistant.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+def fetch_links_and_content(url, base_url, original_path):
+    try:
+        response = requests.get(url, timeout=5)
+        response.raise_for_status()
+    except requests.RequestException:
+        return set(), ""
+
+    soup = BeautifulSoup(response.text, "html.parser")
+    links = set()
+
+    for a_tag in soup.find_all("a", href=True):
+        full_url = urljoin(base_url, a_tag["href"])
+        parsed_url = urlparse(full_url)
+
+        # Ensure link is from the same domain and in the same directory tree
+        if parsed_url.netloc == urlparse(
+            base_url
+        ).netloc and parsed_url.path.startswith(original_path):
+            links.add(full_url)
+
+    plain_text = soup.get_text(separator=" ", strip=True)
+
+    return links, plain_text
+
+
+def crawl_list_of_urls(
+    seed_urls: List[str],
+    object_name: str,
+    max_iterations: int = 10,
+) -> Dict[str, str]:
+    logger.info(
+        "{}.crawl_list_of_urls({}): {} -> {}".format(
+            NAME,
+            len(seed_urls),
+            ", ".join(seed_urls),
+            object_name,
+        )
+    )
+
+    visited: Dict[str, str] = {}
+    queue: Set[str] = set(seed_urls)
+    base_url = urlparse(seed_urls[0]).scheme + "://" + urlparse(seed_urls[0]).netloc
+    original_path = (
+        urlparse(seed_urls[0]).path.rsplit("/", 1)[0] + "/"
+    )  # Get base directory
+
+    iteration: int = 0
+    while queue:
+        current_url = queue.pop()
+        if current_url in visited:
+            continue
+
+        logger.info(f"🔗  {current_url} ...")
+        new_links, content = fetch_links_and_content(
+            current_url, base_url, original_path
+        )
+        visited[current_url] = content
+        queue.update(new_links - visited.keys())
+
+        iteration += 1
+        if max_iterations != -1 and iteration >= max_iterations:
+            logger.warning(f"max iteration of {max_iterations} reached.")
+            break
+
+    if queue:
+        logger.warning(f"queue: {len(queue)}")
+
+    return visited

{blue_assistant-4.142.1 → blue_assistant-4.154.1/blue_assistant.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: blue_assistant
-Version: 4.142.1
+Version: 4.154.1
 Summary: 🧠 An AI Assistant.
 Home-page: https://github.com/kamangir/blue-assistant
 Author: Arash Abadpour (Kamangir)
@@ -30,6 +30,7 @@ Requires-Dist: pylint
 Requires-Dist: pytest
 Requires-Dist: python-dotenv[cli]
 Requires-Dist: tqdm
+Requires-Dist: beautifulsoup4
 Requires-Dist: networkx
 Requires-Dist: pydot
 Dynamic: author
@@ -44,7 +45,7 @@ Dynamic: summary
 
 # 🧠 blue-assistant
 
-🧠 `@assistant` runs AI scripts; DAGs that combine deterministic and AI operations, such as below,
+🧠 `@assistant` runs AI scripts; DAGs that combine deterministic and AI operations.
 
 |   |   |
 | --- | --- |
@@ -59,7 +60,10 @@ graph LR
     assistant_script_list["@assistant<br>script<br>list"]
     assistant_script_run["@assistant<br>script<br>run -<br>&lt;script&gt;<br>&lt;object-name&gt;"]
 
+    assistant_web_crawl["@assistant<br>web<br>crawl -<br>&lt;url-1&gt;+&lt;url-2&gt;+&lt;url-3&gt;<br>&lt;object-name&gt;"]
+
     script["📜 script"]:::folder
+    url["🔗 url"]:::folder
     object["📂 object"]:::folder
 
     script --> assistant_script_list
@@ -68,6 +72,9 @@ graph LR
     object --> assistant_script_run
     assistant_script_run --> object
 
+    url --> assistant_web_crawl
+    assistant_web_crawl --> object
+
     classDef folder fill:#999,stroke:#333,stroke-width:2px;
 ```
 
@@ -76,4 +83,4 @@ graph LR
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.207.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.142.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.207.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.154.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.142.1 → blue_assistant-4.154.1}/blue_assistant.egg-info/SOURCES.txt

@@ -25,6 +25,7 @@ blue_assistant/.abcli/alias.sh
 blue_assistant/.abcli/blue_assistant.sh
 blue_assistant/.abcli/browse.sh
 blue_assistant/.abcli/script.sh
+blue_assistant/.abcli/web.sh
 blue_assistant/.abcli/script/list.sh
 blue_assistant/.abcli/script/run.sh
 blue_assistant/.abcli/tests/README.sh
@@ -32,10 +33,13 @@ blue_assistant/.abcli/tests/help.sh
 blue_assistant/.abcli/tests/script_list.sh
 blue_assistant/.abcli/tests/script_run.sh
 blue_assistant/.abcli/tests/version.sh
+blue_assistant/.abcli/tests/web_crawl.sh
+blue_assistant/.abcli/web/crawl.sh
 blue_assistant/help/__init__.py
 blue_assistant/help/__main__.py
 blue_assistant/help/functions.py
 blue_assistant/help/script.py
+blue_assistant/help/web.py
 blue_assistant/script/__init__.py
 blue_assistant/script/__main__.py
 blue_assistant/script/load.py
@@ -43,7 +47,6 @@ blue_assistant/script/actions/__init__.py
 blue_assistant/script/actions/generate_image.py
 blue_assistant/script/actions/generate_text.py
 blue_assistant/script/actions/generic.py
-blue_assistant/script/actions/skip.py
 blue_assistant/script/repository/__init__.py
 blue_assistant/script/repository/base/__init__.py
 blue_assistant/script/repository/base/classes.py
@@ -59,4 +62,7 @@ blue_assistant/script/repository/hue/classes.py
 blue_assistant/script/repository/orbital_data_explorer/__init__.py
 blue_assistant/script/repository/orbital_data_explorer/classes.py
 blue_assistant/script/repository/orbital_data_explorer/actions/__init__.py
-blue_assistant/script/repository/orbital_data_explorer/actions/researching_the_questions.py
+blue_assistant/script/repository/orbital_data_explorer/actions/researching_the_questions.py
+blue_assistant/web/__init__.py
+blue_assistant/web/__main__.py
+blue_assistant/web/crawl.py

{blue_assistant-4.142.1 → blue_assistant-4.154.1}/blue_assistant.egg-info/requires.txt

@@ -16,5 +16,6 @@ pylint
 pytest
 python-dotenv[cli]
 tqdm
+beautifulsoup4
 networkx
 pydot

{blue_assistant-4.142.1 → blue_assistant-4.154.1}/requirements.txt

@@ -17,6 +17,7 @@ pylint
 pytest
 python-dotenv[cli]
 tqdm
+beautifulsoup4
 
 networkx
 pydot

{blue_assistant-4.142.1 → blue_assistant-4.154.1}/setup.py

@@ -20,6 +20,7 @@ setup(
         f"{NAME}.script.repository.hue",
         f"{NAME}.script.repository.orbital_data_explorer",
         f"{NAME}.script.repository.orbital_data_explorer.actions",
+        f"{NAME}.web",
     ],
     include_package_data=True,
     package_data={

blue_assistant-4.142.1/blue_assistant/script/actions/skip.py

@@ -1,15 +0,0 @@
-from blueness import module
-
-from blue_assistant import NAME
-from blue_assistant.script.repository.base.classes import BaseScript
-from blue_assistant.logger import logger
-
-NAME = module.name(__file__, NAME)
-
-
-def skip_action(
-    script: BaseScript,
-    node_name: str,
-) -> bool:
-    logger.info(f"{NAME}: {script} @ {node_name} ...")
-    return True