blue-assistant 4.142.1__py3-none-any.whl → 4.154.1__py3-none-any.whl

blue_assistant/.abcli/tests/help.sh

@@ -23,6 +23,9 @@ function test_blue_assistant_help() {
         "@assistant script list" \
         "@assistant script run" \
         \
+        "@assistant web" \
+        "@assistant web crawl" \
+        \
         "blue_assistant"; do
         abcli_eval ,$options \
             abcli_help $module

blue_assistant/.abcli/tests/web_crawl.sh

@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+function test_blue_assistant_web_crawl() {
+    local options=$1
+
+    abcli_eval ,$options \
+        blue_assistant_web_crawl \
+        ~upload \
+        https://ode.rsl.wustl.edu/+https://oderest.rsl.wustl.edu/ \
+        test_blue_assistant_web_crawl-$(abcli_string_timestamp_short) \
+        --max_iterations 3
+}

blue_assistant/.abcli/web/crawl.sh

@@ -0,0 +1,26 @@
+#! /usr/bin/env bash
+
+function blue_assistant_web_crawl() {
+    local options=$1
+    local do_dryrun=$(abcli_option_int "$options" dryrun 0)
+    local do_upload=$(abcli_option_int "$options" upload $(abcli_not $do_dryrun))
+
+    local seed_urls=${2:-void}
+
+    local object_name=$(abcli_clarify_object $3 web-crawl-$(abcli_string_timestamp_short))
+
+    abcli_log "crawling $seed_urls -> $object_name ..."
+
+    abcli_eval dryrun=$do_dryrun \
+        python3 -m blue_assistant.web \
+        crawl \
+        --seed_urls $seed_urls \
+        --object_name $object_name \
+        "${@:4}"
+    [[ $? -ne 0 ]] && return 1
+
+    [[ "$do_upload" == 1 ]] &&
+        abcli_upload - $object_name
+
+    return 0
+}

blue_assistant/.abcli/web.sh

@@ -0,0 +1,15 @@
+#! /usr/bin/env bash
+
+function blue_assistant_web() {
+    local task=$(abcli_unpack_keyword $1 help)
+
+    local function_name=blue_assistant_web_$task
+    if [[ $(type -t $function_name) == "function" ]]; then
+        $function_name "${@:2}"
+        return
+    fi
+
+    python3 -m blue_assistant.web "$@"
+}
+
+abcli_source_caller_suffix_path /web

blue_assistant/__init__.py

@@ -4,7 +4,7 @@ ICON = "🧠"
 
 DESCRIPTION = f"{ICON} An AI Assistant."
 
-VERSION = "4.142.1"
+VERSION = "4.154.1"
 
 REPO_NAME = "blue-assistant"
 

blue_assistant/help/functions.py

@@ -5,6 +5,7 @@ from abcli.help.generic import help_functions as generic_help_functions
 
 from blue_assistant import ALIAS
 from blue_assistant.help.script import help_functions as help_script
+from blue_assistant.help.web import help_functions as help_web
 
 
 def help_browse(
@@ -30,5 +31,6 @@ help_functions.update(
     {
         "browse": help_browse,
         "script": help_script,
+        "web": help_web,
     }
 )

blue_assistant/help/web.py

@@ -0,0 +1,33 @@
+from typing import List
+
+from blue_options.terminal import show_usage, xtra
+
+
+def help_crawl(
+    tokens: List[str],
+    mono: bool,
+) -> str:
+    options = xtra("dryrun,~upload", mono=mono)
+
+    args = [
+        "[--max_iterations <100000>]",
+    ]
+
+    return show_usage(
+        [
+            "@assistant",
+            "web",
+            "crawl",
+            f"[{options}]",
+            "<url-1>+<url-2>+<url-3>",
+            "[-|<object-name>]",
+        ]
+        + args,
+        "crawl the web.",
+        mono=mono,
+    )
+
+
+help_functions = {
+    "crawl": help_crawl,
+}

blue_assistant/script/actions/__init__.py

@@ -4,13 +4,11 @@ from blue_assistant.script.repository.base.classes import BaseScript
 from blue_assistant.script.actions.generic import generic_action
 from blue_assistant.script.actions.generate_image import generate_image
 from blue_assistant.script.actions.generate_text import generate_text
-from blue_assistant.script.actions.skip import skip_action
 from blue_assistant.logger import logger
 
 
 dict_of_actions: Dict[str, Callable[[BaseScript, str], bool]] = {
-    "generic_action": generic_action,
+    "generic": generic_action,
     "generate_image": generate_image,
     "generate_text": generate_text,
-    "skip": skip_action,
 }

blue_assistant/script/repository/blue_amo/classes.py

@@ -44,7 +44,7 @@ class BlueAmoScript(GenericScript):
 
         reduce_node = "stitching_the_frames"
         self.G.add_node(reduce_node)
-        self.nodes[reduce_node] = {"action": "skip"}
+        self.nodes[reduce_node] = {"action": "generic"}
 
         for index in range(self.vars["frame_count"]):
             node_name = f"generating_frame_{index+1:03d}"

blue_assistant/web/__main__.py

@@ -0,0 +1,60 @@
+import argparse
+
+from blueness import module
+from blueness.argparse.generic import sys_exit
+
+from blue_assistant import NAME
+from blue_assistant.web.crawl import crawl_list_of_urls
+from blue_assistant.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+parser = argparse.ArgumentParser(NAME)
+parser.add_argument(
+    "task",
+    type=str,
+    help="crawl",
+)
+parser.add_argument(
+    "--max_iterations",
+    type=int,
+    default=10,
+)
+parser.add_argument(
+    "--verbose",
+    type=int,
+    default=0,
+    help="0 | 1",
+)
+parser.add_argument(
+    "--seed_urls",
+    type=str,
+)
+parser.add_argument(
+    "--object_name",
+    type=str,
+)
+args = parser.parse_args()
+
+success = False
+if args.task == "crawl":
+    success = True
+
+    output = crawl_list_of_urls(
+        seed_urls=args.seed_urls.split("+"),
+        object_name=args.object_name,
+        max_iterations=args.max_iterations,
+    )
+
+    if args.verbose == 1:
+        logger.info(f"{len(output)} url(s)")
+        for index, (url, content) in enumerate(output.items()):
+            logger.info(f"#{index: 4} - {url}: {content[:200]}...\n")
+            if index > 10:
+                logger.info("...")
+                break
+
+else:
+    success = None
+
+sys_exit(logger, NAME, args.task, success)

blue_assistant/web/crawl.py

@@ -0,0 +1,82 @@
+from typing import List, Dict, Set
+import requests
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+
+from blueness import module
+
+
+from blue_assistant import NAME
+from blue_assistant.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+def fetch_links_and_content(url, base_url, original_path):
+    try:
+        response = requests.get(url, timeout=5)
+        response.raise_for_status()
+    except requests.RequestException:
+        return set(), ""
+
+    soup = BeautifulSoup(response.text, "html.parser")
+    links = set()
+
+    for a_tag in soup.find_all("a", href=True):
+        full_url = urljoin(base_url, a_tag["href"])
+        parsed_url = urlparse(full_url)
+
+        # Ensure link is from the same domain and in the same directory tree
+        if parsed_url.netloc == urlparse(
+            base_url
+        ).netloc and parsed_url.path.startswith(original_path):
+            links.add(full_url)
+
+    plain_text = soup.get_text(separator=" ", strip=True)
+
+    return links, plain_text
+
+
+def crawl_list_of_urls(
+    seed_urls: List[str],
+    object_name: str,
+    max_iterations: int = 10,
+) -> Dict[str, str]:
+    logger.info(
+        "{}.crawl_list_of_urls({}): {} -> {}".format(
+            NAME,
+            len(seed_urls),
+            ", ".join(seed_urls),
+            object_name,
+        )
+    )
+
+    visited: Dict[str, str] = {}
+    queue: Set[str] = set(seed_urls)
+    base_url = urlparse(seed_urls[0]).scheme + "://" + urlparse(seed_urls[0]).netloc
+    original_path = (
+        urlparse(seed_urls[0]).path.rsplit("/", 1)[0] + "/"
+    )  # Get base directory
+
+    iteration: int = 0
+    while queue:
+        current_url = queue.pop()
+        if current_url in visited:
+            continue
+
+        logger.info(f"🔗  {current_url} ...")
+        new_links, content = fetch_links_and_content(
+            current_url, base_url, original_path
+        )
+        visited[current_url] = content
+        queue.update(new_links - visited.keys())
+
+        iteration += 1
+        if max_iterations != -1 and iteration >= max_iterations:
+            logger.warning(f"max iteration of {max_iterations} reached.")
+            break
+
+    if queue:
+        logger.warning(f"queue: {len(queue)}")
+
+    return visited

{blue_assistant-4.142.1.dist-info → blue_assistant-4.154.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: blue_assistant
-Version: 4.142.1
+Version: 4.154.1
 Summary: 🧠 An AI Assistant.
 Home-page: https://github.com/kamangir/blue-assistant
 Author: Arash Abadpour (Kamangir)
@@ -30,6 +30,7 @@ Requires-Dist: pylint
 Requires-Dist: pytest
 Requires-Dist: python-dotenv[cli]
 Requires-Dist: tqdm
+Requires-Dist: beautifulsoup4
 Requires-Dist: networkx
 Requires-Dist: pydot
 Dynamic: author
@@ -44,7 +45,7 @@ Dynamic: summary
 
 # 🧠 blue-assistant
 
-🧠 `@assistant` runs AI scripts; DAGs that combine deterministic and AI operations, such as below,
+🧠 `@assistant` runs AI scripts; DAGs that combine deterministic and AI operations.
 
 |   |   |
 | --- | --- |
@@ -59,7 +60,10 @@ graph LR
     assistant_script_list["@assistant<br>script<br>list"]
     assistant_script_run["@assistant<br>script<br>run -<br>&lt;script&gt;<br>&lt;object-name&gt;"]
 
+    assistant_web_crawl["@assistant<br>web<br>crawl -<br>&lt;url-1&gt;+&lt;url-2&gt;+&lt;url-3&gt;<br>&lt;object-name&gt;"]
+
     script["📜 script"]:::folder
+    url["🔗 url"]:::folder
     object["📂 object"]:::folder
 
     script --> assistant_script_list
@@ -68,6 +72,9 @@ graph LR
     object --> assistant_script_run
     assistant_script_run --> object
 
+    url --> assistant_web_crawl
+    assistant_web_crawl --> object
+
     classDef folder fill:#999,stroke:#333,stroke-width:2px;
 ```
 
@@ -76,4 +83,4 @@ graph LR
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.207.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.142.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.207.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.154.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.142.1.dist-info → blue_assistant-4.154.1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 blue_assistant/README.py,sha256=aB7xoxE3SpI_QbXzJRqh9R-QSWFJSfX3JG-ZY5nfDD4,1503
-blue_assistant/__init__.py,sha256=dR1gRga50p7ATJUZqjToyki00k-bUoT6VR9PDwi-MQ0,311
+blue_assistant/__init__.py,sha256=sTU1-9vRRZrZ5UxhvlCspgS_j1hjIvM-MWr2wTGXBpo,311
 blue_assistant/__main__.py,sha256=URtal70XZc0--3FDTYWcLtnGOqBYjMX9gt-L1k8hDXI,361
 blue_assistant/config.env,sha256=PUR0GHwzmpm7xNdWHjIv8DzlMAesOAa1NEnYDsIUTs8,211
 blue_assistant/env.py,sha256=e3YPPpMCrKoTcOX8QmzndEwDnM-nC-hFTSUvhnaKpSc,737
@@ -14,30 +14,33 @@ blue_assistant/.abcli/alias.sh,sha256=MCJzXaDnX1QMllWsZJJkDePBYt1nY9ZWa3o4msfGD2
 blue_assistant/.abcli/blue_assistant.sh,sha256=plLTQQerVmfb_SNlOkv0MEaQCF7YdsOHzCq0M3FWT4c,239
 blue_assistant/.abcli/browse.sh,sha256=qZ_RK_WnsjmF-hfWKiMEOnnv22QtZh9HQ0VFJUbP6aI,294
 blue_assistant/.abcli/script.sh,sha256=XIkY4eZyFPKLi_mLoPMbnq76E4K1GG3xxha8VYJC2zI,356
+blue_assistant/.abcli/web.sh,sha256=1HA3u6umxwZro_CnxD2H9_WABypeSfMU-X2ncFMnU8c,344
 blue_assistant/.abcli/script/list.sh,sha256=2lcVfqDfZP50NszF8o5YCo3TrJKeDc_qo7MTAF3XTGw,131
 blue_assistant/.abcli/script/run.sh,sha256=kSXmyM9NUj2X2orSGyu5t_P5frG-gyumbRq-xqF692c,911
 blue_assistant/.abcli/tests/README.sh,sha256=Qs0YUxVB1OZZ70Nqw2kT1LKXeUnC5-XfQRMfqb8Cbwg,152
-blue_assistant/.abcli/tests/help.sh,sha256=mENB9ZNBEEPmIs9tp8WkQW3dq75_US7EI7_-d4IJQpo,724
+blue_assistant/.abcli/tests/help.sh,sha256=h7_Z8yIp7-twQugKxdmsn9g77aHRgj8hj1ZA7jY5jns,794
 blue_assistant/.abcli/tests/script_list.sh,sha256=OVOwWO9wR0eeDZTM6uub-eTKbz3eswU3vEUPWXcK-gQ,178
 blue_assistant/.abcli/tests/script_run.sh,sha256=vfmK8sjkMfSQPwCacQppiL6inMbvQP7nci7qLppFSL0,769
 blue_assistant/.abcli/tests/version.sh,sha256=oR2rvYR8zi-0VDPIdPJsmsmWwYaamT8dmNTqUh3-8Gw,154
+blue_assistant/.abcli/tests/web_crawl.sh,sha256=nQkynpAA0rthCpEmuMAwYrEuBuBBFQ7B1mrplt3VMhY,336
+blue_assistant/.abcli/web/crawl.sh,sha256=M9YoKKJBKZT2OtmFPvRCSSKpiAq0zyacRAVZ6s7i3FM,698
 blue_assistant/help/__init__.py,sha256=ajz1GSNU9xYVrFEDSz6Xwg7amWQ_yvW75tQa1ZvRIWc,3
 blue_assistant/help/__main__.py,sha256=cVejR7OpoWPg0qLbm-PZf5TuJS27x49jzfiyCLyzEns,241
-blue_assistant/help/functions.py,sha256=9WsmXGMN-R7sqlkGLK0nY90Peg8Gah4rIu75QbLhImo,689
+blue_assistant/help/functions.py,sha256=6WgIQr4pGZZ_ljNvW0PAnj8QmqoX5hoKgIY2dk2M59U,777
 blue_assistant/help/script.py,sha256=tofv49tIBqoH8ed9hDCFHqzWaXmyyPofvqElk2n976w,1121
+blue_assistant/help/web.py,sha256=yOtA1IdQLBbhx_0Q1xLNwzdHozXdpALMYtwS5X3889o,575
 blue_assistant/script/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 blue_assistant/script/__main__.py,sha256=eOSOo5yYTPMwIXZ0GkuWkmOcsDWrZtHvClyJizXSk2w,1657
 blue_assistant/script/load.py,sha256=JsDY9T3HTM9vXngvKsA0Mt_erxAnRR_jI62-JhrOBMU,831
-blue_assistant/script/actions/__init__.py,sha256=W0PisTP1H0RRqivzwiaXlN-kvE3m1xlJtn5WGZ-PFtg,625
+blue_assistant/script/actions/__init__.py,sha256=tiNL9HT1ql-Ut6CPYuj6B29PXQPuf-B-z8dJcksWACY,534
 blue_assistant/script/actions/generate_image.py,sha256=PgvOspDV8n2M7ZmgVOdZzJwQ1tnJNJ6V8gV94P74ksA,1336
 blue_assistant/script/actions/generate_text.py,sha256=nD30y8hoFbYoDT2QsrhKCvUfJUrOubiiLl3OjabtASg,1881
 blue_assistant/script/actions/generic.py,sha256=ET1RaKcUABM8HdIv8JecSpUFasYqmwHacL-5LjF-8NM,355
-blue_assistant/script/actions/skip.py,sha256=G9gbGBbOLiCqcsmEUobdoxkB6wohFYmyi1arQGorZSg,352
 blue_assistant/script/repository/__init__.py,sha256=zVI3cubRqM9H6WgF0EUP9idILVLCumPFmJgKPM7iVlM,604
 blue_assistant/script/repository/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 blue_assistant/script/repository/base/classes.py,sha256=-G81it3Beb3fbhE5H6g0BPJwvZKn3qELiBaqRWXP4OY,3717
 blue_assistant/script/repository/blue_amo/__init__.py,sha256=WjL9GIlN-DBnbUMJ8O_FxTp0rcVGlsIS3H9YtXEefTk,76
-blue_assistant/script/repository/blue_amo/classes.py,sha256=vWQ_qMdJ2LmpEjDGFnSxOMZPd34yj-DX4UUTnx5aMtY,2082
+blue_assistant/script/repository/blue_amo/classes.py,sha256=Rl_UX67AlfhVEhv8tfgdWKItOxI3cUfiGsN12-k8sWI,2085
 blue_assistant/script/repository/blue_amo/actions/__init__.py,sha256=je2S21KvYB3QkbABs71parwUh8MCh2mdlNZfLx_QuDg,430
 blue_assistant/script/repository/blue_amo/actions/slicing_into_frames.py,sha256=79SI7_69FKKLeX_jHlfXnUWGtG4Sj7sBJOfeFwK8I9U,1201
 blue_assistant/script/repository/blue_amo/actions/stitching_the_frames.py,sha256=mbXriat6deEAmuo5Y1ValySnUXDENR7TZS_3nVPlQ6M,3622
@@ -49,8 +52,11 @@ blue_assistant/script/repository/orbital_data_explorer/__init__.py,sha256=yy5FtC
 blue_assistant/script/repository/orbital_data_explorer/classes.py,sha256=i4cVCR6ge8FhipPs-H1HZ_5xcok4mzxrqwRr_hLz_UI,657
 blue_assistant/script/repository/orbital_data_explorer/actions/__init__.py,sha256=RcrFUAwnvhuwNh3gC65w9G26vd_cIa7LV1lFvGFcigk,370
 blue_assistant/script/repository/orbital_data_explorer/actions/researching_the_questions.py,sha256=Wb8sU9UmdRRhHUhNj7_VQ74_Bio20xGajzeGz4HJqZQ,369
-blue_assistant-4.142.1.dist-info/LICENSE,sha256=ogEPNDSH0_dhiv_lT3ifVIdgIzHAqNA_SemnxUfPBJk,7048
-blue_assistant-4.142.1.dist-info/METADATA,sha256=ifohOQ1zpJdsKWyUNHlE3Yke1DbnTbFon2EfpwilPRg,3578
-blue_assistant-4.142.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-blue_assistant-4.142.1.dist-info/top_level.txt,sha256=ud0BkBbdOVze13bNqHuhZj1rwCztaBtDf5ChEYzASOs,15
-blue_assistant-4.142.1.dist-info/RECORD,,
+blue_assistant/web/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+blue_assistant/web/__main__.py,sha256=wriAODGqb_WRaXxq5KF5ZJp1pOzuTMlVElDafgZo5f0,1258
+blue_assistant/web/crawl.py,sha256=Sxkxg9b0IsQxL0ecEAcb13JwPOmwkwUie8XUeDChf1c,2249
+blue_assistant-4.154.1.dist-info/LICENSE,sha256=ogEPNDSH0_dhiv_lT3ifVIdgIzHAqNA_SemnxUfPBJk,7048
+blue_assistant-4.154.1.dist-info/METADATA,sha256=64601-HQMnP9wWL1BNdavvvJacKJq8NV_stNLRn6LEk,3815
+blue_assistant-4.154.1.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+blue_assistant-4.154.1.dist-info/top_level.txt,sha256=ud0BkBbdOVze13bNqHuhZj1rwCztaBtDf5ChEYzASOs,15
+blue_assistant-4.154.1.dist-info/RECORD,,

blue_assistant/script/actions/skip.py

@@ -1,15 +0,0 @@
-from blueness import module
-
-from blue_assistant import NAME
-from blue_assistant.script.repository.base.classes import BaseScript
-from blue_assistant.logger import logger
-
-NAME = module.name(__file__, NAME)
-
-
-def skip_action(
-    script: BaseScript,
-    node_name: str,
-) -> bool:
-    logger.info(f"{NAME}: {script} @ {node_name} ...")
-    return True