blue-assistant 4.307.1__tar.gz → 4.319.1__tar.gz

{blue_assistant-4.307.1/blue_assistant.egg-info → blue_assistant-4.319.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: blue_assistant
-Version: 4.307.1
+Version: 4.319.1
 Summary: 🧠 An AI Assistant.
 Home-page: https://github.com/kamangir/blue-assistant
 Author: Arash Abadpour (Kamangir)
@@ -124,4 +124,4 @@ Also home to [`@web`](https://raw.githubusercontent.com/kamangir/blue-assistant/
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.307.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.319.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.307.1 → blue_assistant-4.319.1}/README.md

@@ -79,4 +79,4 @@ Also home to [`@web`](./blue_assistant/web/)
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.307.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.319.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.307.1 → blue_assistant-4.319.1}/blue_assistant/__init__.py

@@ -4,7 +4,7 @@ ICON = "🧠"
 
 DESCRIPTION = f"{ICON} An AI Assistant."
 
-VERSION = "4.307.1"
+VERSION = "4.319.1"
 
 REPO_NAME = "blue-assistant"
 

{blue_assistant-4.307.1 → blue_assistant-4.319.1}/blue_assistant/help/script.py

@@ -37,6 +37,7 @@ def help_run(
     args = [
         "[--test_mode 1]",
         "[--verbose 1]",
+        "[--runnable <~node_1,~node_2>]",
     ]
 
     return show_usage(

{blue_assistant-4.307.1 → blue_assistant-4.319.1}/blue_assistant/script/__main__.py

@@ -48,6 +48,13 @@ parser.add_argument(
     default=1,
     help="0 | 1",
 )
+parser.add_argument(
+    "--runnable",
+    type=str,
+    default="",
+    help="~node_1,~node_2",
+)
+
 args = parser.parse_args()
 
 delim = " " if args.delim == "space" else args.delim
@@ -70,7 +77,9 @@ elif args.task == "run":
     )
 
     if success:
-        success = script.run()
+        success = script.run(
+            runnable=args.runnable,
+        )
 else:
     success = None
 

{blue_assistant-4.307.1 → blue_assistant-4.319.1}/blue_assistant/script/actions/web_crawl.py

@@ -4,6 +4,7 @@ from blue_options.logger import log_list
 
 from blue_assistant import NAME
 from blue_assistant.web.crawl import crawl_list_of_urls
+from blue_assistant.web.functions import normalize_url
 from blue_assistant.script.repository.base.root import RootScript
 from blue_assistant.logger import logger
 
@@ -32,16 +33,16 @@ def web_crawl(
     if seed_url_var_name not in script.vars:
         logger.error(f"{node_name}: {seed_url_var_name}: seed_urls not found in vars.")
         return False
-    seed_urls = script.vars[seed_url_var_name]
+    seed_urls = list({normalize_url(url) for url in script.vars[seed_url_var_name]})
     log_list(logger, "using", seed_urls, "seed url(s)")
 
-    success, _ = crawl_list_of_urls(
+    success, crawl_cache = crawl_list_of_urls(
         seed_urls=seed_urls,
         object_name=script.object_name,
         max_iterations=script.nodes[node_name]["max_iterations"],
         cache_prefix=node_name,
     )
 
-    script.nodes[node_name]["output"] = success
+    script.nodes[node_name]["output"] = crawl_cache
 
     return success

blue_assistant-4.319.1/blue_assistant/script/repository/base/classes.py

@@ -0,0 +1,21 @@
+import copy
+
+from blueness import module
+from blue_objects import file, path
+
+from blue_assistant import NAME
+from blue_assistant.script.repository.base.root import RootScript
+from blue_assistant.script.actions import dict_of_actions
+from blue_assistant.logger import logger
+
+
+NAME = module.name(__file__, NAME)
+
+
+class BaseScript(RootScript):
+    name = path.name(file.path(__file__))
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.dict_of_actions = copy.deepcopy(dict_of_actions)

blue_assistant-4.307.1/blue_assistant/script/repository/base/classes.py → blue_assistant-4.319.1/blue_assistant/script/repository/base/root.py

@@ -1,25 +1,18 @@
-from typing import Dict, List, Callable
 import os
-import networkx as nx
+from typing import Dict, List
 from functools import reduce
+import networkx as nx
 from tqdm import tqdm
-import copy
 
-from blueness import module
-from blue_objects import file, objects, path
+from blue_options.options import Options
+from blue_objects import file, path, objects
 from blue_objects.metadata import post_to_object
 from blueflow.workflow import dot_file
 
-from blue_assistant import NAME
-from blue_assistant.script.repository.base.root import RootScript
-from blue_assistant.script.actions import dict_of_actions
 from blue_assistant.logger import logger
 
 
-NAME = module.name(__file__, NAME)
-
-
-class BaseScript(RootScript):
+class RootScript:
     name = path.name(file.path(__file__))
 
     def __init__(
@@ -28,13 +21,15 @@ class BaseScript(RootScript):
         test_mode: bool = False,
         verbose: bool = False,
     ):
+        self.nodes_changed = False
+
         self.object_name = object_name
 
         self.test_mode = test_mode
 
         self.verbose = verbose
 
-        self.dict_of_actions = copy.deepcopy(dict_of_actions)
+        self.dict_of_actions = {}
 
         metadata_filename = os.path.join(
             file.path(__file__),
@@ -115,7 +110,9 @@ class BaseScript(RootScript):
             text = text.replace(f":::{var_name}", str(var_value))
 
         for node_name, node in self.nodes.items():
-            text = text.replace(f":::{node_name}", node.get("output", ""))
+            node_output = node.get("output", "")
+            if isinstance(node_output, str):
+                text = text.replace(f":::{node_name}", node_output)
 
         return text
 
@@ -174,14 +171,26 @@ class BaseScript(RootScript):
         logger.error(f"{action_name}: action not found.")
         return False
 
-    def run(self) -> bool:
+    def run(
+        self,
+        runnable: str = "",
+    ) -> bool:
         logger.info(f"{self.name}.run -> {self.object_name}")
 
+        if runnable:
+            logger.info(f"applying runnables: {runnable}")
+            runnable_options = Options(runnable)
+            for node_name, node_is_runnable in runnable_options.items():
+                logger.info(f"{node_name}.runnable={node_is_runnable}")
+                self.nodes[node_name]["runnable"] = node_is_runnable
+
         success: bool = True
         while (
             not all(self.nodes[node].get("completed", False) for node in self.nodes)
             and success
         ):
+            self.nodes_changed = False
+
             for node_name in tqdm(self.nodes):
                 if self.nodes[node_name].get("completed", False):
                     continue
@@ -212,6 +221,22 @@ class BaseScript(RootScript):
 
                 self.nodes[node_name]["completed"] = True
 
+                cache_filename = self.nodes[node_name].get("cache", "")
+                if cache_filename:
+                    if not file.save_text(
+                        objects.path_of(
+                            object_name=self.object_name,
+                            filename=cache_filename,
+                        ),
+                        [self.nodes[node_name].get("output", "")],
+                    ):
+                        success = False
+                        break
+
+                if self.nodes_changed:
+                    logger.info("🪄  nodes changed.")
+                    break
+
         if not post_to_object(
             self.object_name,
             "output",

{blue_assistant-4.307.1 → blue_assistant-4.319.1}/blue_assistant/script/repository/blue_amo/classes.py

@@ -14,17 +14,8 @@ NAME = module.name(__file__, NAME)
 class BlueAmoScript(BaseScript):
     name = path.name(file.path(__file__))
 
-    def __init__(
-        self,
-        object_name: str,
-        test_mode: bool = False,
-        verbose: bool = False,
-    ):
-        super().__init__(
-            object_name=object_name,
-            test_mode=test_mode,
-            verbose=verbose,
-        )
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
 
         self.dict_of_actions.update(dict_of_actions)
 

blue_assistant-4.319.1/blue_assistant/script/repository/orbital_data_explorer/actions/__init__.py

@@ -0,0 +1,11 @@
+from typing import Dict, Callable
+
+from blue_assistant.script.repository.base.classes import BaseScript
+from blue_assistant.script.repository.orbital_data_explorer.actions import (
+    expanding_the_extractions,
+)
+
+
+dict_of_actions: Dict[str, Callable[[BaseScript, str], bool]] = {
+    "expanding_the_extractions": expanding_the_extractions.expanding_the_extractions,
+}

blue_assistant-4.319.1/blue_assistant/script/repository/orbital_data_explorer/actions/expanding_the_extractions.py

@@ -0,0 +1,109 @@
+import copy
+from typing import Dict
+
+from blueness import module
+from blue_options.logger import log_dict, log_list
+from blue_objects import file, objects
+from blue_objects.metadata import get_from_object
+
+from blue_assistant import NAME
+from blue_assistant.script.repository.base.classes import BaseScript
+from blue_assistant.web.functions import url_to_filename
+from blue_assistant.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+def expanding_the_extractions(
+    script: BaseScript,
+    node_name: str,
+) -> bool:
+    map_node_name = "extraction"
+
+    crawl_cache: Dict[str, str] = get_from_object(
+        script.object_name,
+        "web_crawl_cache",
+        {},
+    )
+    log_dict(logger, "using", crawl_cache, "crawled url(s)")
+
+    list_of_urls = [
+        url
+        for url, content_type in crawl_cache.items()
+        if "html" in content_type
+        and not file.exists(
+            objects.path_of(
+                object_name=script.object_name,
+                filename="{}_cache/{}.txt".format(
+                    map_node_name,
+                    url_to_filename(url),
+                ),
+            )
+        )
+    ]
+    log_list(logger, "using", list_of_urls, "crawled unextracted html(s).")
+
+    max_nodes = min(
+        len(list_of_urls),
+        script.nodes[node_name]["max_nodes"],
+    )
+    logger.info(
+        "{}: expanding {} X {}...".format(
+            NAME,
+            map_node_name,
+            max_nodes,
+        )
+    )
+
+    map_node = script.nodes[map_node_name]
+    del script.nodes[map_node_name]
+    script.G.remove_node(map_node_name)
+
+    reduce_node_name = "generating_summary"
+    for index in range(max_nodes):
+        url = list_of_urls[index]
+        index_node_name = f"{map_node_name}_{index+1:03d}"
+
+        success, url_content = file.load_yaml(
+            filename=objects.path_of(
+                object_name=script.object_name,
+                filename="web_crawl_cache/{}.yaml".format(
+                    url_to_filename(url),
+                ),
+            ),
+        )
+        if not success:
+            logger.warning(f"{url}: failed to load url content.")
+            continue
+        if "text" not in url_content:
+            logger.warning(f"{url}: no text found in url content.")
+            continue
+
+        logger.info(f"{url} -{map_node_name}-> {index_node_name}")
+
+        script.nodes[index_node_name] = copy.deepcopy(map_node)
+
+        script.nodes[index_node_name]["prompt"] = map_node["prompt"].replace(
+            ":::url_content",
+            url_content["text"],
+        )
+
+        script.nodes[index_node_name]["url"] = url
+        script.nodes[index_node_name]["cache"] = "{}_cache/{}.txt".format(
+            map_node_name,
+            url_to_filename(url),
+        )
+
+        script.G.add_node(index_node_name)
+        script.G.add_edge(
+            index_node_name,
+            node_name,
+        )
+        script.G.add_edge(
+            reduce_node_name,
+            index_node_name,
+        )
+
+    script.nodes_changed = True
+
+    return script.save_graph()

blue_assistant-4.319.1/blue_assistant/script/repository/orbital_data_explorer/classes.py

@@ -0,0 +1,15 @@
+from blue_objects import file, path
+
+from blue_assistant.script.repository.base.classes import BaseScript
+from blue_assistant.script.repository.orbital_data_explorer.actions import (
+    dict_of_actions,
+)
+
+
+class OrbitalDataExplorerScript(BaseScript):
+    name = path.name(file.path(__file__))
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.dict_of_actions.update(dict_of_actions)

{blue_assistant-4.307.1 → blue_assistant-4.319.1}/blue_assistant/web/crawl.py

@@ -35,7 +35,7 @@ def crawl_list_of_urls(
             seed_urls
             + get_from_object(
                 object_name,
-                f"{cache_prefix}_crawl_queue",
+                f"{cache_prefix}_queue",
                 [],
             )
         )
@@ -44,7 +44,7 @@ def crawl_list_of_urls(
 
     crawl_cache: Dict[str, str] = get_from_object(
         object_name,
-        f"{cache_prefix}_crawl_cache",
+        f"{cache_prefix}_cache",
         {},
     )
     log_dict(logger, "loaded cache:", crawl_cache, "url(s)")
@@ -75,7 +75,7 @@ def crawl_list_of_urls(
             if not file.save_yaml(
                 filename=objects.path_of(
                     object_name=object_name,
-                    filename="{}_crawl_cache/{}.yaml".format(
+                    filename="{}_cache/{}.yaml".format(
                         cache_prefix,
                         url_to_filename(url),
                     ),
@@ -108,12 +108,12 @@ def crawl_list_of_urls(
     if not (
         post_to_object(
             object_name,
-            f"{cache_prefix}_crawl_cache",
+            f"{cache_prefix}_cache",
             crawl_cache,
         )
         and post_to_object(
             object_name,
-            f"{cache_prefix}_crawl_queue",
+            f"{cache_prefix}_queue",
             queue,
         )
     ):

{blue_assistant-4.307.1 → blue_assistant-4.319.1}/blue_assistant/web/fetch.py

@@ -9,6 +9,7 @@ from blueness import module
 from blue_options.logger import log_long_text, log_list
 
 from blue_assistant import NAME
+from blue_assistant.web.functions import normalize_url
 from blue_assistant.logger import logger
 
 warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
@@ -49,8 +50,7 @@ def fetch_links_and_text(
         for a_tag in soup.find_all("a", href=True):
             a_url = urljoin(url, a_tag["href"])
 
-            if "#" in a_url:
-                a_url = a_url.split("#", 1)[0]
+            a_url = normalize_url(a_url)
 
             if a_url.startswith(url):
                 if url not in list_of_urls:

{blue_assistant-4.307.1 → blue_assistant-4.319.1}/blue_assistant/web/functions.py

@@ -1,4 +1,15 @@
 import re
+from urllib.parse import urlparse, urlunparse
+
+
+# https://chatgpt.com/c/67d733a0-4be4-8005-bf52-fb9ba32487c2
+def normalize_url(url: str) -> str:
+    return urlunparse(
+        urlparse(url)._replace(
+            query="",
+            fragment="",
+        ),
+    )
 
 
 def url_to_filename(

{blue_assistant-4.307.1 → blue_assistant-4.319.1/blue_assistant.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: blue_assistant
-Version: 4.307.1
+Version: 4.319.1
 Summary: 🧠 An AI Assistant.
 Home-page: https://github.com/kamangir/blue-assistant
 Author: Arash Abadpour (Kamangir)
@@ -124,4 +124,4 @@ Also home to [`@web`](https://raw.githubusercontent.com/kamangir/blue-assistant/
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.307.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.319.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.307.1 → blue_assistant-4.319.1}/blue_assistant.egg-info/SOURCES.txt

@@ -71,6 +71,8 @@ blue_assistant/script/repository/hue/classes.py
 blue_assistant/script/repository/hue/colors.py
 blue_assistant/script/repository/orbital_data_explorer/__init__.py
 blue_assistant/script/repository/orbital_data_explorer/classes.py
+blue_assistant/script/repository/orbital_data_explorer/actions/__init__.py
+blue_assistant/script/repository/orbital_data_explorer/actions/expanding_the_extractions.py
 blue_assistant/web/__init__.py
 blue_assistant/web/__main__.py
 blue_assistant/web/crawl.py

{blue_assistant-4.307.1 → blue_assistant-4.319.1}/setup.py

@@ -18,6 +18,7 @@ setup(
         f"{NAME}.script.repository.blue_amo.actions",
         f"{NAME}.script.repository.hue",
         f"{NAME}.script.repository.orbital_data_explorer",
+        f"{NAME}.script.repository.orbital_data_explorer.actions",
         f"{NAME}.web",
     ],
     include_package_data=True,

blue_assistant-4.307.1/blue_assistant/script/repository/base/root.py

@@ -1,5 +0,0 @@
-from blue_objects import file, objects, path
-
-
-class RootScript:
-    name = path.name(file.path(__file__))

blue_assistant-4.307.1/blue_assistant/script/repository/orbital_data_explorer/classes.py

@@ -1,7 +0,0 @@
-from blue_objects import file, path
-
-from blue_assistant.script.repository.base.classes import BaseScript
-
-
-class OrbitalDataExplorerScript(BaseScript):
-    name = path.name(file.path(__file__))