blue-assistant 4.307.1__tar.gz → 4.325.1__tar.gz

{blue_assistant-4.307.1/blue_assistant.egg-info → blue_assistant-4.325.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: blue_assistant
-Version: 4.307.1
+Version: 4.325.1
 Summary: 🧠 An AI Assistant.
 Home-page: https://github.com/kamangir/blue-assistant
 Author: Arash Abadpour (Kamangir)
@@ -124,4 +124,4 @@ Also home to [`@web`](https://raw.githubusercontent.com/kamangir/blue-assistant/
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.307.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.325.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.307.1 → blue_assistant-4.325.1}/README.md

@@ -79,4 +79,4 @@ Also home to [`@web`](./blue_assistant/web/)
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.307.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.325.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.307.1 → blue_assistant-4.325.1}/blue_assistant/.abcli/script/run.sh

@@ -8,8 +8,9 @@ function blue_assistant_script_run() {
 
     local script_options=$2
     local script_name=$(abcli_option "$script_options" script base)
+    local script_version=$(abcli_option "$script_options" version base)
 
-    local object_name=$(abcli_clarify_object $3 $script_name-$(abcli_string_timestamp_short))
+    local object_name=$(abcli_clarify_object $3 $script_name-$script_version-$(abcli_string_timestamp_short))
     [[ "$do_download" == 1 ]] &&
         abcli_download - $object_name
 
@@ -19,6 +20,7 @@ function blue_assistant_script_run() {
         python3 -m blue_assistant.script \
         run \
         --script_name $script_name \
+        --script_version $script_version \
         --object_name $object_name \
         "${@:4}"
     [[ $? -ne 0 ]] && return 1

{blue_assistant-4.307.1 → blue_assistant-4.325.1}/blue_assistant/__init__.py

@@ -4,7 +4,7 @@ ICON = "🧠"
 
 DESCRIPTION = f"{ICON} An AI Assistant."
 
-VERSION = "4.307.1"
+VERSION = "4.325.1"
 
 REPO_NAME = "blue-assistant"
 

{blue_assistant-4.307.1 → blue_assistant-4.325.1}/blue_assistant/help/script.py

@@ -1,8 +1,10 @@
 from typing import List
+from functools import reduce
 
 from blue_options.terminal import show_usage, xtra
 
 from blue_assistant.script.repository import list_of_script_names
+from blue_assistant.script.repository.functions import get_script_versions
 
 
 def help_list(
@@ -32,13 +34,28 @@ def help_run(
 ) -> str:
     options = xtra("~download,dryrun,~upload", mono=mono)
 
-    script_options = "script=<script>"
+    script_options = "script=<name>,version=<version>"
 
     args = [
         "[--test_mode 1]",
         "[--verbose 1]",
+        "[--runnable <~node_1,~node_2>]",
     ]
 
+    def script_version_details(script_name: str) -> List[str]:
+        list_of_script_versions = get_script_versions(script_name)
+
+        return (
+            [
+                "{}: {}".format(
+                    script_name,
+                    " | ".join(list_of_script_versions),
+                )
+            ]
+            if list_of_script_versions
+            else []
+        )
+
     return show_usage(
         [
             "@assistant",
@@ -49,9 +66,16 @@ def help_run(
             "[-|<object-name>]",
         ]
         + args,
-        "run <object-name>.",
+        "run <script-name>/<script-version> in <object-name>.",
         {
-            "script: {}".format(" | ".join(list_of_script_names)): [],
+            "name: {}".format(" | ".join(list_of_script_names)): reduce(
+                lambda x, y: x + y,
+                [
+                    script_version_details(script_name)
+                    for script_name in list_of_script_names
+                ],
+                [],
+            )
         },
         mono=mono,
     )

{blue_assistant-4.307.1 → blue_assistant-4.325.1}/blue_assistant/script/__main__.py

@@ -21,6 +21,11 @@ parser.add_argument(
     type=str,
     help=" | ".join(list_of_script_names),
 )
+parser.add_argument(
+    "--script_version",
+    type=str,
+    default="base",
+)
 parser.add_argument(
     "--object_name",
     type=str,
@@ -48,6 +53,13 @@ parser.add_argument(
     default=1,
     help="0 | 1",
 )
+parser.add_argument(
+    "--runnable",
+    type=str,
+    default="",
+    help="~node_1,~node_2",
+)
+
 args = parser.parse_args()
 
 delim = " " if args.delim == "space" else args.delim
@@ -64,13 +76,16 @@ if args.task == "list":
 elif args.task == "run":
     success, script = load_script(
         script_name=args.script_name,
+        script_version=args.script_version,
         object_name=args.object_name,
         test_mode=args.test_mode == 1,
         verbose=args.verbose == 1,
     )
 
     if success:
-        success = script.run()
+        success = script.run(
+            runnable=args.runnable,
+        )
 else:
     success = None
 

{blue_assistant-4.307.1 → blue_assistant-4.325.1}/blue_assistant/script/actions/web_crawl.py

@@ -4,6 +4,7 @@ from blue_options.logger import log_list
 
 from blue_assistant import NAME
 from blue_assistant.web.crawl import crawl_list_of_urls
+from blue_assistant.web.functions import normalize_url
 from blue_assistant.script.repository.base.root import RootScript
 from blue_assistant.logger import logger
 
@@ -32,16 +33,16 @@ def web_crawl(
     if seed_url_var_name not in script.vars:
         logger.error(f"{node_name}: {seed_url_var_name}: seed_urls not found in vars.")
         return False
-    seed_urls = script.vars[seed_url_var_name]
+    seed_urls = list({normalize_url(url) for url in script.vars[seed_url_var_name]})
     log_list(logger, "using", seed_urls, "seed url(s)")
 
-    success, _ = crawl_list_of_urls(
+    success, crawl_cache = crawl_list_of_urls(
         seed_urls=seed_urls,
         object_name=script.object_name,
         max_iterations=script.nodes[node_name]["max_iterations"],
         cache_prefix=node_name,
     )
 
-    script.nodes[node_name]["output"] = success
+    script.nodes[node_name]["output"] = crawl_cache
 
     return success

{blue_assistant-4.307.1 → blue_assistant-4.325.1}/blue_assistant/script/load.py

@@ -8,8 +8,11 @@ from blue_assistant.logger import logger
 def load_script(
     script_name: str,
     object_name: str,
+    script_version: str = "base",
     test_mode: bool = False,
+    log: bool = True,
     verbose: bool = False,
+    save_graph: bool = True,
 ) -> Tuple[bool, BaseScript]:
     found: bool = False
     script_class: Type[BaseScript] = BaseScript
@@ -22,8 +25,13 @@ def load_script(
     if not found:
         logger.error(f"{script_name}: script not found.")
 
-    return found, script_class(
+    script = script_class(
+        script_version=script_version,
         object_name=object_name,
         test_mode=test_mode,
+        log=log,
         verbose=verbose,
+        save_graph=save_graph,
     )
+
+    return found and script.valid, script

blue_assistant-4.325.1/blue_assistant/script/repository/base/classes.py

@@ -0,0 +1,21 @@
+import copy
+
+from blueness import module
+from blue_objects import file, path
+
+from blue_assistant import NAME
+from blue_assistant.script.repository.base.root import RootScript
+from blue_assistant.script.actions import dict_of_actions
+from blue_assistant.logger import logger
+
+
+NAME = module.name(__file__, NAME)
+
+
+class BaseScript(RootScript):
+    name = path.name(file.path(__file__))
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.dict_of_actions = copy.deepcopy(dict_of_actions)

blue_assistant-4.307.1/blue_assistant/script/repository/base/classes.py → blue_assistant-4.325.1/blue_assistant/script/repository/base/root.py

@@ -1,40 +1,40 @@
-from typing import Dict, List, Callable
 import os
-import networkx as nx
+from typing import Dict, List
 from functools import reduce
+import networkx as nx
 from tqdm import tqdm
-import copy
 
-from blueness import module
-from blue_objects import file, objects, path
-from blue_objects.metadata import post_to_object
+from blue_options.options import Options
+from blue_objects import file, path, objects
+from blue_objects.metadata import post_to_object, get_from_object
 from blueflow.workflow import dot_file
 
-from blue_assistant import NAME
-from blue_assistant.script.repository.base.root import RootScript
-from blue_assistant.script.actions import dict_of_actions
 from blue_assistant.logger import logger
 
 
-NAME = module.name(__file__, NAME)
-
-
-class BaseScript(RootScript):
+class RootScript:
     name = path.name(file.path(__file__))
 
     def __init__(
         self,
         object_name: str,
+        script_version: str = "base",
         test_mode: bool = False,
+        log: bool = True,
         verbose: bool = False,
+        save_graph: bool = True,
     ):
+        self.valid = False
+
+        self.nodes_changed = False
+
         self.object_name = object_name
 
         self.test_mode = test_mode
 
         self.verbose = verbose
 
-        self.dict_of_actions = copy.deepcopy(dict_of_actions)
+        self.dict_of_actions = {}
 
         metadata_filename = os.path.join(
             file.path(__file__),
@@ -43,64 +43,133 @@ class BaseScript(RootScript):
         )
         self.metadata: Dict
         success, self.metadata = file.load_yaml(metadata_filename)
-        assert success, f"cannot load {self.name}/metadata.yaml"
+        if not success:
+            logger.error(f"cannot load {self.name}/metadata.yaml")
+            return
+
+        # validate and set
+        last_script_version = get_from_object(
+            object_name,
+            "script_version",
+            script_version,
+        )
+        if script_version != last_script_version:
+            logger.error(
+                "script version in {} is {} != {}".format(
+                    object_name,
+                    last_script_version,
+                    script_version,
+                )
+            )
+            return
+        if not post_to_object(
+            object_name,
+            "script_version",
+            script_version,
+        ):
+            return
+        if log:
+            logger.info(f"script version: {script_version}")
 
         self.metadata.setdefault("script", {})
-        assert isinstance(
-            self.script,
-            dict,
-        ), "script: expected dict, received {}.".format(
-            self.script.__class__.__name__,
-        )
+        if not isinstance(self.script, dict):
+            logger.error(
+                "script: expected dict, received {}.".format(
+                    self.script.__class__.__name__,
+                )
+            )
+            return
 
         self.script.setdefault("nodes", {})
-        assert isinstance(
+        if not isinstance(
             self.nodes,
             dict,
-        ), "nodes: expected dict, received {}.".format(
-            self.nodes.__class__.__name__,
-        )
+        ):
+            logger.error(
+                "nodes: expected dict, received {}.".format(
+                    self.nodes.__class__.__name__,
+                )
+            )
+            return
 
         self.script.setdefault("vars", {})
-        assert isinstance(
+        if not isinstance(
             self.vars,
             dict,
-        ), "vars: expected dict, received {}.".format(
-            self.vars.__class__.__name__,
-        )
+        ):
+            logger.error(
+                "vars: expected dict, received {}.".format(
+                    self.vars.__class__.__name__,
+                )
+            )
+            return
+
+        self.script.setdefault("versions", {})
+        if not isinstance(
+            self.versions,
+            dict,
+        ):
+            logger.error(
+                "versions: expected dict, received {}.".format(
+                    self.versions.__class__.__name__,
+                )
+            )
+            return
+
+        if script_version != "base":
+            if script_version not in self.versions:
+                logger.error(f"{script_version}: script version not found.")
+                return
+
+            for thing in ["vars", "nodes"]:
+                updates = self.versions[script_version].get(thing, {})
+                if len(updates) and log:
+                    logger.info(f"{thing}.update({updates})")
+                self.metadata["script"][thing].update(updates)
 
         if self.test_mode:
-            logger.info("🧪  test mode is on.")
+            if log:
+                logger.info("🧪  test mode is on.")
 
             if "test_mode" in self.script:
                 updates = self.script["test_mode"]
-                logger.info(f"🧪  vars.update({updates})")
+                if log:
+                    logger.info(f"🧪  vars.update({updates})")
                 self.vars.update(updates)
 
             for node_name, node in self.nodes.items():
                 if "test_mode" in node:
                     updates = node["test_mode"]
-                    logger.info(f"🧪  {node_name}.update({updates})")
+                    if log:
+                        logger.info(f"🧪  {node_name}.update({updates})")
                     node.update(updates)
 
-        logger.info(
-            "loaded {} node(s): {}".format(
-                len(self.nodes),
-                ", ".join(self.nodes.keys()),
+        if log:
+            logger.info(
+                "loaded {} node(s): {}".format(
+                    len(self.nodes),
+                    ", ".join(self.nodes.keys()),
+                )
             )
-        )
 
-        logger.info(
-            "loaded {} var(s): {}".format(
-                len(self.vars),
-                ", ".join(self.vars.keys()),
+            logger.info(
+                "loaded {} var(s): {}".format(
+                    len(self.vars),
+                    ", ".join(self.vars.keys()),
+                )
             )
-        )
         if verbose:
             for var_name, var_value in self.vars.items():
                 logger.info("{}: {}".format(var_name, var_value))
 
-        assert self.generate_graph(), "cannot generate graph."
+        if not self.generate_graph(
+            log=log,
+            verbose=verbose,
+            save_graph=save_graph,
+        ):
+            return
+
+        self.valid = True
 
     def __str__(self) -> str:
         return "{}[{} var(s), {} node(s) -> {}]".format(
@@ -115,13 +184,17 @@ class BaseScript(RootScript):
             text = text.replace(f":::{var_name}", str(var_value))
 
         for node_name, node in self.nodes.items():
-            text = text.replace(f":::{node_name}", node.get("output", ""))
+            node_output = node.get("output", "")
+            if isinstance(node_output, str):
+                text = text.replace(f":::{node_name}", node_output)
 
         return text
 
     def generate_graph(
         self,
+        log: bool = True,
         verbose: bool = False,
+        save_graph: bool = True,
     ) -> bool:
         self.G: nx.DiGraph = nx.DiGraph()
 
@@ -146,7 +219,7 @@ class BaseScript(RootScript):
                 if dependency:
                     self.G.add_edge(node_name, dependency)
 
-        return self.save_graph()
+        return self.save_graph(log=log) if save_graph else True
 
     def get_context(
         self,
@@ -174,14 +247,26 @@ class BaseScript(RootScript):
         logger.error(f"{action_name}: action not found.")
         return False
 
-    def run(self) -> bool:
+    def run(
+        self,
+        runnable: str = "",
+    ) -> bool:
         logger.info(f"{self.name}.run -> {self.object_name}")
 
+        if runnable:
+            logger.info(f"applying runnables: {runnable}")
+            runnable_options = Options(runnable)
+            for node_name, node_is_runnable in runnable_options.items():
+                logger.info(f"{node_name}.runnable={node_is_runnable}")
+                self.nodes[node_name]["runnable"] = node_is_runnable
+
         success: bool = True
         while (
             not all(self.nodes[node].get("completed", False) for node in self.nodes)
             and success
         ):
+            self.nodes_changed = False
+
             for node_name in tqdm(self.nodes):
                 if self.nodes[node_name].get("completed", False):
                     continue
@@ -212,6 +297,22 @@ class BaseScript(RootScript):
 
                 self.nodes[node_name]["completed"] = True
 
+                cache_filename = self.nodes[node_name].get("cache", "")
+                if cache_filename:
+                    if not file.save_text(
+                        objects.path_of(
+                            object_name=self.object_name,
+                            filename=cache_filename,
+                        ),
+                        [self.nodes[node_name].get("output", "")],
+                    ):
+                        success = False
+                        break
+
+                if self.nodes_changed:
+                    logger.info("🪄  nodes changed.")
+                    break
+
         if not post_to_object(
             self.object_name,
             "output",
@@ -221,7 +322,10 @@ class BaseScript(RootScript):
 
         return success
 
-    def save_graph(self) -> bool:
+    def save_graph(
+        self,
+        log: bool = True,
+    ) -> bool:
         return dot_file.save_to_file(
             objects.path_of(
                 filename="workflow.dot",
@@ -235,6 +339,7 @@ class BaseScript(RootScript):
                 ]
             ),
             add_legend=False,
+            log=log,
         )
 
     # Aliases
@@ -249,3 +354,7 @@ class BaseScript(RootScript):
     @property
     def vars(self) -> Dict:
         return self.metadata["script"]["vars"]
+
+    @property
+    def versions(self) -> Dict:
+        return self.metadata["script"]["versions"]

{blue_assistant-4.307.1 → blue_assistant-4.325.1}/blue_assistant/script/repository/blue_amo/classes.py

@@ -14,35 +14,33 @@ NAME = module.name(__file__, NAME)
 class BlueAmoScript(BaseScript):
     name = path.name(file.path(__file__))
 
-    def __init__(
-        self,
-        object_name: str,
-        test_mode: bool = False,
-        verbose: bool = False,
-    ):
-        super().__init__(
-            object_name=object_name,
-            test_mode=test_mode,
-            verbose=verbose,
-        )
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
 
         self.dict_of_actions.update(dict_of_actions)
 
     def generate_graph(
         self,
+        log: bool = True,
         verbose: bool = False,
+        save_graph: bool = True,
     ) -> bool:
-        if not super().generate_graph(verbose=verbose):
+        if not super().generate_graph(
+            log=log,
+            verbose=verbose,
+            save_graph=save_graph,
+        ):
             return False
 
         map_node_name = "generating_the_frames"
-        logger.info(
-            "{}: expanding {} X {}...".format(
-                NAME,
-                map_node_name,
-                self.vars["frame_count"],
+        if log:
+            logger.info(
+                "{}: expanding {} X {}...".format(
+                    NAME,
+                    map_node_name,
+                    self.vars["frame_count"],
+                )
             )
-        )
 
         map_node = self.nodes[map_node_name]
         del self.nodes[map_node_name]
@@ -64,4 +62,4 @@ class BlueAmoScript(BaseScript):
                 node_name,
             )
 
-        return self.save_graph()
+        return self.save_graph(log=log) if save_graph else True

blue_assistant-4.325.1/blue_assistant/script/repository/functions.py

@@ -0,0 +1,18 @@
+from typing import List, Tuple
+
+from blue_options import string
+
+from blue_assistant.script.load import load_script
+
+
+def get_script_versions(
+    script_name: str,
+) -> List[str]:
+    success, script = load_script(
+        script_name=script_name,
+        object_name=string.timestamp(),
+        log=False,
+        save_graph=False,
+    )
+
+    return list(script.versions.keys()) if success else []

blue_assistant-4.325.1/blue_assistant/script/repository/orbital_data_explorer/actions/__init__.py

@@ -0,0 +1,11 @@
+from typing import Dict, Callable
+
+from blue_assistant.script.repository.base.classes import BaseScript
+from blue_assistant.script.repository.orbital_data_explorer.actions import (
+    expanding_the_extractions,
+)
+
+
+dict_of_actions: Dict[str, Callable[[BaseScript, str], bool]] = {
+    "expanding_the_extractions": expanding_the_extractions.expanding_the_extractions,
+}

blue_assistant-4.325.1/blue_assistant/script/repository/orbital_data_explorer/actions/expanding_the_extractions.py

@@ -0,0 +1,109 @@
+import copy
+from typing import Dict
+
+from blueness import module
+from blue_options.logger import log_dict, log_list
+from blue_objects import file, objects
+from blue_objects.metadata import get_from_object
+
+from blue_assistant import NAME
+from blue_assistant.script.repository.base.classes import BaseScript
+from blue_assistant.web.functions import url_to_filename
+from blue_assistant.logger import logger
+
+NAME = module.name(__file__, NAME)
+
+
+def expanding_the_extractions(
+    script: BaseScript,
+    node_name: str,
+) -> bool:
+    map_node_name = "extraction"
+
+    crawl_cache: Dict[str, str] = get_from_object(
+        script.object_name,
+        "web_crawl_cache",
+        {},
+    )
+    log_dict(logger, "using", crawl_cache, "crawled url(s)")
+
+    list_of_urls = [
+        url
+        for url, content_type in crawl_cache.items()
+        if "html" in content_type
+        and not file.exists(
+            objects.path_of(
+                object_name=script.object_name,
+                filename="{}_cache/{}.txt".format(
+                    map_node_name,
+                    url_to_filename(url),
+                ),
+            )
+        )
+    ]
+    log_list(logger, "using", list_of_urls, "crawled unextracted html(s).")
+
+    max_nodes = min(
+        len(list_of_urls),
+        script.nodes[node_name]["max_nodes"],
+    )
+    logger.info(
+        "{}: expanding {} X {}...".format(
+            NAME,
+            map_node_name,
+            max_nodes,
+        )
+    )
+
+    map_node = script.nodes[map_node_name]
+    del script.nodes[map_node_name]
+    script.G.remove_node(map_node_name)
+
+    reduce_node_name = "generating_summary"
+    for index in range(max_nodes):
+        url = list_of_urls[index]
+        index_node_name = f"{map_node_name}_{index+1:03d}"
+
+        success, url_content = file.load_yaml(
+            filename=objects.path_of(
+                object_name=script.object_name,
+                filename="web_crawl_cache/{}.yaml".format(
+                    url_to_filename(url),
+                ),
+            ),
+        )
+        if not success:
+            logger.warning(f"{url}: failed to load url content.")
+            continue
+        if "text" not in url_content:
+            logger.warning(f"{url}: no text found in url content.")
+            continue
+
+        logger.info(f"{url} -{map_node_name}-> {index_node_name}")
+
+        script.nodes[index_node_name] = copy.deepcopy(map_node)
+
+        script.nodes[index_node_name]["prompt"] = map_node["prompt"].replace(
+            ":::url_content",
+            url_content["text"],
+        )
+
+        script.nodes[index_node_name]["url"] = url
+        script.nodes[index_node_name]["cache"] = "{}_cache/{}.txt".format(
+            map_node_name,
+            url_to_filename(url),
+        )
+
+        script.G.add_node(index_node_name)
+        script.G.add_edge(
+            index_node_name,
+            node_name,
+        )
+        script.G.add_edge(
+            reduce_node_name,
+            index_node_name,
+        )
+
+    script.nodes_changed = True
+
+    return script.save_graph()

blue_assistant-4.325.1/blue_assistant/script/repository/orbital_data_explorer/classes.py

@@ -0,0 +1,15 @@
+from blue_objects import file, path
+
+from blue_assistant.script.repository.base.classes import BaseScript
+from blue_assistant.script.repository.orbital_data_explorer.actions import (
+    dict_of_actions,
+)
+
+
+class OrbitalDataExplorerScript(BaseScript):
+    name = path.name(file.path(__file__))
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.dict_of_actions.update(dict_of_actions)

{blue_assistant-4.307.1 → blue_assistant-4.325.1}/blue_assistant/web/crawl.py

@@ -35,7 +35,7 @@ def crawl_list_of_urls(
             seed_urls
             + get_from_object(
                 object_name,
-                f"{cache_prefix}_crawl_queue",
+                f"{cache_prefix}_queue",
                 [],
             )
         )
@@ -44,7 +44,7 @@ def crawl_list_of_urls(
 
     crawl_cache: Dict[str, str] = get_from_object(
         object_name,
-        f"{cache_prefix}_crawl_cache",
+        f"{cache_prefix}_cache",
         {},
     )
     log_dict(logger, "loaded cache:", crawl_cache, "url(s)")
@@ -75,7 +75,7 @@ def crawl_list_of_urls(
             if not file.save_yaml(
                 filename=objects.path_of(
                     object_name=object_name,
-                    filename="{}_crawl_cache/{}.yaml".format(
+                    filename="{}_cache/{}.yaml".format(
                         cache_prefix,
                         url_to_filename(url),
                     ),
@@ -108,12 +108,12 @@ def crawl_list_of_urls(
     if not (
         post_to_object(
             object_name,
-            f"{cache_prefix}_crawl_cache",
+            f"{cache_prefix}_cache",
             crawl_cache,
         )
         and post_to_object(
             object_name,
-            f"{cache_prefix}_crawl_queue",
+            f"{cache_prefix}_queue",
             queue,
         )
     ):

{blue_assistant-4.307.1 → blue_assistant-4.325.1}/blue_assistant/web/fetch.py

@@ -9,6 +9,7 @@ from blueness import module
 from blue_options.logger import log_long_text, log_list
 
 from blue_assistant import NAME
+from blue_assistant.web.functions import normalize_url
 from blue_assistant.logger import logger
 
 warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
@@ -49,8 +50,7 @@ def fetch_links_and_text(
         for a_tag in soup.find_all("a", href=True):
             a_url = urljoin(url, a_tag["href"])
 
-            if "#" in a_url:
-                a_url = a_url.split("#", 1)[0]
+            a_url = normalize_url(a_url)
 
             if a_url.startswith(url):
                 if url not in list_of_urls:

{blue_assistant-4.307.1 → blue_assistant-4.325.1}/blue_assistant/web/functions.py

@@ -1,4 +1,15 @@
 import re
+from urllib.parse import urlparse, urlunparse
+
+
+# https://chatgpt.com/c/67d733a0-4be4-8005-bf52-fb9ba32487c2
+def normalize_url(url: str) -> str:
+    return urlunparse(
+        urlparse(url)._replace(
+            query="",
+            fragment="",
+        ),
+    )
 
 
 def url_to_filename(

{blue_assistant-4.307.1 → blue_assistant-4.325.1/blue_assistant.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: blue_assistant
-Version: 4.307.1
+Version: 4.325.1
 Summary: 🧠 An AI Assistant.
 Home-page: https://github.com/kamangir/blue-assistant
 Author: Arash Abadpour (Kamangir)
@@ -124,4 +124,4 @@ Also home to [`@web`](https://raw.githubusercontent.com/kamangir/blue-assistant/
 
 [![pylint](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pylint.yml) [![pytest](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/pytest.yml) [![bashtest](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml/badge.svg)](https://github.com/kamangir/blue-assistant/actions/workflows/bashtest.yml) [![PyPI version](https://img.shields.io/pypi/v/blue-assistant.svg)](https://pypi.org/project/blue-assistant/) [![PyPI - Downloads](https://img.shields.io/pypi/dd/blue-assistant)](https://pypistats.org/packages/blue-assistant)
 
-built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.307.1`](https://github.com/kamangir/blue-assistant).
+built by 🌀 [`blue_options-4.240.1`](https://github.com/kamangir/awesome-bash-cli), based on 🧠 [`blue_assistant-4.325.1`](https://github.com/kamangir/blue-assistant).

{blue_assistant-4.307.1 → blue_assistant-4.325.1}/blue_assistant.egg-info/SOURCES.txt

@@ -56,6 +56,7 @@ blue_assistant/script/actions/generate_text.py
 blue_assistant/script/actions/generic.py
 blue_assistant/script/actions/web_crawl.py
 blue_assistant/script/repository/__init__.py
+blue_assistant/script/repository/functions.py
 blue_assistant/script/repository/base/__init__.py
 blue_assistant/script/repository/base/classes.py
 blue_assistant/script/repository/base/root.py
@@ -71,6 +72,8 @@ blue_assistant/script/repository/hue/classes.py
 blue_assistant/script/repository/hue/colors.py
 blue_assistant/script/repository/orbital_data_explorer/__init__.py
 blue_assistant/script/repository/orbital_data_explorer/classes.py
+blue_assistant/script/repository/orbital_data_explorer/actions/__init__.py
+blue_assistant/script/repository/orbital_data_explorer/actions/expanding_the_extractions.py
 blue_assistant/web/__init__.py
 blue_assistant/web/__main__.py
 blue_assistant/web/crawl.py

{blue_assistant-4.307.1 → blue_assistant-4.325.1}/setup.py

@@ -18,6 +18,7 @@ setup(
         f"{NAME}.script.repository.blue_amo.actions",
         f"{NAME}.script.repository.hue",
         f"{NAME}.script.repository.orbital_data_explorer",
+        f"{NAME}.script.repository.orbital_data_explorer.actions",
         f"{NAME}.web",
     ],
     include_package_data=True,

blue_assistant-4.307.1/blue_assistant/script/repository/base/root.py

@@ -1,5 +0,0 @@
-from blue_objects import file, objects, path
-
-
-class RootScript:
-    name = path.name(file.path(__file__))

blue_assistant-4.307.1/blue_assistant/script/repository/orbital_data_explorer/classes.py

@@ -1,7 +0,0 @@
-from blue_objects import file, path
-
-from blue_assistant.script.repository.base.classes import BaseScript
-
-
-class OrbitalDataExplorerScript(BaseScript):
-    name = path.name(file.path(__file__))