kash-shell 0.3.30__py3-none-any.whl → 0.3.33__py3-none-any.whl

kash/actions/core/chat.py

@@ -28,6 +28,7 @@ log = get_logger(__name__)
 @kash_action(
     expected_args=ONE_OR_NO_ARGS,
     precondition=is_chat,
+    output_format=Format.yaml,
     uses_selection=False,
     interactive_input=True,
     cacheable=False,

kash/actions/core/markdownify_html.py

@@ -13,7 +13,9 @@ from kash.web_content.web_extract_readabilipy import extract_text_readabilipy
 log = get_logger(__name__)
 
 
-@kash_action(precondition=is_url_resource | has_html_body, mcp_tool=True)
+@kash_action(
+    precondition=is_url_resource | has_html_body, output_format=Format.markdown, mcp_tool=True
+)
 def markdownify_html(item: Item) -> Item:
     """
     Converts raw HTML or the URL of an HTML page to Markdown, fetching with the content

kash/actions/core/readability.py

@@ -9,10 +9,7 @@ from kash.web_content.web_extract_readabilipy import extract_text_readabilipy
 log = get_logger(__name__)
 
 
-@kash_action(
-    precondition=is_url_resource | has_html_body,
-    mcp_tool=True,
-)
+@kash_action(precondition=is_url_resource | has_html_body, output_format=Format.html)
 def readability(item: Item) -> Item:
     """
     Extracts clean HTML from a raw HTML item.

kash/actions/core/render_as_html.py

@@ -15,6 +15,7 @@ log = get_logger(__name__)
 @kash_action(
     expected_args=ONE_OR_MORE_ARGS,
     precondition=(has_html_body | has_simple_text_body) & ~has_fullpage_html_body,
+    output_format=Format.html,
     params=(Param("no_title", "Don't add a title to the page body.", type=bool),),
 )
 def render_as_html(input: ActionInput, no_title: bool = False) -> ActionResult:

kash/actions/core/show_webpage.py

@@ -6,11 +6,13 @@ from kash.exec_model.args_model import ONE_OR_MORE_ARGS
 from kash.exec_model.commands_model import Command
 from kash.exec_model.shell_model import ShellResult
 from kash.model import ActionInput, ActionResult
+from kash.utils.file_utils.file_formats_model import Format
 
 
 @kash_action(
     expected_args=ONE_OR_MORE_ARGS,
     precondition=(has_html_body | has_simple_text_body) & ~has_fullpage_html_body,
+    output_format=Format.html,
 )
 def show_webpage(input: ActionInput) -> ActionResult:
     """

kash/config/logger.py

@@ -281,7 +281,7 @@ def _do_logging_setup(log_settings: LogSettings):
 def prefix(line: str, emoji: str = "", warn_emoji: str = "") -> str:
     prefix = task_stack_prefix_str()
     emojis = f"{warn_emoji}{emoji}".strip()
-    return " ".join(filter(None, [prefix, emojis, line]))
+    return "".join(filter(None, [prefix, emojis, line]))
 
 
 def prefix_args(

kash/config/text_styles.py

@@ -262,7 +262,7 @@ PROMPT_ASSIST = "(assistant) ❯"
 
 EMOJI_HINT = "👉"
 
-EMOJI_MSG_INDENT = "⋮"
+EMOJI_MSG_INDENT = "⋮   "
 
 EMOJI_START = "[➤]"
 

kash/docs/markdown/topics/a2_installation.md

@@ -60,8 +60,9 @@ These are for `kash-media` but you can use a `kash-shell` for a more basic setup
 
    ```shell
    sudo apt-get update
-   sudo apt-get install -y libgl1 ffmpeg libmagic-dev 
-   # For the additional command-line tools, pixi is better on Ubuntu:
+   sudo apt-get install -y libgl1 ffmpeg libmagic-dev imagemagick bat ripgrep hexyl
+
+   # Or for additional command-line tools, pixi is better on Ubuntu:
    curl -fsSL https://pixi.sh/install.sh | sh
    . ~/.bashrc
    pixi global install ripgrep bat eza hexyl imagemagick zoxide

kash/exec/action_decorators.py

@@ -37,6 +37,7 @@ from kash.model.params_model import Param, ParamDeclarations, TypedParamValues
 from kash.model.preconditions_model import Precondition
 from kash.utils.common.function_inspect import FuncParam, inspect_function_params
 from kash.utils.errors import InvalidDefinition
+from kash.utils.file_utils.file_formats_model import Format
 
 log = get_logger(__name__)
 
@@ -204,6 +205,7 @@ def kash_action(
     arg_type: ArgType = ArgType.Locator,
     expected_args: ArgCount = ONE_ARG,
     output_type: ItemType = ItemType.doc,
+    output_format: Format | None = None,
     expected_outputs: ArgCount = ONE_ARG,
     params: ParamDeclarations = (),
     run_per_item: bool | None = None,
@@ -318,6 +320,7 @@ def kash_action(
                 self.arg_type = arg_type
                 self.uses_selection = uses_selection
                 self.output_type = output_type
+                self.output_format = output_format
                 self.interactive_input = interactive_input
                 self.live_output = live_output
                 self.mcp_tool = mcp_tool
@@ -397,9 +400,8 @@ def kash_action(
                 context = ExecContext(action, current_runtime_settings())
 
             # Run the action.
-            result, _, _ = run_action_with_caching(context, action_input)
-
-            return result
+            result_with_paths = run_action_with_caching(context, action_input)
+            return result_with_paths.result
 
         if is_simple_func:
             # Need to convert back to a SimpleActionFunction.

kash/exec/action_exec.py

@@ -1,5 +1,5 @@
 import time
-from dataclasses import replace
+from dataclasses import dataclass, replace
 from pathlib import Path
 
 from prettyfmt import fmt_lines, fmt_path, plural
@@ -29,8 +29,10 @@ from kash.model.operations_model import Input, Operation, Source
 from kash.model.params_model import ALL_COMMON_PARAMS, GLOBAL_PARAMS, RawParamValues
 from kash.model.paths_model import StorePath
 from kash.shell.output.shell_output import PrintHooks
+from kash.utils.common.s3_utils import get_s3_parent_folder, s3_sync_to_folder
 from kash.utils.common.task_stack import task_stack
 from kash.utils.common.type_utils import not_none
+from kash.utils.common.url import Url, is_s3_url
 from kash.utils.errors import ContentError, InvalidOutput, get_nonfatal_exceptions
 from kash.workspaces import Selection, current_ws
 
@@ -182,7 +184,7 @@ def run_action_operation(
             this_op = replace(operation, arguments=[operation.arguments[i]])
         else:
             this_op = operation
-        item.update_history(Source(operation=this_op, output_num=i, cacheable=action.cacheable))
+        item.update_source(Source(operation=this_op, output_num=i, cacheable=action.cacheable))
 
     # Override the state if appropriate (this handles marking items as transient).
     if settings.override_state:
@@ -339,20 +341,41 @@ def save_action_result(
     return result_store_paths, archived_store_paths
 
 
+@dataclass(frozen=True)
+class ResultWithPaths:
+    """
+    Result of an action, including the store paths of any S3 items created.
+    """
+
+    result: ActionResult
+    result_paths: list[StorePath]
+    archived_paths: list[StorePath]
+    s3_paths: list[Url]
+
+
 def run_action_with_caching(
     exec_context: ExecContext, action_input: ActionInput
-) -> tuple[ActionResult, list[StorePath], list[StorePath]]:
+) -> ResultWithPaths:
     """
     Run an action, including validation, only rerunning if `rerun` requested or
     result is not already present. Returns the result, the store paths of the
     result items, and the store paths of any archived items.
 
+    Also handles optional S3 syncing if the input was from S3.
+
     Note: Mutates the input but only to add `context` to each item.
     """
     action = exec_context.action
     settings = exec_context.settings
     ws = settings.workspace
 
+    # If the input is from S3, we note the parent folder to copy the output back to.
+    s3_parent_folder = None
+    if exec_context.settings.sync_to_s3 and action_input.items and action_input.items[0].url:
+        url = action_input.items[0].url
+        if url and is_s3_url(url):
+            s3_parent_folder = get_s3_parent_folder(url)
+
     # Assemble the operation and validate the action input.
     operation = validate_action_input(exec_context, ws, action, action_input)
 
@@ -408,7 +431,26 @@ def run_action_with_caching(
     finally:
         action_input.clear_context()
 
-    return result, result_store_paths, archived_store_paths
+    # If the action created an S3 item, we copy it back to the same S3 parent folder.
+    # Only do this for the first result, for simplicity.
+    s3_urls: list[Url] = []
+    if s3_parent_folder and len(result_store_paths) > 0:
+        log.warning(
+            "Source was an S3 path so syncing result S3: %s -> %s",
+            result_store_paths[0],
+            s3_parent_folder,
+        )
+        s3_urls = s3_sync_to_folder(
+            result_store_paths[0], s3_parent_folder, include_sidematter=True
+        )
+        log.message("Synced result to S3:\n%s", fmt_lines(s3_urls))
+
+    return ResultWithPaths(
+        result=result,
+        result_paths=result_store_paths,
+        archived_paths=archived_store_paths,
+        s3_paths=s3_urls,
+    )
 
 
 def run_action_with_shell_context(
@@ -486,7 +528,10 @@ def run_action_with_shell_context(
     input = prepare_action_input(*args, refetch=refetch)
 
     # Finally, run the action.
-    result, result_store_paths, archived_store_paths = run_action_with_caching(context, input)
+    result_with_paths = run_action_with_caching(context, input)
+    result = result_with_paths.result
+    result_store_paths = result_with_paths.result_paths
+    archived_store_paths = result_with_paths.archived_paths
 
     # Implement any path operations from the output and/or select the final output
     if not internal_call:

kash/exec/fetch_url_items.py

@@ -98,9 +98,11 @@ def fetch_url_item_content(
     from kash.workspaces import current_ws
 
     ws = current_ws()
-    if not refetch and item.title and item.description and item.body:
+    # We could check for description too, but many pages don't have one.
+    has_key_content = item.title and (not item.has_body or item.body)
+    if not refetch and has_key_content:
         log.info(
-            "Already have title, description, and body, will not fetch: %s",
+            "Already have title so assuming metadata is up to date, will not fetch: %s",
             item.fmt_loc(),
         )
         return FetchItemResult(item, was_cached=True)

kash/exec/llm_transforms.py

@@ -1,4 +1,5 @@
 from dataclasses import replace
+from typing import Unpack
 
 from chopdiff.docs import DiffFilter, TextDoc
 from chopdiff.transforms import WindowSettings, filtered_transform
@@ -12,7 +13,7 @@ from kash.llm_utils.fuzzy_parsing import strip_markdown_fence
 from kash.llm_utils.llm_completion import llm_template_completion
 from kash.llm_utils.llm_messages import Message, MessageTemplate
 from kash.model.actions_model import LLMOptions
-from kash.model.items_model import Item
+from kash.model.items_model import Item, ItemType, ItemUpdateOptions
 from kash.utils.errors import InvalidInput
 from kash.utils.file_utils.file_formats_model import Format
 from kash.utils.text_handling.doc_normalization import normalize_formatting
@@ -88,10 +89,11 @@ def llm_transform_str(options: LLMOptions, input_str: str, check_no_results: boo
 def llm_transform_item(
     item: Item,
     model: LLMName | None = None,
+    *,
     normalize: bool = True,
     strip_fence: bool = True,
     check_no_results: bool = True,
-    format: Format | None = None,
+    **updates: Unpack[ItemUpdateOptions],
 ) -> Item:
     """
     Main function for running an LLM action on an item.
@@ -99,6 +101,14 @@ def llm_transform_item(
     Model may be overridden by an explicit model parameter.
     Also by default cleans up and normalizes output as Markdown.
     """
+    # Default to Markdown docs.
+    if "format" not in updates:
+        updates["format"] = Format.markdown
+    if "type" not in updates:
+        updates["type"] = ItemType.doc
+    if "body" not in updates:
+        updates["body"] = None
+
     if not item.context:
         raise InvalidInput(f"LLM actions expect a context on input item: {item}")
     action = item.context.action
@@ -112,13 +122,12 @@ def llm_transform_item(
     log.message("LLM transform from action `%s` on item: %s", action.name, item)
     log.message("LLM options: %s", action.llm_options)
 
-    format = format or item.format or Format.markdown
-    result_item = item.derived_copy(body=None, format=format)
+    result_item = item.derived_copy(**updates)
     result_str = llm_transform_str(llm_options, item.body, check_no_results=check_no_results)
     if strip_fence:
         result_str = strip_markdown_fence(result_str)
     if normalize:
-        result_str = normalize_formatting(result_str, format=format)
+        result_str = normalize_formatting(result_str, format=updates["format"])
 
     result_item.body = result_str
     return result_item

kash/exec/runtime_settings.py

@@ -102,6 +102,7 @@ def kash_runtime(
     override_state: State | None = None,
     tmp_output: bool = False,
     no_format: bool = False,
+    sync_to_s3: bool = False,
 ) -> RuntimeSettingsManager:
     """
     Set a specific kash execution context for a with block.
@@ -131,5 +132,6 @@ def kash_runtime(
         override_state=override_state,
         tmp_output=tmp_output,
         no_format=no_format,
+        sync_to_s3=sync_to_s3,
     )
     return RuntimeSettingsManager(settings=settings)

kash/file_storage/file_store.py

@@ -16,19 +16,15 @@ from typing_extensions import override
 
 from kash.config.logger import get_log_settings, get_logger
 from kash.config.text_styles import EMOJI_SAVED
+from kash.file_storage.item_id_index import ItemIdIndex
 from kash.file_storage.metadata_dirs import MetadataDirs
-from kash.file_storage.store_filenames import (
-    folder_for_type,
-    join_suffix,
-    parse_item_filename,
-)
-from kash.model.items_model import Item, ItemId, ItemType
+from kash.file_storage.store_filenames import folder_for_type, join_suffix
+from kash.model.items_model import Item, ItemType
 from kash.model.paths_model import StorePath
 from kash.shell.output.shell_output import PrintHooks
 from kash.utils.common.format_utils import fmt_loc
-from kash.utils.common.uniquifier import Uniquifier
 from kash.utils.common.url import Locator, UnresolvedLocator, Url, is_url
-from kash.utils.errors import FileExists, FileNotFound, InvalidFilename, SkippableError
+from kash.utils.errors import FileExists, FileNotFound
 from kash.utils.file_utils.file_formats_model import Format
 from kash.utils.file_utils.file_walk import walk_by_dir
 from kash.utils.file_utils.ignore_files import IgnoreChecker, add_to_ignore
@@ -94,9 +90,8 @@ class FileStore(Workspace):
         self.info_logged = False
         self.warnings: list[str] = []
 
-        # TODO: Move this to its own IdentifierIndex class, and make it exactly mirror disk state.
-        self.uniquifier = Uniquifier()
-        self.id_map: dict[ItemId, StorePath] = {}
+        # Index of item identifiers and unique slug history
+        self.id_index = ItemIdIndex()
 
         self.dirs = MetadataDirs(base_dir=self.base_dir, is_global_ws=self.is_global_ws)
         if not auto_init and not self.dirs.is_initialized():
@@ -133,7 +128,7 @@ class FileStore(Workspace):
     def _id_index_init(self):
         num_dups = 0
         for store_path in self.walk_items():
-            dup_path = self._id_index_item(store_path)
+            dup_path = self.id_index.index_item(store_path, self.load)
             if dup_path:
                 num_dups += 1
 
@@ -142,62 +137,6 @@ class FileStore(Workspace):
                 f"Found {num_dups} duplicate items in store. See `logs` for details."
             )
 
-    @synchronized
-    def _id_index_item(self, store_path: StorePath) -> StorePath | None:
-        """
-        Update metadata index with a new item.
-        """
-        name, item_type, _format, file_ext = parse_item_filename(store_path)
-        if not file_ext:
-            log.debug(
-                "Skipping file with unrecognized name or extension: %s",
-                fmt_path(store_path),
-            )
-            return None
-
-        full_suffix = join_suffix(item_type.name, file_ext.name) if item_type else file_ext.name
-        self.uniquifier.add(name, full_suffix)
-
-        dup_path = None
-
-        try:
-            item = self.load(store_path)
-            item_id = item.item_id()
-            if item_id:
-                old_path = self.id_map.get(item_id)
-                if old_path and old_path != store_path:
-                    dup_path = old_path
-                    log.info(
-                        "Duplicate items (%s):\n%s",
-                        item_id,
-                        fmt_lines([old_path, store_path]),
-                    )
-                self.id_map[item_id] = store_path
-        except (ValueError, SkippableError) as e:
-            log.warning(
-                "Could not load file, skipping from store index: %s: %s",
-                fmt_path(store_path),
-                e,
-            )
-
-        return dup_path
-
-    @synchronized
-    def _id_unindex_item(self, store_path: StorePath):
-        """
-        Remove an item from the metadata index.
-        """
-        try:
-            item = self.load(store_path)
-            item_id = item.item_id()
-            if item_id:
-                try:
-                    self.id_map.pop(item_id, None)
-                except KeyError:
-                    pass  # If we happen to reload a store it might no longer be in memory.
-        except (FileNotFoundError, InvalidFilename):
-            pass
-
     def resolve_to_store_path(self, path: Path | StorePath) -> StorePath | None:
         """
         Return a StorePath if the given path is within the store, otherwise None.
@@ -232,7 +171,6 @@ class FileStore(Workspace):
         """
         return (self.base_dir / store_path).exists()
 
-    @synchronized
     def _pick_filename_for(self, item: Item, *, overwrite: bool = False) -> tuple[str, str | None]:
         """
         Get a suitable filename for this item. If `overwrite` is true, use the the slugified
@@ -251,7 +189,7 @@ class FileStore(Workspace):
         slug = item.slug_name()
         full_suffix = item.get_full_suffix()
         # Get a unique name per item type.
-        unique_slug, old_slugs = self.uniquifier.uniquify_historic(slug, full_suffix)
+        unique_slug, old_slugs = self.id_index.uniquify_slug(slug, full_suffix)
 
         # Suffix files with both item type and a suitable file extension.
         new_unique_filename = join_suffix(unique_slug, full_suffix)
@@ -274,24 +212,34 @@ class FileStore(Workspace):
         Best effort to see if an item with the same identity is already in the store.
         """
         item_id = item.item_id()
-        log.info("Looking for item by id: %s", item_id)
+        log.info("Looking for item by id:\n%s", fmt_lines([item, item_id]))
         if not item_id:
             return None
         else:
-            store_path = self.id_map.get(item_id)
+            store_path = self.id_index.find_store_path_by_id(item_id)
             if not store_path:
-                # Just in case the id_map is not complete, check the default path too.
-                default_path = self.default_path_for(item)
-                if self.exists(default_path):
-                    old_item = self.load(default_path)
-                    if old_item.item_id() == item_id:
-                        log.info(
-                            "Item with the same id already saved (disk check):\n%s",
-                            fmt_lines([fmt_loc(default_path), item_id]),
-                        )
-                        store_path = default_path
-                        self.id_map[item_id] = default_path
-                        return default_path
+                # Just in case the index is not complete, check the other paths too
+                possible_paths = [
+                    p
+                    for p in [
+                        item.store_path,
+                        self.store_path_for(item)[0],
+                        self.default_path_for(item),
+                    ]
+                    if p
+                ]
+                for p in possible_paths:
+                    if self.exists(p):
+                        old_item = self.load(p)
+                        if old_item.item_id() == item_id:
+                            log.info(
+                                "Item with the same id already saved (disk check):\n%s",
+                                fmt_lines([fmt_loc(p), item_id]),
+                            )
+                            # Ensure index is updated consistently and with logging
+                            self.id_index.index_item(p, self.load)
+                            return p
+                log.info("Also checked paths but no id match:\n%s", fmt_lines(possible_paths))
             if store_path and self.exists(store_path):
                 log.info(
                     "Item with the same id already saved (disk check):\n%s",
@@ -321,9 +269,13 @@ class FileStore(Workspace):
             return self._tmp_path_for(item), None
         elif item.store_path:
             return StorePath(item.store_path), None
-        elif item_id in self.id_map and self.exists(self.id_map[item_id]):
+        elif (
+            item_id
+            and (existing := self.id_index.find_store_path_by_id(item_id))
+            and self.exists(existing)
+        ):
             # If this item has an identity and we've saved under that id before, use the same store path.
-            store_path = self.id_map[item_id]
+            store_path = existing
             log.info(
                 "When picking a store path, found an existing item with same id:\n%s",
                 fmt_lines([fmt_loc(store_path), item_id]),
@@ -415,6 +367,8 @@ class FileStore(Workspace):
             # Indicate this is an item with a store path, not an external path.
             # Keep external_path set so we know body is in that file.
             item.store_path = str(rel_path)
+            # Ensure index is updated for items written directly into the store.
+            self.id_index.index_item(StorePath(rel_path), self.load)
             return StorePath(rel_path)
         else:
             # Otherwise it's still in memory or in a file outside the workspace and we need to save it.
@@ -490,7 +444,7 @@ class FileStore(Workspace):
 
         # Update in-memory store_path only after successful save.
         item.store_path = str(store_path)
-        self._id_index_item(store_path)
+        self.id_index.index_item(store_path, self.load)
 
         if not skipped_save:
             log.message("%s Saved item: %s", EMOJI_SAVED, fmt_loc(store_path))
@@ -536,6 +490,7 @@ class FileStore(Workspace):
 
         if isinstance(locator, StorePath) and not reimport:
             log.info("Store path already imported: %s", fmt_loc(locator))
+            self.id_index.index_item(locator, self.load)
             return locator
         elif is_url(locator):
             # Import a URL as a resource.
@@ -669,7 +624,7 @@ class FileStore(Workspace):
         """
         self.selections.remove_values(store_paths)
         for store_path in store_paths:
-            self._id_unindex_item(store_path)
+            self.id_index.unindex_item(store_path, self.load)
         # TODO: Update metadata of all relations that point to this path too.
 
     @synchronized
@@ -679,8 +634,8 @@ class FileStore(Workspace):
         """
         self.selections.replace_values(replacements)
         for store_path, new_store_path in replacements:
-            self._id_unindex_item(store_path)
-            self._id_index_item(new_store_path)
+            self.id_index.unindex_item(store_path, self.load)
+            self.id_index.index_item(new_store_path, self.load)
         # TODO: Update metadata of all relations that point to this path too.
 
     def archive(
@@ -708,12 +663,13 @@ class FileStore(Workspace):
         if not orig_path.exists():
             log.warning("Item to archive not found: %s", fmt_loc(orig_path))
             return store_path
+        # Remove references (including id_map) before moving so we can load the item to compute id.
+        self._remove_references([store_path])
         if with_sidematter:
             move_sidematter(orig_path, full_archive_path)
         else:
             os.makedirs(full_archive_path.parent, exist_ok=True)
             shutil.move(orig_path, full_archive_path)
-        self._remove_references([store_path])
 
         archive_path = StorePath(self.dirs.archive_dir / store_path)
         return archive_path
@@ -732,6 +688,8 @@ class FileStore(Workspace):
             move_sidematter(full_input_path, original_path)
         else:
             shutil.move(full_input_path, original_path)
+        # Re-index after restoring from archive.
+        self.id_index.index_item(store_path, self.load)
         return StorePath(store_path)
 
     @synchronized
@@ -748,7 +706,7 @@ class FileStore(Workspace):
         log.message(
             "Using workspace: %s (%s items)",
             fmt_path(self.base_dir, rel_to_cwd=False),
-            len(self.uniquifier),
+            len(self.id_index),
         )
         log.message(
             "Logging to: %s",