kash-shell 0.3.28__py3-none-any.whl → 0.3.33__py3-none-any.whl

kash/file_storage/item_file_format.py

@@ -1,8 +1,16 @@
 from pathlib import Path
 
-from frontmatter_format import FmStyle, fmf_has_frontmatter, fmf_read, fmf_write
+from frontmatter_format import (
+    FmStyle,
+    fmf_has_frontmatter,
+    fmf_read,
+    fmf_read_frontmatter,
+    fmf_write,
+)
 from funlog import tally_calls
 from prettyfmt import custom_key_sort, fmt_size_human
+from sidematter_format import Sidematter
+from strif import atomic_output_file, single_line
 
 from kash.config.logger import get_logger
 from kash.model.items_model import ITEM_FIELDS, Item
@@ -22,19 +30,28 @@ _item_cache = MtimeCache[Item](max_size=2000, name="Item")
 
 
 @tally_calls()
-def write_item(item: Item, path: Path, normalize: bool = True):
+def write_item(item: Item, path: Path, *, normalize: bool = True, use_frontmatter: bool = True):
     """
-    Write a text item to a file with standard frontmatter format YAML.
+    Write a text item to a file with standard frontmatter format YAML or sidematter format.
     By default normalizes formatting of the body text and updates the item's body.
+
+    If `use_frontmatter` is True, uses frontmatter on the file for metadata, and omits
+    the sidematter metadata file.
+
+    This function does not explicitly write sidematter assets; these can be written
+    separately.
     """
     item.validate()
-    if item.format and not item.format.supports_frontmatter:
+    if use_frontmatter and item.format and not item.format.supports_frontmatter:
         raise ValueError(f"Item format `{item.format.value}` does not support frontmatter: {item}")
 
     # Clear cache before writing.
     _item_cache.delete(path)
 
+    title = item.title
     if normalize:
+        if item.title:
+            title = single_line(item.title)
         body = normalize_formatting(item.body_text(), item.format)
     else:
         body = item.body_text()
@@ -65,29 +82,44 @@ def write_item(item: Item, path: Path, normalize: bool = True):
 
     log.debug("Writing item to %s: body length %s, metadata %s", path, len(body), item.metadata())
 
-    fmf_write(
-        path,
-        body,
-        item.metadata(),
-        style=fm_style,
-        key_sort=ITEM_FIELD_SORT,
-        make_parents=True,
-    )
+    # Use sidematter format
+    spath = Sidematter(path)
+    if use_frontmatter:
+        # Use frontmatter format
+        fmf_write(
+            path,
+            body,
+            item.metadata(),
+            style=fm_style,
+            key_sort=ITEM_FIELD_SORT,
+            make_parents=True,
+        )
+    else:
+        # Write the main file with just the body (no frontmatter)
+        with atomic_output_file(path, make_parents=True) as f:
+            f.write_text(body, encoding="utf-8")
+
+        # Sidematter metadata
+        spath.write_meta(item.metadata(), key_sort=ITEM_FIELD_SORT)
 
     # Update cache.
     _item_cache.update(path, item)
 
-    # Update the item's body to reflect normalization.
+    # Update the item.
+    item.title = title
     item.body = body
 
 
-def read_item(path: Path, base_dir: Path | None) -> Item:
+def read_item(path: Path, base_dir: Path | None, preserve_filename: bool = True) -> Item:
     """
-    Read an item from a file. Uses `base_dir` to resolve paths, so the item's
+    Read a text item from a file. Uses `base_dir` to resolve paths, so the item's
     `store_path` will be set and be relative to `base_dir`.
 
-    If frontmatter format YAML is present, it is parsed. If not, the item will
-    be a resource with a format inferred from the file extension or the content.
+    Automatically detects and reads sidematter format (metadata in .meta.yml/.meta.json
+    sidecar files), which takes precedence over frontmatter when present. Falls back to
+    frontmatter format YAML if no sidematter is found. If neither is present, the item
+    will be a resource with a format inferred from the file extension or the content.
+
     The `store_path` will be the path relative to the `base_dir`, if the file
     is within `base_dir`, or otherwise the `external_path` will be set to the path
     it was read from.
@@ -98,20 +130,48 @@ def read_item(path: Path, base_dir: Path | None) -> Item:
         log.debug("Cache hit for item: %s", path)
         return cached_item
 
-    return _read_item_uncached(path, base_dir)
+    return _read_item_uncached(path, base_dir, preserve_filename=preserve_filename)
 
 
 @tally_calls()
-def _read_item_uncached(path: Path, base_dir: Path | None) -> Item:
+def _read_item_uncached(
+    path: Path,
+    base_dir: Path | None,
+    *,
+    preserve_filename: bool = True,
+    prefer_frontmatter: bool = True,
+) -> Item:
+    # First, try to resolve sidematter
+    sidematter = Sidematter(path).resolve(use_frontmatter=False)
+
+    # Use sidematter metadata unless we find and prefer frontmatter for metadata.
     has_frontmatter = fmf_has_frontmatter(path)
-    body = metadata = None
-    if has_frontmatter:
+    frontmatter_meta = prefer_frontmatter and has_frontmatter and fmf_read_frontmatter(path)
+    if sidematter.meta and not frontmatter_meta:
+        metadata = sidematter.meta
+        body, _frontmatter_metadata = fmf_read(path)
+        log.debug(
+            "Read item from sidematter %s: body length %s, metadata %s",
+            sidematter.meta_path,
+            len(body),
+            metadata,
+        )
+    elif has_frontmatter:
         body, metadata = fmf_read(path)
-        log.debug("Read item from %s: body length %s, metadata %s", path, len(body), metadata)
+        log.debug(
+            "Read item from %s: body length %s, metadata %s",
+            path,
+            len(body),
+            metadata,
+        )
+    else:
+        # Not readable, binary or otherwise.
+        metadata = None
+        body = None
 
-        path = path.resolve()
-        if base_dir:
-            base_dir = base_dir.resolve()
+    path = path.resolve()
+    if base_dir:
+        base_dir = base_dir.resolve()
 
     # Ensure store_path is used if it's within the base_dir, and
     # external_path otherwise.
@@ -128,7 +188,8 @@ def _read_item_uncached(path: Path, base_dir: Path | None) -> Item:
             metadata, body=body, store_path=store_path, external_path=external_path
         )
     else:
-        # This is a file without frontmatter. Infer format from the file and content,
+        # This is a file without frontmatter or sidematter.
+        # Infer format from the file and content,
         # and use store_path or external_path as appropriate.
         item = Item.from_external_path(path)
         if item.format:
@@ -154,6 +215,10 @@ def _read_item_uncached(path: Path, base_dir: Path | None) -> Item:
                 item.body = f.read()
             item.external_path = None
 
+    # Preserve the original filename.
+    if preserve_filename:
+        item.original_filename = path.name
+
     # Update modified time.
     item.set_modified(path.stat().st_mtime)
 

kash/file_storage/item_id_index.py

@@ -0,0 +1,128 @@
+from __future__ import annotations
+
+import threading
+from collections.abc import Callable
+
+from prettyfmt import fmt_lines, fmt_path
+
+from kash.config.logger import get_logger
+from kash.file_storage.store_filenames import join_suffix, parse_item_filename
+from kash.model.items_model import Item, ItemId
+from kash.model.paths_model import StorePath
+from kash.utils.common.uniquifier import Uniquifier
+from kash.utils.errors import InvalidFilename, SkippableError
+
+log = get_logger(__name__)
+
+
+class ItemIdIndex:
+    """
+    Index of item identities and historical filenames within a workspace.
+
+    - Tracks a mapping of `ItemId -> StorePath` for quick lookups
+    - Tracks historical slugs via `Uniquifier` to generate unique names consistently
+
+    TODO: Should add a file system watcher to make this always consistent with disk state.
+    """
+
+    def __init__(self) -> None:
+        self._lock = threading.RLock()
+        self.uniquifier = Uniquifier()
+        self.id_map: dict[ItemId, StorePath] = {}
+
+    def reset(self) -> None:
+        """
+        Clear all index state.
+        """
+        with self._lock:
+            log.info("ItemIdIndex: reset")
+            self.uniquifier = Uniquifier()
+            self.id_map.clear()
+
+    def __len__(self) -> int:
+        """
+        Number of unique names tracked.
+        """
+        with self._lock:
+            return len(self.uniquifier)
+
+    def uniquify_slug(self, slug: str, full_suffix: str) -> tuple[str, list[str]]:
+        """
+        Return a unique slug and historic slugs for the given suffix.
+        """
+        with self._lock:
+            # This updates internal history as a side-effect. Log for consistency.
+            log.info("ItemIdIndex: uniquify slug '%s' with suffix '%s'", slug, full_suffix)
+            return self.uniquifier.uniquify_historic(slug, full_suffix)
+
+    def index_item(
+        self, store_path: StorePath, load_item: Callable[[StorePath], Item]
+    ) -> StorePath | None:
+        """
+        Update the index with an item at `store_path`.
+        Returns store path of any duplicate item with the same id, otherwise None.
+        """
+        name, item_type, _format, file_ext = parse_item_filename(store_path)
+        if not file_ext:
+            log.debug(
+                "Skipping file with unrecognized name or extension: %s",
+                fmt_path(store_path),
+            )
+            return None
+
+        with self._lock:
+            full_suffix = join_suffix(item_type.name, file_ext.name) if item_type else file_ext.name
+            # Track unique name history
+            self.uniquifier.add(name, full_suffix)
+
+        log.info("ItemIdIndex: indexing %s", fmt_path(store_path))
+
+        # Load item outside the lock to avoid holding it during potentially slow I/O
+        try:
+            item = load_item(store_path)
+        except (ValueError, SkippableError) as e:
+            log.warning(
+                "ItemIdIndex: could not index file, skipping: %s: %s",
+                fmt_path(store_path),
+                e,
+            )
+            return None
+
+        dup_path: StorePath | None = None
+        with self._lock:
+            item_id = item.item_id()
+            if item_id:
+                old_path = self.id_map.get(item_id)
+                if old_path and old_path != store_path:
+                    dup_path = old_path
+                    log.info(
+                        "ItemIdIndex: duplicate id detected %s:\n%s",
+                        item_id,
+                        fmt_lines([old_path, store_path]),
+                    )
+                self.id_map[item_id] = store_path
+                log.info("ItemIdIndex: set id %s -> %s", item_id, fmt_path(store_path))
+
+        return dup_path
+
+    def unindex_item(self, store_path: StorePath, load_item: Callable[[StorePath], Item]) -> None:
+        """
+        Remove an item from the id index.
+        """
+        try:
+            # Load item outside the lock to avoid holding it during potentially slow I/O
+            item = load_item(store_path)
+            item_id = item.item_id()
+            if item_id:
+                with self._lock:
+                    try:
+                        self.id_map.pop(item_id, None)
+                        log.info("ItemIdIndex: removed id %s for %s", item_id, fmt_path(store_path))
+                    except KeyError:
+                        pass
+        except (FileNotFoundError, InvalidFilename):
+            pass
+
+    def find_store_path_by_id(self, item_id: ItemId) -> StorePath | None:
+        with self._lock:
+            return self.id_map.get(item_id)

kash/help/help_types.py

@@ -5,7 +5,7 @@ from dataclasses import dataclass
 from enum import Enum
 from typing import TYPE_CHECKING, ClassVar
 
-from flowmark.sentence_split_regex import split_sentences_regex
+from flowmark import split_sentences_regex
 from rich.console import Group
 from rich.text import Text
 from strif import abbrev_str, single_line

kash/llm_utils/llms.py

@@ -13,6 +13,10 @@ class LLM(LLMName, Enum):
     """
 
     # https://platform.openai.com/docs/models
+    gpt_5 = LLMName("gpt-5")
+    gpt_5_mini = LLMName("gpt-5-mini")
+    gpt_5_nano = LLMName("gpt-5-nano")
+    gpt_5_chat = LLMName("gpt-5-chat")
     o4_mini = LLMName("o4-mini")
     o3 = LLMName("o3")
     o3_pro = LLMName("o3-pro")
@@ -25,11 +29,12 @@ class LLM(LLMName, Enum):
     gpt_4o = LLMName("gpt-4o")
     gpt_4o_mini = LLMName("gpt-4o-mini")
     gpt_4 = LLMName("gpt-4")
-
     gpt_4_1_mini = LLMName("gpt-4.1-mini")
     gpt_4_1_nano = LLMName("gpt-4.1-nano")
 
     # https://docs.anthropic.com/en/docs/about-claude/models/all-models
+
+    claude_4_1_opus = LLMName("claude-opus-4-1")
     claude_4_opus = LLMName("claude-opus-4-20250514")
     claude_4_sonnet = LLMName("claude-sonnet-4-20250514")
     claude_3_7_sonnet = LLMName("claude-3-7-sonnet-latest")

kash/local_server/local_server_commands.py

@@ -49,7 +49,8 @@ def local_server_logs(follow: bool = False) -> None:
     """
     Show the logs from the kash local (UI and MCP) servers.
 
-    :param follow: Follow the file as it grows.
+    Args:
+        follow: Follow the file as it grows.
     """
     log_path = global_settings().local_server_log_path
     if not log_path.exists():

kash/mcp/mcp_server_commands.py

@@ -54,8 +54,9 @@ def mcp_logs(follow: bool = False, all: bool = False) -> None:
     """
     Show the logs from the MCP server and CLI proxy process.
 
-    :param follow: Follow the file as it grows.
-    :param all: Show all logs, not just the server logs, including Claude Desktop logs if found.
+    Args:
+        follow: Follow the file as it grows.
+        all: Show all logs, not just the server logs, including Claude Desktop logs if found.
     """
     from kash.mcp.mcp_cli import MCP_CLI_LOG_PATH
 

kash/mcp/mcp_server_routes.py

@@ -6,8 +6,10 @@ from dataclasses import dataclass
 
 from funlog import log_calls
 from mcp.server.lowlevel import Server
+from mcp.server.lowlevel.server import StructuredContent, UnstructuredContent
 from mcp.types import Prompt, Resource, TextContent, Tool
 from prettyfmt import fmt_path
+from pydantic import BaseModel
 from strif import AtomicVar
 
 from kash.config.capture_output import CapturedOutput, captured_output
@@ -20,6 +22,7 @@ from kash.model.actions_model import Action, ActionResult
 from kash.model.exec_model import ExecContext
 from kash.model.params_model import TypedParamValues
 from kash.model.paths_model import StorePath
+from kash.utils.common.url import Url
 
 log = get_logger(__name__)
 
@@ -109,6 +112,22 @@ def get_published_tools() -> list[Tool]:
         return []
 
 
+class StructuredActionResult(BaseModel):
+    """
+    Error from an MCP tool call.
+    """
+
+    s3_paths: list[Url] | None = None
+    """If the tool created an S3 item, the S3 paths of the created items."""
+
+    error: str | None = None
+    """If the tool had an error, the error message."""
+
+    # TODO: Include other metadata.
+    # metadata: dict[str, Any] | None = None
+    # """Metadata about the action result."""
+
+
 @dataclass(frozen=True)
 class ToolResult:
     """
@@ -119,6 +138,7 @@ class ToolResult:
     captured_output: CapturedOutput
     action_result: ActionResult
     result_store_paths: list[StorePath]
+    result_s3_paths: list[Url]
     error: Exception | None = None
 
     @property
@@ -168,12 +188,13 @@ class ToolResult:
         # TODO: Add more info on how to find the logs.
         return "Check kash logs for details."
 
-    def formatted_for_client(self) -> list[TextContent]:
+    def as_mcp_content(self) -> tuple[UnstructuredContent, StructuredContent]:
         """
-        Convert the tool result to content for the client LLM.
+        Convert the tool result to content for the MCP client.
         """
+        structured = StructuredActionResult()
         if self.error:
-            return [
+            unstructured = [
                 TextContent(
                     text=f"The tool `{self.action.name}` had an error: {self.error}.\n\n"
                     + self.check_logs_message,
@@ -194,7 +215,7 @@ class ToolResult:
             if not chat_result:
                 chat_result = "No result. Check kash logs for details."
 
-            return [
+            unstructured = [
                 TextContent(
                     text=f"{self.output_summary}\n\n"
                     f"{self.output_content}\n\n"
@@ -202,10 +223,15 @@ class ToolResult:
                     type="text",
                 ),
             ]
+            structured = StructuredActionResult(s3_paths=self.result_s3_paths)
+
+        return unstructured, structured.model_dump()
 
 
 @log_calls(level="info")
-def run_mcp_tool(action_name: str, arguments: dict) -> list[TextContent]:
+def run_mcp_tool(
+    action_name: str, arguments: dict
+) -> tuple[UnstructuredContent, StructuredContent]:
     """
     Run the action as a tool.
     """
@@ -222,6 +248,7 @@ def run_mcp_tool(action_name: str, arguments: dict) -> list[TextContent]:
                 refetch=False,  # Using the file caches.
                 # Keeping all transient files for now, but maybe make transient?
                 override_state=None,
+                sync_to_s3=True,  # Enable S3 syncing for MCP tools.
             ) as exec_settings:
                 action_cls = look_up_action_class(action_name)
 
@@ -237,9 +264,9 @@ def run_mcp_tool(action_name: str, arguments: dict) -> list[TextContent]:
                 context = ExecContext(action=action, settings=exec_settings)
                 action_input = prepare_action_input(*input_items)
 
-                result, result_store_paths, _archived_store_paths = run_action_with_caching(
-                    context=context, action_input=action_input
-                )
+                result_with_paths = run_action_with_caching(context, action_input)
+                result = result_with_paths.result
+                result_store_paths = result_with_paths.result_paths
 
         # Return final result, formatted for the LLM to understand.
         return ToolResult(
@@ -247,8 +274,9 @@ def run_mcp_tool(action_name: str, arguments: dict) -> list[TextContent]:
             captured_output=capture.output,
             action_result=result,
             result_store_paths=result_store_paths,
+            result_s3_paths=result_with_paths.s3_paths,
             error=None,
-        ).formatted_for_client()
+        ).as_mcp_content()
 
     except Exception as e:
         log.exception("Error running mcp tool")
@@ -258,7 +286,7 @@ def run_mcp_tool(action_name: str, arguments: dict) -> list[TextContent]:
                 + "Check kash logs for details.",
                 type="text",
             )
-        ]
+        ], StructuredActionResult(error=str(e)).model_dump()
 
 
 def create_base_server() -> Server:
@@ -288,7 +316,9 @@ def create_base_server() -> Server:
         return []
 
     @app.call_tool()
-    async def handle_tool(name: str, arguments: dict) -> list[TextContent]:
+    async def handle_tool(
+        name: str, arguments: dict
+    ) -> tuple[UnstructuredContent, StructuredContent]:
         try:
             if name not in _mcp_published_actions.copy():
                 log.error(f"Unknown tool requested: {name}")
@@ -303,6 +333,6 @@ def create_base_server() -> Server:
                     text=f"Error executing tool {name}: {e}",
                     type="text",
                 )
-            ]
+            ], StructuredActionResult(error=str(e)).model_dump()
 
     return app

kash/model/actions_model.py

@@ -21,9 +21,8 @@ from kash.exec_model.args_model import NO_ARGS, ONE_ARG, ArgCount, ArgType, Sign
 from kash.exec_model.shell_model import ShellResult
 from kash.llm_utils import LLM, LLMName
 from kash.llm_utils.llm_messages import Message, MessageTemplate
-from kash.model.exec_model import ExecContext
+from kash.model.exec_model import ActionContext, ExecContext
 from kash.model.items_model import UNTITLED, Format, Item, ItemType
-from kash.model.operations_model import Operation, Source
 from kash.model.params_model import (
     ALL_COMMON_PARAMS,
     COMMON_SHELL_PARAMS,
@@ -61,6 +60,17 @@ class ActionInput:
         """An empty input, for when an action processes no items."""
         return ActionInput(items=[])
 
+    # XXX For convenience, we have the ability to include the context on each item
+    # (this helps soper-item functions don't have to take context args everywhere).
+    # TODO: Probably better to move this to a context var.
+    def set_context(self, context: ActionContext) -> None:
+        for item in self.items:
+            item.context = context
+
+    def clear_context(self) -> None:
+        for item in self.items:
+            item.context = None
+
 
 class PathOpType(Enum):
     archive = "archive"
@@ -111,6 +121,10 @@ class ActionResult:
             self.replaces_input or self.skip_duplicates or self.path_ops or self.shell_result
         )
 
+    def set_context(self, context: ActionContext) -> None:
+        for item in self.items:
+            item.context = context
+
     def clear_context(self):
         for item in self.items:
             item.context = None
@@ -232,7 +246,17 @@ class Action(ABC):
 
     output_type: ItemType = ItemType.doc
     """
-    The type of the output item(s), which for now are all assumed to be of the same type.
+    The type of the output item(s). If an action returns multiple output types,
+    this will be the output type of the first output.
+    This is mainly used for preassembly for the cache check if an output already exists.
+    """
+
+    output_format: Format | None = None
+    """
+    The format of the output item(s). The default is to assume it is the same
+    format as the input. If an action returns multiple output formats,
+    this will be the format of the first output.
+    This is mainly used for preassembly for the cache check if an output already exists.
     """
 
     expected_outputs: ArgCount = ONE_ARG
@@ -505,33 +529,17 @@ class Action(ABC):
                 fmt_lines(overrides),
             )
 
-    def _preassemble_one(
-        self,
-        operation: Operation,
-        input: ActionInput,
-        output_num: int,
-        type: ItemType,
-        **kwargs,
-    ) -> Item:
+    def format_title(self, prev_title: str | None) -> str:
         """
-        Preassemble a single empty output item from the given input items. Include the title,
-        type, and last Operation so we can do an identity check if the output already exists.
+        Format the title for an output item of this action.
         """
-        primary_input = input.items[output_num]
-        item = primary_input.derived_copy(type=type, body=None, **kwargs)
-
+        prev_title = prev_title or UNTITLED
         if self.title_template:
-            item.title = self.title_template.format(
-                title=primary_input.title or UNTITLED, action_name=self.name
-            )
-
-        item.update_history(
-            Source(operation=operation, output_num=output_num, cacheable=self.cacheable)
-        )
-
-        return item
+            return self.title_template.format(title=prev_title, action_name=self.name)
+        else:
+            return prev_title
 
-    def preassemble(self, operation: Operation, input: ActionInput) -> ActionResult | None:
+    def preassemble_result(self, context: ActionContext) -> ActionResult | None:
         """
         Actions can have a separate preliminary step to pre-assemble outputs. This allows
         us to determine the title and types for the output items and check if they were
@@ -542,16 +550,19 @@ class Action(ABC):
         """
         can_preassemble = self.cacheable and self.expected_outputs == ONE_ARG
         log.info(
-            "Preassemble check for `%s` is %s (%s with cacheable=%s)",
+            "Preassemble check for `%s`: can_preassemble=%s (expected_outputs=%s, cacheable=%s)",
             self.name,
             can_preassemble,
             self.expected_outputs,
             self.cacheable,
         )
         if can_preassemble:
-            return ActionResult(
-                [self._preassemble_one(operation, input, output_num=0, type=self.output_type)]
-            )
+            # Using first input to determine the output title.
+            primary_input = context.action_input.items[0]
+            # In this case we only expect one output, of the type specified by the action.
+            primary_output = primary_input.derived_copy(context, 0, type=context.action.output_type)
+            log.info("Preassembled output: source %s, %s", primary_output.source, primary_output)
+            return ActionResult([primary_output])
         else:
             # Caching disabled.
             return None
@@ -600,7 +611,7 @@ class Action(ABC):
         return input_schema
 
     @abstractmethod
-    def run(self, input: ActionInput, context: ExecContext) -> ActionResult:
+    def run(self, input: ActionInput, context: ActionContext) -> ActionResult:
         pass
 
     def __repr__(self):
@@ -626,7 +637,7 @@ class PerItemAction(Action, ABC):
         super().__post_init__()
 
     @override
-    def run(self, input: ActionInput, context: ExecContext) -> ActionResult:
+    def run(self, input: ActionInput, context: ActionContext) -> ActionResult:
         log.info("Running action `%s` per-item.", self.name)
         for item in input.items:
             item.context = context
@@ -642,3 +653,4 @@ class PerItemAction(Action, ABC):
 # Handle circular dependency in Python dataclasses.
 rebuild_dataclass(Item)  # pyright: ignore
 rebuild_dataclass(ExecContext)  # pyright: ignore
+rebuild_dataclass(ActionContext)  # pyright: ignore