kash-shell 0.3.11__py3-none-any.whl → 0.3.13__py3-none-any.whl

kash/media_base/media_cache.py

@@ -1,4 +1,5 @@
 import os
+from functools import cache
 from pathlib import Path
 
 from prettyfmt import fmt_lines, fmt_path
@@ -11,7 +12,6 @@ from kash.media_base.media_services import (
     download_media_by_service,
     get_media_services,
 )
-from kash.media_base.transcription_deepgram import deepgram_transcribe_audio
 from kash.utils.common.format_utils import fmt_loc
 from kash.utils.common.url import Url, as_file_url, is_url
 from kash.utils.errors import FileNotFound, InvalidInput, UnexpectedError
@@ -22,7 +22,14 @@ log = get_logger(__name__)
 
 # FIXME: Hard-coded dependency for now. Would be better to make it settable.
 # transcribe_audio = whisper_transcribe_audio_small
-transcribe_audio = deepgram_transcribe_audio
+
+
+@cache
+def get_transcriber():
+    from kash.media_base.transcription_deepgram import deepgram_transcribe_audio
+
+    transcribe_audio = deepgram_transcribe_audio
+    return transcribe_audio
 
 
 # For simplicity we assume all audio is converted to mp3.
@@ -83,7 +90,7 @@ class MediaCache(DirStore):
             url,
             fmt_path(downsampled_audio_file),
         )
-        transcript = transcribe_audio(downsampled_audio_file, language=language)
+        transcript = get_transcriber()(downsampled_audio_file, language=language)
         self._write_transcript(url, transcript)
         return transcript
 

kash/media_base/transcription_deepgram.py

@@ -1,8 +1,10 @@
+from __future__ import annotations
+
 from os.path import getsize
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 from clideps.env_vars.dotenv_utils import load_dotenv_paths
-from deepgram import ListenRESTClient, PrerecordedResponse
 from httpx import Timeout
 
 from kash.config.logger import CustomLogger, get_logger
@@ -10,6 +12,9 @@ from kash.config.settings import global_settings
 from kash.media_base.transcription_format import SpeakerSegment, format_speaker_segments
 from kash.utils.errors import ApiError, ContentError
 
+if TYPE_CHECKING:
+    from deepgram import PrerecordedResponse
+
 log: CustomLogger = get_logger(__name__)
 
 
@@ -19,7 +24,15 @@ def deepgram_transcribe_raw(
     """
     Transcribe an audio file using Deepgram and return the raw response.
     """
-    from deepgram import ClientOptionsFromEnv, DeepgramClient, FileSource, PrerecordedOptions
+    # Slow import, do lazily.
+    from deepgram import (
+        ClientOptionsFromEnv,
+        DeepgramClient,
+        FileSource,
+        ListenRESTClient,
+        PrerecordedOptions,
+        PrerecordedResponse,
+    )
 
     size = getsize(audio_file_path)
     log.info(

kash/model/__init__.py

@@ -20,7 +20,6 @@ from kash.model.actions_model import (
     Action,
     ActionInput,
     ActionResult,
-    ExecContext,
     LLMOptions,
     PathOp,
     PathOpType,
@@ -33,6 +32,7 @@ from kash.model.compound_actions_model import (
     look_up_actions,
 )
 from kash.model.concept_model import Concept, canonicalize_concept, normalize_concepts
+from kash.model.exec_model import ExecContext
 from kash.model.graph_model import GraphData, Link, Node
 from kash.model.items_model import (
     SLUG_MAX_LEN,

kash/model/actions_model.py

@@ -4,7 +4,6 @@ from abc import ABC, abstractmethod
 from dataclasses import Field as DataclassField
 from dataclasses import field, replace
 from enum import Enum
-from pathlib import Path
 from textwrap import dedent
 from typing import Any, TypeVar, cast
 
@@ -20,10 +19,10 @@ from typing_extensions import override
 from kash.config.logger import get_logger
 from kash.exec_model.args_model import NO_ARGS, ONE_ARG, ArgCount, ArgType, Signature
 from kash.exec_model.shell_model import ShellResult
-from kash.file_storage.file_store import FileStore
 from kash.llm_utils import LLM, LLMName
 from kash.llm_utils.llm_messages import Message, MessageTemplate
-from kash.model.items_model import UNTITLED, Item, ItemType, State
+from kash.model.exec_model import ExecContext
+from kash.model.items_model import UNTITLED, Item, ItemType
 from kash.model.operations_model import Operation, Source
 from kash.model.params_model import (
     ALL_COMMON_PARAMS,
@@ -38,7 +37,6 @@ from kash.model.preconditions_model import Precondition
 from kash.utils.common.parse_key_vals import format_key_value
 from kash.utils.common.type_utils import not_none
 from kash.utils.errors import InvalidDefinition, InvalidInput
-from kash.workspaces.workspaces import get_ws
 
 log = get_logger(__name__)
 
@@ -64,53 +62,6 @@ class ActionInput:
         return ActionInput(items=[])
 
 
-@dataclass(frozen=True)
-class ExecContext:
-    """
-    An action and its context for execution. This is a good place for settings
-    that apply to any action and are bothersome to pass as parameters.
-    """
-
-    action: Action
-    """The action being executed."""
-
-    workspace_dir: Path
-    """The workspace directory in which the action is being executed."""
-
-    rerun: bool = False
-    """If True, always run actions, even cacheable ones that have results."""
-
-    refetch: bool = False
-    """If True, will refetch items even if they are already in the content caches."""
-
-    override_state: State | None = None
-    """If specified, override the state of result items. Useful to mark items as transient."""
-
-    tmp_output: bool = False
-    """If True, will save output items to a temporary file."""
-
-    no_format: bool = False
-    """If True, will not normalize the output item's body text formatting (for Markdown)."""
-
-    @property
-    def workspace(self) -> FileStore:
-        return get_ws(self.workspace_dir)
-
-    @property
-    def runtime_options(self) -> dict[str, str]:
-        """Return non-default runtime options."""
-        opts: dict[str, str] = {}
-        # Only these two settings directly affect the output:
-        if self.no_format:
-            opts["no_format"] = "true"
-        if self.override_state:
-            opts["override_state"] = self.override_state.name
-        return opts
-
-    def __repr__(self):
-        return abbrev_obj(self, field_max_len=80)
-
-
 class PathOpType(Enum):
     archive = "archive"
     select = "select"
@@ -139,6 +90,9 @@ class ActionResult:
     replaces_input: bool = False
     """If True, a hint to archive the input items."""
 
+    overwrite: bool = False
+    """If True, will not pick unique output paths to save to, overwriting existing files of the same name."""
+
     skip_duplicates: bool = False
     """If True, do not save duplicate items (based on identity)."""
 
@@ -365,8 +319,8 @@ class Action(ABC):
         """
         Declaration sanity checks.
         """
-        if not self.name or not self.description:
-            raise InvalidDefinition("Action must have a name and description")
+        if not self.name:
+            raise InvalidDefinition("Action must have a name")
 
         for param in self.params:
             if not self.has_param(param.name):
@@ -535,7 +489,7 @@ class Action(ABC):
                 log.info("Ignoring parameter for action `%s`: `%s`", self.name, param_name)
 
         if overrides:
-            log.message(
+            log.info(
                 "Overriding parameters for action `%s`:\n%s",
                 self.name,
                 fmt_lines(overrides),
@@ -677,3 +631,4 @@ class PerItemAction(Action, ABC):
 
 # Handle circular dependency in Python dataclasses.
 rebuild_dataclass(Item)  # pyright: ignore
+rebuild_dataclass(ExecContext)  # pyright: ignore

kash/model/exec_model.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from prettyfmt import abbrev_obj
+from pydantic.dataclasses import dataclass
+
+from kash.config.logger import get_logger
+from kash.model.items_model import State
+
+if TYPE_CHECKING:
+    from kash.file_storage.file_store import FileStore
+    from kash.model.actions_model import Action
+
+
+log = get_logger(__name__)
+
+
+@dataclass(frozen=True)
+class RuntimeSettings:
+    """
+    Workspace and other runtime settings that may be set across runs of
+    one or more actions.
+    """
+
+    workspace_dir: Path
+    """The workspace directory in which the action is being executed."""
+
+    rerun: bool = False
+    """If True, always run actions, even cacheable ones that have results."""
+
+    refetch: bool = False
+    """If True, will refetch items even if they are already in the content caches."""
+
+    override_state: State | None = None
+    """If specified, override the state of result items. Useful to mark items as transient."""
+
+    tmp_output: bool = False
+    """If True, will save output items to a temporary file."""
+
+    no_format: bool = False
+    """If True, will not normalize the output item's body text formatting (for Markdown)."""
+
+    @property
+    def workspace(self) -> FileStore:
+        from kash.workspaces.workspaces import get_ws
+
+        return get_ws(self.workspace_dir)
+
+    @property
+    def non_default_options(self) -> dict[str, str]:
+        """
+        Summarize non-default runtime options as a dict.
+        """
+        opts: dict[str, str] = {}
+        # Only these two settings directly affect the output:
+        if self.no_format:
+            opts["no_format"] = "true"
+        if self.override_state:
+            opts["override_state"] = self.override_state.name
+        return opts
+
+    def __repr__(self):
+        return abbrev_obj(self, field_max_len=80)
+
+
+@dataclass(frozen=True)
+class ExecContext:
+    """
+    An action and its context for execution. This is a good place for settings
+    that apply to any action and are bothersome to pass as parameters.
+    """
+
+    action: Action
+    """The action being executed."""
+
+    settings: RuntimeSettings
+    """The workspace and other run-time settings for the action."""

kash/model/items_model.py

@@ -24,15 +24,17 @@ from kash.model.concept_model import canonicalize_concept
 from kash.model.media_model import MediaMetadata
 from kash.model.operations_model import OperationSummary, Source
 from kash.model.paths_model import StorePath, fmt_store_path
-from kash.text_handling.markdown_render import markdown_to_html
 from kash.utils.common.format_utils import fmt_loc, html_to_plaintext, plaintext_to_html
 from kash.utils.common.url import Locator, Url
 from kash.utils.errors import FileFormatError
 from kash.utils.file_formats.chat_format import ChatHistory
+from kash.utils.file_utils.file_formats import MimeType
 from kash.utils.file_utils.file_formats_model import FileExt, Format
+from kash.utils.text_handling.markdown_render import markdown_to_html
+from kash.utils.text_handling.markdown_utils import first_heading
 
 if TYPE_CHECKING:
-    from kash.model.actions_model import ExecContext
+    from kash.model.exec_model import ExecContext
     from kash.workspaces import Workspace
 
 log = get_logger(__name__)
@@ -178,9 +180,7 @@ class ItemId:
         if item.type == ItemType.resource and item.format == Format.url and item.url:
             item_id = ItemId(item.type, IdType.url, canonicalize_url(item.url))
         elif item.type == ItemType.concept and item.title:
-            item_id = ItemId(
-                item.type, IdType.concept, canonicalize_concept(item.title)
-            )
+            item_id = ItemId(item.type, IdType.concept, canonicalize_concept(item.title))
         elif item.source and item.source.cacheable:
             # We know the source of this and if the action was cacheable, we can create
             # an identity based on the source.
@@ -281,11 +281,9 @@ class Item:
         """
         item_dict = {**item_dict, **kwargs}
 
-        info_prefix = (
-            f"{fmt_store_path(item_dict['store_path'])}: "
-            if "store_path" in item_dict
-            else ""
-        )
+        info_prefix = ""
+        if "store_path" in item_dict and item_dict["store_path"]:
+            info_prefix = f"{fmt_store_path(item_dict['store_path'])}: "
 
         # Metadata formats might change over time so it's important to gracefully handle issues.
         def set_field(key: str, default: Any, cls_: type[T]) -> T:
@@ -314,9 +312,7 @@ class Item:
         body = item_dict.get("body")
         history = [OperationSummary(**op) for op in item_dict.get("history", [])]
         relations = (
-            ItemRelations(**item_dict["relations"])
-            if "relations" in item_dict
-            else ItemRelations()
+            ItemRelations(**item_dict["relations"]) if "relations" in item_dict else ItemRelations()
         )
         store_path = item_dict.get("store_path")
 
@@ -334,9 +330,7 @@ class Item:
         ]
         all_fields = [f.name for f in cls.__dataclass_fields__.values()]
         allowed_fields = [f for f in all_fields if f not in excluded_fields]
-        other_metadata = {
-            key: value for key, value in item_dict.items() if key in allowed_fields
-        }
+        other_metadata = {key: value for key, value in item_dict.items() if key in allowed_fields}
         unexpected_metadata = {
             key: value for key, value in item_dict.items() if key not in all_fields
         }
@@ -366,15 +360,19 @@ class Item:
         cls,
         path: Path | str,
         item_type: ItemType | None = None,
+        *,
         title: str | None = None,
+        original_filename: str | None = None,
+        mime_type: MimeType | None = None,
     ) -> Item:
         """
         Create a resource Item for a file with a format inferred from the file extension
         or the content. Only sets basic metadata. Does not read the content. Will set
         `format` and `file_ext` if possible but will leave them as None if unrecognized.
+        If `mime_type` is provided, it can help determine the file extension.
         """
         from kash.file_storage.store_filenames import parse_item_filename
-        from kash.utils.file_utils.file_formats_model import detect_file_format
+        from kash.utils.file_utils.file_formats_model import choose_file_ext, detect_file_format
 
         # Will raise error for unrecognized file ext.
         _name, filename_item_type, format, file_ext = parse_item_filename(path)
@@ -385,16 +383,19 @@ class Item:
         if not item_type:
             # Default to doc for general text files and resource for everything else.
             item_type = (
-                ItemType.doc
-                if format and format.supports_frontmatter
-                else ItemType.resource
+                ItemType.doc if format and format.supports_frontmatter else ItemType.resource
             )
+        # Do our best to determine a good file extension if it's not already on the filename.
+        if not file_ext and mime_type:
+            file_ext = choose_file_ext(path, mime_type)
+
         item = cls(
             type=item_type,
             title=title,
             file_ext=file_ext,
             format=format,
             external_path=str(path),
+            original_filename=original_filename,
         )
 
         # Update modified time from the file system.
@@ -438,11 +439,9 @@ class Item:
         if not self.format:
             raise ValueError(f"Item has no format: {self}")
         if self.type.expects_body and self.format.has_body and not self.body:
-            raise ValueError(
-                f"Item type `{self.type.value}` is text but has no body: {self}"
-            )
+            raise ValueError(f"Item type `{self.type.value}` is text but has no body: {self}")
 
-    def absolute_path(self, ws: "Workspace | None" = None) -> Path:  # noqa: UP037
+    def absolute_path(self, ws: Workspace | None = None) -> Path:
         """
         Get the absolute path to the item. Throws `ValueError` if the item has no
         store path. If no workspace is provided, uses the current workspace.
@@ -493,9 +492,7 @@ class Item:
                 return {k: serialize(v) for k, v in v.items()}
             elif isinstance(v, Enum):
                 return v.value
-            elif hasattr(
-                v, "as_dict"
-            ):  # Handle Operation or any object with as_dict method.
+            elif hasattr(v, "as_dict"):  # Handle Operation or any object with as_dict method.
                 return v.as_dict()
             elif is_dataclass(v) and not isinstance(v, type):
                 # Handle Python and Pydantic dataclasses.
@@ -520,49 +517,87 @@ class Item:
 
         return item_dict
 
-    def display_title(self) -> str:
+    def filename_stem(self) -> str | None:
         """
-        A display title for this item. Same as abbrev_title() but will fall back
-        to the filename if it is available.
+        If the item has an existing or previous filename, return its stem,
+        for use in picking new filenames.
         """
-        display_title = self.title
-        if not display_title and self.store_path:
-            display_title = Path(self.store_path).name
-        if not display_title:
-            display_title = self.abbrev_title()
-        return display_title
+        from kash.file_storage.store_filenames import parse_item_filename
 
-    def abbrev_title(self, max_len: int = 100, add_ops_suffix: bool = True) -> str:
+        # Prefer original to external, e.g. if we know the original but the external might
+        # be a cache filename.
+        path = self.store_path or self.original_filename or self.external_path
+        if path:
+            path_name, _item_type, _format, _file_ext = parse_item_filename(Path(path).name)
+        else:
+            path_name = None
+        return path_name
+
+    def slug_name(self, max_len: int = SLUG_MAX_LEN, prefer_title: bool = False) -> str:
+        """
+        Get a readable slugified name for this item, either from a previous filename
+        or from slugifying the title or content. May not be unique.
+        """
+        filename_stem = self.filename_stem()
+        if filename_stem and not prefer_title:
+            return slugify_snake(filename_stem)
+        else:
+            return slugify_snake(self.abbrev_title(max_len=max_len, add_ops_suffix=True))
+
+    def default_filename(self) -> str:
         """
-        Get or infer a title for this item, falling back to the filename, URL,
-        description, or finally body text.
-        Optionally, include the last operation as a parenthetical at the end of the title.
+        Get the default filename for an item based on slugifying its title or other
+        metadata. May not be unique.
         """
-        # Special case for URLs with no title..
+        from kash.file_storage.store_filenames import join_suffix
+
+        slug = self.slug_name()
+        full_suffix = self.get_full_suffix()
+        return join_suffix(slug, full_suffix)
+
+    def abbrev_title(
+        self,
+        *,
+        max_len: int = 100,
+        add_ops_suffix: bool = False,
+        pull_body_heading: bool = False,
+    ) -> str:
+        """
+        Get or infer a title for this item, falling back to the filename, URL, description, or
+        finally body text. Optionally, include the last operation as a parenthetical at the end
+        of the title. Will use "Untitled" if all else fails.
+        """
+        # First special case: if we are pulling the title from the body header, check
+        # that.
+        if not self.title and pull_body_heading:
+            heading = self.body_heading()
+            if heading:
+                return heading
+
+        # Next special case: URLs with no title use the url itself.
         if not self.title and self.url:
             return abbrev_str(self.url, max_len)
 
-        # Special case for filenames with no title.
-        path_name = (
-            (self.store_path and Path(self.store_path).name)
-            or (self.external_path and Path(self.external_path).name)
-            or (self.original_filename and Path(self.original_filename).name)
-        )
+        filename_stem = self.filename_stem()
 
         # Use the title or the path if possible, falling back to description or even body text.
         title_raw_text = (
             self.title
-            or path_name
+            or filename_stem
             or self.description
             or (not self.is_binary and self.abbrev_body(max_len))
             or UNTITLED
         )
 
         suffix = ""
-        if add_ops_suffix and self.type not in [ItemType.concept, ItemType.resource]:
-            # For notes, exports, etc but not for concepts, add a parenthical note
-            # indicating the last operation, if there was one. This makes filename slugs
-            # more readable.
+        # For docs, etc but not for concepts/resources/exports, add a parenthical note
+        # indicating the last operation, if there was one. This makes filename slugs
+        # more readable.
+        if add_ops_suffix and self.type not in [
+            ItemType.concept,
+            ItemType.resource,
+            ItemType.export,
+        ]:
             last_op = self.history and self.history[-1].action_name
             if last_op:
                 step_num = len(self.history) + 1 if self.history else 1
@@ -579,9 +614,36 @@ class Item:
 
         return final_text
 
+    def display_title(self) -> str:
+        """
+        A display title for this item. Same as abbrev_title() but will fall back
+        to the filename if it is available.
+        """
+        display_title = self.title
+        if not display_title and self.store_path:
+            display_title = Path(self.store_path).name
+        if not display_title:
+            display_title = self.abbrev_title()
+        return display_title
+
+    def abbrev_description(self, max_len: int = 1000) -> str:
+        """
+        Get or infer description.
+        """
+        return abbrev_on_words(html_to_plaintext(self.description or self.body or ""), max_len)
+
+    def body_heading(self) -> str | None:
+        """
+        Get the first h1 or h2 heading from the body text, if present.
+        """
+        if self.format in [Format.markdown, Format.md_html]:
+            return first_heading(self.body_text(), allowed_tags=("h1", "h2"))
+        # TODO: Support HTML <h1> and <h2> as well.
+        return None
+
     def abbrev_body(self, max_len: int) -> str:
         """
-        Get a cut off version of the body text. Must not be a binary Item.
+        Get an abbreviated version of the body text. Must not be a binary Item.
         Abbreviates YAML bodies like {"role": "user", "content": "Hello"} to "user Hello".
         """
         body_text = self.body_text()[:max_len]
@@ -604,23 +666,6 @@ class Item:
         """
         return bool(self.body and self.body.strip())
 
-    def slug_name(self, max_len: int = SLUG_MAX_LEN) -> str:
-        """
-        Get a readable slugified version of the title or filename or content
-        appropriate for this item. May not be unique.
-        """
-        title = self.abbrev_title(max_len=max_len)
-        slug = slugify_snake(title)
-        return slug
-
-    def abbrev_description(self, max_len: int = 1000) -> str:
-        """
-        Get or infer description.
-        """
-        return abbrev_on_words(
-            html_to_plaintext(self.description or self.body or ""), max_len
-        )
-
     def read_as_config(self) -> Any:
         """
         If it is a config Item, return the parsed YAML.
@@ -639,8 +684,6 @@ class Item:
         """
         if self.file_ext:
             return self.file_ext
-        if self.is_binary and not self.file_ext:
-            raise ValueError(f"Binary Items must have a file extension: {self}")
         inferred_ext = self.format and self.format.file_ext
         if not inferred_ext:
             raise ValueError(f"Cannot infer file extension for Item: {self}")
@@ -656,6 +699,9 @@ class Item:
         elif self.type == ItemType.script:
             # Same for kash/xonsh scripts.
             return f"{self.type.value}.{FileExt.xsh.value}"
+        elif self.type == ItemType.export:
+            # For exports, skip the item type to keep it maximally compatible for external tools.
+            return f"{self.get_file_ext().value}"
         else:
             return f"{self.type.value}.{self.get_file_ext().value}"
 
@@ -668,11 +714,19 @@ class Item:
         return "\n\n".join(part for part in parts if part)
 
     def body_text(self) -> str:
+        """
+        Body text of the item, also validating that the item is not binary.
+        """
         if self.is_binary:
             raise ValueError("Cannot get text content of a binary Item")
         return self.body or ""
 
     def body_as_html(self) -> str:
+        """
+        Body of the item, converted to HTML format. Validates that the body format can be
+        converted and then converts plaintext or Markdown to HTML. Simply returns the body
+        if it is already HTML.
+        """
         if self.format == Format.html:
             return self.body_text()
         elif self.format == Format.plaintext:
@@ -708,12 +762,10 @@ class Item:
         self, update_timestamp: bool = True, **other_updates: Unpack[ItemUpdateOptions]
     ) -> Item:
         """
-        Copy item with the given field updates. Resets store_path to None. Updates
-        created time if requested.
+        Copy item with the given field updates. Resets `store_path` to None but preserves
+        other fields, including the body. Updates created time if requested.
         """
-        new_fields = self._copy_and_update(
-            update_timestamp=update_timestamp, **other_updates
-        )
+        new_fields = self._copy_and_update(update_timestamp=update_timestamp, **other_updates)
         return Item(**new_fields)
 
     def merged_copy(self, other: Item) -> Item:
@@ -734,7 +786,7 @@ class Item:
             if self.relations.derived_from:
                 log.message(
                     "Deriving from an item that has not been saved so using "
-                    "its derived_from relation: %s on %s",
+                    "upstream derived_from relation: %s on %s",
                     self.relations.derived_from,
                     self,
                 )
@@ -768,9 +820,7 @@ class Item:
 
         # Fall back to action title template if we have it and title wasn't explicitly set.
         if "title" not in updates:
-            prev_title = self.title or (
-                Path(self.store_path).stem if self.store_path else UNTITLED
-            )
+            prev_title = self.title or (Path(self.store_path).stem if self.store_path else UNTITLED)
             if self.context:
                 action = self.context.action
                 new_item.title = action.title_template.format(