kash-shell 0.3.9__py3-none-any.whl → 0.3.11__py3-none-any.whl

kash/model/actions_model.py

@@ -9,12 +9,12 @@ from textwrap import dedent
 from typing import Any, TypeVar, cast
 
 from chopdiff.docs import DiffFilter
-from chopdiff.docs.token_diffs import DIFF_FILTER_NONE
 from chopdiff.transforms import WINDOW_NONE, WindowSettings
 from flowmark import fill_text
 from prettyfmt import abbrev_obj, fmt_lines
 from pydantic.dataclasses import dataclass, rebuild_dataclass
 from pydantic.json_schema import JsonSchemaValue
+from strif import StringTemplate
 from typing_extensions import override
 
 from kash.config.logger import get_logger
@@ -27,13 +27,15 @@ from kash.model.items_model import UNTITLED, Item, ItemType, State
 from kash.model.operations_model import Operation, Source
 from kash.model.params_model import (
     ALL_COMMON_PARAMS,
+    COMMON_SHELL_PARAMS,
+    RUNTIME_ACTION_PARAMS,
+    Param,
     ParamDeclarations,
     TypedParamValues,
 )
 from kash.model.paths_model import StorePath
 from kash.model.preconditions_model import Precondition
 from kash.utils.common.parse_key_vals import format_key_value
-from kash.utils.common.string_template import StringTemplate
 from kash.utils.common.type_utils import not_none
 from kash.utils.errors import InvalidDefinition, InvalidInput
 from kash.workspaces.workspaces import get_ws
@@ -65,7 +67,8 @@ class ActionInput:
 @dataclass(frozen=True)
 class ExecContext:
     """
-    An action and its context for execution.
+    An action and its context for execution. This is a good place for settings
+    that apply to any action and are bothersome to pass as parameters.
     """
 
     action: Action
@@ -77,13 +80,33 @@ class ExecContext:
     rerun: bool = False
     """If True, always run actions, even cacheable ones that have results."""
 
+    refetch: bool = False
+    """If True, will refetch items even if they are already in the content caches."""
+
     override_state: State | None = None
     """If specified, override the state of result items. Useful to mark items as transient."""
 
+    tmp_output: bool = False
+    """If True, will save output items to a temporary file."""
+
+    no_format: bool = False
+    """If True, will not normalize the output item's body text formatting (for Markdown)."""
+
     @property
     def workspace(self) -> FileStore:
         return get_ws(self.workspace_dir)
 
+    @property
+    def runtime_options(self) -> dict[str, str]:
+        """Return non-default runtime options."""
+        opts: dict[str, str] = {}
+        # Only these two settings directly affect the output:
+        if self.no_format:
+            opts["no_format"] = "true"
+        if self.override_state:
+            opts["override_state"] = self.override_state.name
+        return opts
+
     def __repr__(self):
         return abbrev_obj(self, field_max_len=80)
 
@@ -175,7 +198,7 @@ class LLMOptions:
     system_message: Message = Message("")
     body_template: MessageTemplate = MessageTemplate("{body}")
     windowing: WindowSettings = WINDOW_NONE
-    diff_filter: DiffFilter = DIFF_FILTER_NONE
+    diff_filter: DiffFilter | None = None
 
     def updated_with(self, param_name: str, value: Any) -> LLMOptions:
         """Update option from an action parameter."""
@@ -409,6 +432,17 @@ class Action(ABC):
         # Update corresponding LLM option if appropriate.
         self.llm_options = self.llm_options.updated_with(param_name, value)
 
+    @property
+    def shell_params(self) -> list[Param]:
+        """
+        List of parameters that are relevant to shell usage.
+        """
+        return (
+            list(self.params)
+            + list(RUNTIME_ACTION_PARAMS.values())
+            + list(COMMON_SHELL_PARAMS.values())
+        )
+
     def param_value_summary(self) -> dict[str, str]:
         """
         Readable, serializable summary of the action's non-default parameters, to include in

kash/model/concept_model.py

@@ -0,0 +1,30 @@
+from typing import NewType
+
+from kash.utils.lang_utils.capitalization import capitalize_cms
+
+Concept = NewType("Concept", str)
+
+
+def canonicalize_concept(concept: str, capitalize: bool = True) -> Concept:
+    """
+    Convert a concept string (general name, person, etc.) to a canonical form.
+    Drop any extraneous Markdown bullets. Drop any quoted phrases (e.g. book titles etc)
+    for consistency.
+    """
+    concept = concept.strip("-* ")
+    for quote in ['"', "'"]:
+        if concept.startswith(quote) and concept.endswith(quote):
+            concept = concept[1:-1]
+    if capitalize:
+        return Concept(capitalize_cms(concept))
+    else:
+        return Concept(concept)
+
+
+def normalize_concepts(
+    concepts: list[str], sort_dedup: bool = True, capitalize: bool = True
+) -> list[Concept]:
+    if sort_dedup:
+        return sorted(set(canonicalize_concept(concept, capitalize) for concept in concepts))
+    else:
+        return [canonicalize_concept(concept, capitalize) for concept in concepts]

kash/model/items_model.py

@@ -6,7 +6,7 @@ from dataclasses import asdict, field, is_dataclass
 from datetime import UTC, datetime
 from enum import Enum
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, TypeVar
+from typing import TYPE_CHECKING, Any, NotRequired, TypedDict, TypeVar, Unpack
 
 from frontmatter_format import from_yaml_string, new_yaml
 from prettyfmt import (
@@ -19,12 +19,12 @@ from prettyfmt import (
 from pydantic.dataclasses import dataclass
 from strif import abbrev_str, format_iso_timestamp
 
-from kash.concepts.concept_formats import canonicalize_concept
 from kash.config.logger import get_logger
+from kash.model.concept_model import canonicalize_concept
 from kash.model.media_model import MediaMetadata
 from kash.model.operations_model import OperationSummary, Source
 from kash.model.paths_model import StorePath, fmt_store_path
-from kash.text_handling.markdown_util import markdown_to_html
+from kash.text_handling.markdown_render import markdown_to_html
 from kash.utils.common.format_utils import fmt_loc, html_to_plaintext, plaintext_to_html
 from kash.utils.common.url import Locator, Url
 from kash.utils.errors import FileFormatError
@@ -33,6 +33,7 @@ from kash.utils.file_utils.file_formats_model import FileExt, Format
 
 if TYPE_CHECKING:
     from kash.model.actions_model import ExecContext
+    from kash.workspaces import Workspace
 
 log = get_logger(__name__)
 
@@ -119,6 +120,28 @@ class IdType(Enum):
     source = "source"
 
 
+class ItemUpdateOptions(TypedDict, total=False):
+    """
+    Keyword arguments that can be passed to update an Item.
+    """
+
+    type: NotRequired[ItemType]
+    state: NotRequired[State]
+    title: NotRequired[str | None]
+    url: NotRequired[Url | None]
+    description: NotRequired[str | None]
+    format: NotRequired[Format | None]
+    file_ext: NotRequired[FileExt | None]
+    body: NotRequired[str | None]
+    external_path: NotRequired[str | None]
+    original_filename: NotRequired[str | None]
+    relations: NotRequired[ItemRelations]
+    source: NotRequired[Source | None]
+    history: NotRequired[list[OperationSummary] | None]
+    thumbnail_url: NotRequired[Url | None]
+    extra: NotRequired[dict | None]
+
+
 @dataclass(frozen=True)
 class ItemId:
     """
@@ -155,7 +178,9 @@ class ItemId:
         if item.type == ItemType.resource and item.format == Format.url and item.url:
             item_id = ItemId(item.type, IdType.url, canonicalize_url(item.url))
         elif item.type == ItemType.concept and item.title:
-            item_id = ItemId(item.type, IdType.concept, canonicalize_concept(item.title))
+            item_id = ItemId(
+                item.type, IdType.concept, canonicalize_concept(item.title)
+            )
         elif item.source and item.source.cacheable:
             # We know the source of this and if the action was cacheable, we can create
             # an identity based on the source.
@@ -257,7 +282,9 @@ class Item:
         item_dict = {**item_dict, **kwargs}
 
         info_prefix = (
-            f"{fmt_store_path(item_dict['store_path'])}: " if "store_path" in item_dict else ""
+            f"{fmt_store_path(item_dict['store_path'])}: "
+            if "store_path" in item_dict
+            else ""
         )
 
         # Metadata formats might change over time so it's important to gracefully handle issues.
@@ -287,7 +314,9 @@ class Item:
         body = item_dict.get("body")
         history = [OperationSummary(**op) for op in item_dict.get("history", [])]
         relations = (
-            ItemRelations(**item_dict["relations"]) if "relations" in item_dict else ItemRelations()
+            ItemRelations(**item_dict["relations"])
+            if "relations" in item_dict
+            else ItemRelations()
         )
         store_path = item_dict.get("store_path")
 
@@ -305,12 +334,18 @@ class Item:
         ]
         all_fields = [f.name for f in cls.__dataclass_fields__.values()]
         allowed_fields = [f for f in all_fields if f not in excluded_fields]
-        other_metadata = {key: value for key, value in item_dict.items() if key in allowed_fields}
+        other_metadata = {
+            key: value for key, value in item_dict.items() if key in allowed_fields
+        }
         unexpected_metadata = {
             key: value for key, value in item_dict.items() if key not in all_fields
         }
         if unexpected_metadata:
-            log.info("Skipping unexpected metadata on item: %s%s", info_prefix, unexpected_metadata)
+            log.info(
+                "Skipping unexpected metadata on item: %s%s",
+                info_prefix,
+                unexpected_metadata,
+            )
 
         result = cls(
             type=type_,
@@ -328,7 +363,10 @@ class Item:
 
     @classmethod
     def from_external_path(
-        cls, path: Path | str, item_type: ItemType | None = None, title: str | None = None
+        cls,
+        path: Path | str,
+        item_type: ItemType | None = None,
+        title: str | None = None,
     ) -> Item:
         """
         Create a resource Item for a file with a format inferred from the file extension
@@ -347,7 +385,9 @@ class Item:
         if not item_type:
             # Default to doc for general text files and resource for everything else.
             item_type = (
-                ItemType.doc if format and format.supports_frontmatter else ItemType.resource
+                ItemType.doc
+                if format and format.supports_frontmatter
+                else ItemType.resource
             )
         item = cls(
             type=item_type,
@@ -398,7 +438,22 @@ class Item:
         if not self.format:
             raise ValueError(f"Item has no format: {self}")
         if self.type.expects_body and self.format.has_body and not self.body:
-            raise ValueError(f"Item type `{self.type.value}` is text but has no body: {self}")
+            raise ValueError(
+                f"Item type `{self.type.value}` is text but has no body: {self}"
+            )
+
+    def absolute_path(self, ws: "Workspace | None" = None) -> Path:  # noqa: UP037
+        """
+        Get the absolute path to the item. Throws `ValueError` if the item has no
+        store path. If no workspace is provided, uses the current workspace.
+        """
+        from kash.workspaces import current_ws
+
+        if not self.store_path:
+            raise ValueError("Item has no store path")
+        if not ws:
+            ws = current_ws()
+        return ws.base_dir / self.store_path
 
     @property
     def is_binary(self) -> bool:
@@ -438,7 +493,9 @@ class Item:
                 return {k: serialize(v) for k, v in v.items()}
             elif isinstance(v, Enum):
                 return v.value
-            elif hasattr(v, "as_dict"):  # Handle Operation or any object with as_dict method.
+            elif hasattr(
+                v, "as_dict"
+            ):  # Handle Operation or any object with as_dict method.
                 return v.as_dict()
             elif is_dataclass(v) and not isinstance(v, type):
                 # Handle Python and Pydantic dataclasses.
@@ -486,17 +543,16 @@ class Item:
             return abbrev_str(self.url, max_len)
 
         # Special case for filenames with no title.
-        path_stem = (
-            (self.store_path and Path(self.store_path).stem)
-            or (self.external_path and Path(self.external_path).stem)
-            or (self.original_filename and Path(self.original_filename).stem)
+        path_name = (
+            (self.store_path and Path(self.store_path).name)
+            or (self.external_path and Path(self.external_path).name)
+            or (self.original_filename and Path(self.original_filename).name)
         )
-        if not self.title and path_stem:
-            return abbrev_str(path_stem, max_len)
 
-        # Otherwise, use the title, description, or body text.
+        # Use the title or the path if possible, falling back to description or even body text.
         title_raw_text = (
             self.title
+            or path_name
             or self.description
             or (not self.is_binary and self.abbrev_body(max_len))
             or UNTITLED
@@ -541,6 +597,13 @@ class Item:
 
         return body_text[:max_len]
 
+    @property
+    def has_body(self) -> bool:
+        """
+        True if the item has a non-empty body.
+        """
+        return bool(self.body and self.body.strip())
+
     def slug_name(self, max_len: int = SLUG_MAX_LEN) -> str:
         """
         Get a readable slugified version of the title or filename or content
@@ -554,7 +617,9 @@ class Item:
         """
         Get or infer description.
         """
-        return abbrev_on_words(html_to_plaintext(self.description or self.body or ""), max_len)
+        return abbrev_on_words(
+            html_to_plaintext(self.description or self.body or ""), max_len
+        )
 
     def read_as_config(self) -> Any:
         """
@@ -585,7 +650,6 @@ class Item:
         """
         Get the full file extension suffix (e.g. "note.md") for this item.
         """
-
         if self.type == ItemType.extension:
             # Python files cannot have more than one . in them.
             return f"{FileExt.py.value}"
@@ -619,7 +683,10 @@ class Item:
         raise ValueError(f"Cannot convert item of type {self.format} to HTML: {self}")
 
     def _copy_and_update(
-        self, other: Item | None = None, update_timestamp: bool = False, **other_updates
+        self,
+        other: Item | None = None,
+        update_timestamp: bool = False,
+        **other_updates: Unpack[ItemUpdateOptions],
     ) -> dict[str, Any]:
         overrides: dict[str, Any] = {"store_path": None, "modified_at": None}
         if update_timestamp:
@@ -637,12 +704,16 @@ class Item:
 
         return fields
 
-    def new_copy_with(self, update_timestamp: bool = True, **other_updates) -> Item:
+    def new_copy_with(
+        self, update_timestamp: bool = True, **other_updates: Unpack[ItemUpdateOptions]
+    ) -> Item:
         """
         Copy item with the given field updates. Resets store_path to None. Updates
         created time if requested.
         """
-        new_fields = self._copy_and_update(update_timestamp=update_timestamp, **other_updates)
+        new_fields = self._copy_and_update(
+            update_timestamp=update_timestamp, **other_updates
+        )
         return Item(**new_fields)
 
     def merged_copy(self, other: Item) -> Item:
@@ -653,7 +724,7 @@ class Item:
         merged_fields = self._copy_and_update(other, update_timestamp=False)
         return Item(**merged_fields)
 
-    def derived_copy(self, type: ItemType, **other_updates) -> Item:
+    def derived_copy(self, **updates: Unpack[ItemUpdateOptions]) -> Item:
         """
         Same as `new_copy_with()`, but also makes any other updates and updates the
         `derived_from` relation. If we also have an action context, then use the
@@ -678,8 +749,12 @@ class Item:
         else:
             derived_from = [StorePath(self.store_path)]
 
-        updates = other_updates.copy()
-        updates["type"] = type
+        updates = updates.copy()
+
+        # If format was specified and user didn't specify file_ext, then infer it.
+        if "file_ext" not in updates and "format" in updates:
+            assert updates["format"] is not None
+            updates["file_ext"] = updates["format"].file_ext
 
         # External resource paths only make sense for resources, so clear them out if new item
         # is not a resource.
@@ -691,15 +766,22 @@ class Item:
         if derived_from:
             new_item.update_relations(derived_from=derived_from)
 
-        # Fall back to action title template if we have it and it wasn't explicitly set.
-        if "title" not in other_updates:
+        # Fall back to action title template if we have it and title wasn't explicitly set.
+        if "title" not in updates:
+            prev_title = self.title or (
+                Path(self.store_path).stem if self.store_path else UNTITLED
+            )
             if self.context:
                 action = self.context.action
                 new_item.title = action.title_template.format(
-                    title=self.title or UNTITLED, action_name=action.name
+                    title=prev_title, action_name=action.name
                 )
             else:
-                log.warning("Deriving an item without action context, will omit title: %s", self)
+                log.warning(
+                    "Deriving an item without action context so keeping previous title: %s",
+                    self,
+                )
+                new_item.title = f"{prev_title} (derived copy)"
 
         return new_item
 
@@ -727,7 +809,8 @@ class Item:
 
     def content_equals(self, other: Item) -> bool:
         """
-        Check if two items have identical content, ignoring timestamps and store path.
+        Check if two items have identical content, ignoring timestamps, store path,
+        and any trailing newlines or whitespace.
         """
         # Check relevant metadata fields.
         self_fields = self.__dict__.copy()

kash/model/params_model.py

@@ -294,6 +294,18 @@ COMMON_ACTION_PARAMS: dict[str, Param] = {
         type=DocSelection,
         default_value=DocSelection.full,
     ),
+    "s3_bucket": Param(
+        "s3_bucket",
+        "The S3 bucket to upload to.",
+        type=str,
+        default_value=None,
+    ),
+    "s3_prefix": Param(
+        "s3_prefix",
+        "The S3 prefix to upload to (with or without a trailing slash).",
+        type=str,
+        default_value=None,
+    ),
 }
 
 # Extra parameters that are available when an action is invoked from the shell.
@@ -305,6 +317,18 @@ RUNTIME_ACTION_PARAMS: dict[str, Param] = {
         "it produces an output item that already exists.",
         type=bool,
     ),
+    "refetch": Param(
+        "refetch",
+        "Forcing re-fetching of any content, not using media or content caches.",
+        type=bool,
+        default_value=False,
+    ),
+    "no_format": Param(
+        "no_format",
+        "Do not auto-format (normalize) Markdown outputs.",
+        type=bool,
+        default_value=False,
+    ),
 }
 
 

kash/shell/completions/completion_scoring.py

@@ -7,6 +7,7 @@ from datetime import UTC, datetime
 from pathlib import Path
 from typing import TypeVar
 
+import rich
 from strif import abbrev_str
 from thefuzz import fuzz
 
@@ -22,7 +23,7 @@ log = get_logger(__name__)
 T = TypeVar("T")
 
 # Scores less than this can be dropped early.
-MIN_CUTOFF = Score(60)
+MIN_CUTOFF = Score(70)
 
 
 def linear_boost(score: Score, min_score: Score) -> Score:
@@ -171,10 +172,29 @@ def score_phrase(prefix: str, text: str) -> Score:
     Could experiment with this more but it's a rough attempt to balance
     full matches and prefix matches.
     """
-    return Score(
-        0.4 * fuzz.token_set_ratio(prefix, text)
-        + 0.4 * fuzz.partial_ratio(prefix, text)
-        + 0.2 * fuzz.token_sort_ratio(prefix, text)
+    if len(prefix) > 5:
+        return Score(
+            0.4 * fuzz.ratio(prefix, text)
+            + 0.3 * fuzz.token_set_ratio(prefix, text)
+            + 0.3 * fuzz.partial_ratio(prefix, text),
+        )
+    else:
+        return Score(0.6 * fuzz.ratio(prefix, text) + 0.4 * fuzz.token_set_ratio(prefix, text))
+
+
+def print_all_scores(prefix: str, text: str):
+    rich.inspect(
+        {
+            "ratio": fuzz.ratio(prefix, text),
+            "partial_ratio": fuzz.partial_ratio(prefix, text),
+            "token_sort_ratio": fuzz.token_sort_ratio(prefix, text),
+            "token_set_ratio": fuzz.token_set_ratio(prefix, text),
+            "partial_token_set_ratio": fuzz.partial_token_set_ratio(prefix, text),
+            "final": score_phrase(prefix, text),
+        },
+        methods=False,
+        docs=False,
+        sort=False,
     )
 
 
@@ -281,3 +301,15 @@ def score_snippet(query: str, snippet: CommentedCommand) -> Score:
     )
     # Bias a little toward command matches.
     return Score(max(command_score, 0.7 * comment_score))
+
+
+## Tests
+
+
+def test_score_phrase():
+    assert score_phrase("hello world", "hello world") == Score(100)
+    assert Score(90) >= score_phrase("hello world", "hello there world") >= Score(80)
+    assert Score(70) >= score_phrase("hello world", "hello there there") >= Score(60)
+    assert Score(85) >= score_phrase("duf", "df") >= Score(75)
+    assert Score(70) >= score_phrase("wik", "awk") >= Score(60)
+    assert Score(60) >= score_phrase("wiki", "awk") >= Score(50)

kash/shell/output/kerm_codes.py

@@ -72,13 +72,12 @@ from html import escape
 from typing import Annotated, Literal, Self, TypeAlias
 from urllib.parse import parse_qs, quote, urlencode, urlparse
 
+from clideps.terminal.osc_utils import OscStr, osc8_link, osc8_link_codes, osc8_link_rich, osc_code
 from prompt_toolkit.formatted_text import FormattedText
 from pydantic import BaseModel, Field, TypeAdapter, model_validator
 from rich.style import Style
 from rich.text import Text
 
-from kash.shell.utils.osc_utils import OscStr, osc8_link, osc8_link_codes, osc8_link_rich, osc_code
-
 KC_VERSION = 0
 """Version of the Kerm codes format. Update when we make breaking changes."""
 

kash/shell/output/shell_formatting.py

@@ -8,10 +8,7 @@ from flowmark import Wrap, fill_text
 from rich.console import Group
 from rich.text import Text
 
-from kash.config.text_styles import (
-    STYLE_HINT,
-    format_success_emoji,
-)
+from kash.config.text_styles import COLOR_FAILURE, COLOR_SUCCESS, STYLE_HINT
 from kash.shell.output.kmarkdown import KMarkdown
 
 
@@ -84,6 +81,19 @@ def format_paragraphs(*paragraphs: str | Text | Group) -> Group:
     return Group(*text)
 
 
+EMOJI_TRUE = "✔︎"
+
+EMOJI_FALSE = "✘"
+
+
+def success_emoji(value: bool, success_only: bool = False) -> str:
+    return EMOJI_TRUE if value else " " if success_only else EMOJI_FALSE
+
+
+def format_success_emoji(value: bool, success_only: bool = False) -> Text:
+    return Text(success_emoji(value, success_only), style=COLOR_SUCCESS if value else COLOR_FAILURE)
+
+
 def format_success(message: str | Text) -> Text:
     return Text.assemble(format_success_emoji(True), message)
 

kash/shell/shell_main.py

@@ -16,12 +16,12 @@ import threading
 import xonsh.main
 from strif import quote_if_needed
 
-from kash.config.setup import setup
+from kash.config.setup import kash_setup
 from kash.shell.utils.argparse_utils import WrappedColorFormatter
 from kash.shell.version import get_full_version_name, get_version
 from kash.xonsh_custom.custom_shell import install_to_xonshrc, start_shell
 
-setup(rich_logging=True)  # Set up logging first.
+kash_setup(rich_logging=True)  # Set up logging first.
 
 
 __version__ = get_version()

kash/shell/utils/exception_printing.py

@@ -46,5 +46,11 @@ def wrap_with_exception_printing(func: Callable[..., R]) -> Callable[[list[str]]
             log.error(f"[{COLOR_ERROR}]Command error:[/{COLOR_ERROR}] %s", summarize_traceback(e))
             log.info("Command error details: %s", e, exc_info=True)
             return None
+        except Exception as e:
+            # Note xonsh can log exceptions but it will be in the xonsh call stack, which is
+            # useless for the user. Better to log all unexpected exception call stack
+            # here to our logs.
+            log.info("Command error details: %s", e, exc_info=True)
+            raise
 
     return command