kash-shell 0.3.17__py3-none-any.whl → 0.3.20__py3-none-any.whl

kash/actions/core/{markdownify.py → markdownify_html.py}

@@ -11,13 +11,10 @@ from kash.web_content.web_extract_readabilipy import extract_text_readabilipy
 log = get_logger(__name__)
 
 
-@kash_action(
-    precondition=is_url_resource | has_html_body,
-    mcp_tool=True,
-)
-def markdownify(item: Item) -> Item:
+@kash_action(precondition=is_url_resource | has_html_body, mcp_tool=True)
+def markdownify_html(item: Item) -> Item:
     """
-    Converts a URL or raw HTML item to Markdown, fetching with the content
+    Converts raw HTML or the URL of an HTML page to Markdown, fetching with the content
     cache if needed. Also uses readability to clean up the HTML.
     """
 

kash/actions/core/minify_html.py

@@ -0,0 +1,41 @@
+from kash.exec import kash_action
+from kash.exec.preconditions import has_fullpage_html_body
+from kash.model import Format, Item, Param
+from kash.utils.errors import InvalidInput
+from kash.workspaces.workspaces import current_ws
+
+
+@kash_action(
+    precondition=has_fullpage_html_body,
+    params=(
+        Param("no_js_min", "Disable JS minification", bool),
+        Param("no_css_min", "Disable CSS minification", bool),
+    ),
+)
+def minify_html(item: Item) -> Item:
+    """
+    Minify an HTML item's content using [html-minifier-terser](https://github.com/terser/html-minifier-terser).
+
+    Also supports Tailwind CSS v4 compilation and inlining, if any Tailwind
+    CSS v4 CDN script tags are found.
+
+    The terser minification seems a bit slower but more robust than
+    [minify-html](https://github.com/wilsonzlin/minify-html).
+    """
+    from minify_tw_html import minify_tw_html
+
+    if not item.store_path:
+        raise InvalidInput(f"Missing store path: {item}")
+
+    ws = current_ws()
+    input_path = ws.base_dir / item.store_path
+
+    output_item = item.derived_copy(format=Format.html, body=None)
+    output_path = ws.target_path_for(output_item)
+
+    minify_tw_html(input_path, output_path)
+
+    output_item.body = output_path.read_text()
+    output_item.external_path = str(output_path)  # Indicate item is already saved.
+
+    return output_item

kash/commands/base/show_command.py

@@ -1,6 +1,7 @@
 from kash.config.logger import get_logger
 from kash.config.text_styles import STYLE_HINT
 from kash.exec import assemble_path_args, kash_command
+from kash.exec_model.shell_model import ShellResult
 from kash.model.paths_model import StorePath
 from kash.shell.output.shell_output import cprint
 from kash.shell.utils.native_utils import ViewMode, terminal_show_image, view_file_native
@@ -19,7 +20,8 @@ def show(
     thumbnail: bool = False,
     browser: bool = False,
     plain: bool = False,
-) -> None:
+    noselect: bool = False,
+) -> ShellResult:
     """
     Show the contents of a file if one is given, or the first file if multiple files
     are selected. Will try to use native apps or web browser to display the file if
@@ -33,6 +35,7 @@ def show(
     :param thumbnail: If there is a thumbnail image, show it too.
     :param browser: Force display with your default web browser.
     :param plain: Use plain view in the console (this is `bat`'s `plain` style).
+    :param noselect: Disable default behavior where `show` also will `select` the file.
     """
     view_mode = (
         ViewMode.console
@@ -63,9 +66,16 @@ def show(
             view_file_native(ws.base_dir / input_path, view_mode=view_mode, plain=plain)
         else:
             view_file_native(input_path, view_mode=view_mode, plain=plain)
+        if not noselect:
+            from kash.commands.workspace.selection_commands import select
+
+            select(input_path)
+            return ShellResult(show_selection=True)
     except (InvalidInput, InvalidState):
         if path:
             # If path is absolute or we couldbn't get a selection, just show the file.
             view_file_native(path, view_mode=view_mode)
         else:
             raise InvalidInput("No selection")
+
+    return ShellResult(show_selection=False)

kash/commands/workspace/workspace_commands.py

@@ -23,14 +23,12 @@ from kash.exec import (
     resolve_locator_arg,
 )
 from kash.exec.action_registry import get_all_actions_defaults
-from kash.exec.fetch_url_metadata import fetch_url_metadata
+from kash.exec.fetch_url_items import fetch_url_item
 from kash.exec.precondition_checks import actions_matching_paths
 from kash.exec.precondition_registry import get_all_preconditions
-from kash.exec.preconditions import is_url_resource
 from kash.exec_model.shell_model import ShellResult
 from kash.local_server.local_url_formatters import local_url_formatter
 from kash.media_base import media_tools
-from kash.media_base.media_services import is_media_url
 from kash.model.items_model import Item, ItemType
 from kash.model.params_model import GLOBAL_PARAMS
 from kash.model.paths_model import StorePath, fmt_store_path
@@ -54,12 +52,11 @@ from kash.utils.common.format_utils import fmt_loc
 from kash.utils.common.obj_replace import remove_values
 from kash.utils.common.parse_key_vals import parse_key_value
 from kash.utils.common.type_utils import not_none
-from kash.utils.common.url import Url, is_url, parse_http_url
+from kash.utils.common.url import Url
 from kash.utils.errors import InvalidInput
 from kash.utils.file_formats.chat_format import tail_chat_history
 from kash.utils.file_utils.dir_info import is_nonempty_dir
 from kash.utils.file_utils.file_formats_model import Format
-from kash.utils.text_handling.doc_normalization import can_normalize
 from kash.web_content.file_cache_utils import cache_file
 from kash.workspaces import (
     current_ws,
@@ -189,85 +186,6 @@ def cache_content(*urls_or_paths: str, refetch: bool = False) -> None:
         PrintHooks.spacer()
 
 
-@kash_command
-def download(*urls_or_paths: str, refetch: bool = False, no_format: bool = False) -> ShellResult:
-    """
-    Download a URL or resource. Uses cached content if available, unless `refetch` is true.
-    Inputs can be URLs or paths to URL resources.
-    Creates both resource and document versions for text content.
-
-    :param no_format: If true, do not also normalize Markdown content.
-    """
-    ws = current_ws()
-    saved_paths = []
-
-    for url_or_path in urls_or_paths:
-        locator = resolve_locator_arg(url_or_path)
-        url: Url | None = None
-
-        # Get the URL from the locator
-        if not isinstance(locator, Path) and is_url(locator):
-            url = Url(locator)
-        elif isinstance(locator, StorePath):
-            url_item = ws.load(locator)
-            if is_url_resource(url_item):
-                url = url_item.url
-
-        if not url:
-            raise InvalidInput(f"Not a URL or URL resource: {fmt_loc(locator)}")
-
-        # Handle media URLs differently
-        if is_media_url(url):
-            log.message(
-                "URL is a media URL, so adding as a resource and will cache media: %s", fmt_loc(url)
-            )
-            store_path = ws.import_item(url, as_type=ItemType.resource, reimport=refetch)
-            saved_paths.append(store_path)
-            media_tools.cache_media(url)
-        else:
-            # Cache the content first
-            expiration_sec = 0 if refetch else None
-            cache_result = cache_file(url, expiration_sec=expiration_sec)
-            original_filename = Path(parse_http_url(url).path).name
-            mime_type = cache_result.content.headers and cache_result.content.headers.mime_type
-
-            # Create a resource item
-            resource_item = Item.from_external_path(
-                cache_result.content.path,
-                ItemType.resource,
-                url=url,
-                mime_type=mime_type,
-                original_filename=original_filename,
-            )
-            # For initial content, do not format or add frontmatter.
-            store_path = ws.save(resource_item, no_frontmatter=True, no_format=True)
-            saved_paths.append(store_path)
-            select(store_path)
-
-            # Also create a doc version for text content if we want to normalize formatting.
-            if resource_item.format and can_normalize(resource_item.format) and not no_format:
-                doc_item = Item.from_external_path(
-                    cache_result.content.path,
-                    ItemType.doc,
-                    url=url,
-                    mime_type=mime_type,
-                    original_filename=original_filename,
-                )
-                # Now use default formatting and frontmatter.
-                doc_store_path = ws.save(doc_item)
-                saved_paths.append(doc_store_path)
-                select(doc_store_path)
-
-    print_status(
-        "Downloaded %s %s:\n%s",
-        len(saved_paths),
-        plural("item", len(saved_paths)),
-        fmt_lines(saved_paths),
-    )
-
-    return ShellResult(show_selection=True)
-
-
 @kash_command
 def history(max: int = 30, raw: bool = False) -> None:
     """
@@ -536,10 +454,14 @@ def save_clipboard(
 
 
 @kash_command
-def fetch_metadata(*files_or_urls: str, refetch: bool = False) -> ShellResult:
+def fetch_url(*files_or_urls: str, refetch: bool = False) -> ShellResult:
     """
-    Fetch metadata for the given URLs or resources. Imports new URLs and saves back
-    the fetched metadata for existing resources.
+    Fetch content and metadata for the given URLs or resources, saving to the
+    current workspace.
+
+    Imports new URLs and saves back the fetched metadata for existing resources.
+    Also saves a resource item with the content of the URL, either HTML, text, or
+    of any other type.
 
     Skips items that already have a title and description, unless `refetch` is true.
     Skips (with a warning) items that are not URL resources.
@@ -552,7 +474,7 @@ def fetch_metadata(*files_or_urls: str, refetch: bool = False) -> ShellResult:
     store_paths = []
     for locator in locators:
         try:
-            fetched_item = fetch_url_metadata(locator, refetch=refetch)
+            fetched_item = fetch_url_item(locator, refetch=refetch)
             store_paths.append(fetched_item.store_path)
         except InvalidInput as e:
             log.warning(

kash/config/colors.py

@@ -139,14 +139,16 @@ web_light_translucent = SimpleNamespace(
     bg_header=hsl_to_hex("hsla(188, 42%, 70%, 0.2)"),
     bg_alt=hsl_to_hex("hsla(39, 24%, 90%, 0.3)"),
     bg_alt_solid=hsl_to_hex("hsla(39, 24%, 97%, 1)"),
+    bg_selected=hsl_to_hex("hsla(188, 44%, 94%, 0.95)"),
     text=hsl_to_hex("hsl(188, 39%, 11%)"),
     code=hsl_to_hex("hsl(44, 38%, 23%)"),
     border=hsl_to_hex("hsl(188, 8%, 50%)"),
     border_hint=hsl_to_hex("hsla(188, 8%, 72%, 0.3)"),
     border_accent=hsl_to_hex("hsla(305, 18%, 65%, 0.85)"),
     hover=hsl_to_hex("hsl(188, 12%, 84%)"),
-    hover_bg=hsl_to_hex("hsla(188, 12%, 94%, 0.8)"),
+    hover_bg=hsl_to_hex("hsla(188, 44%, 94%, 1)"),
     hint=hsl_to_hex("hsl(188, 11%, 65%)"),
+    hint_strong=hsl_to_hex("hsl(188, 11%, 46%)"),
     hint_gentle=hsl_to_hex("hsla(188, 11%, 65%, 0.2)"),
     tooltip_bg=hsl_to_hex("hsla(188, 6%, 37%, 0.7)"),
     popover_bg=hsl_to_hex("hsla(188, 6%, 37%, 0.7)"),
@@ -170,14 +172,16 @@ web_dark_translucent = SimpleNamespace(
     bg_header=hsl_to_hex("hsla(188, 42%, 20%, 0.3)"),
     bg_alt=hsl_to_hex("hsla(220, 14%, 12%, 0.5)"),
     bg_alt_solid=hsl_to_hex("hsl(220, 14%, 12%)"),
+    bg_selected=hsl_to_hex("hsla(188, 12%, 50%, 0.95)"),
     text=hsl_to_hex("hsl(188, 10%, 90%)"),
     code=hsl_to_hex("hsl(44, 38%, 72%)"),
     border=hsl_to_hex("hsl(188, 8%, 25%)"),
     border_hint=hsl_to_hex("hsla(188, 8%, 35%, 0.3)"),
     border_accent=hsl_to_hex("hsla(305, 30%, 55%, 0.85)"),
     hover=hsl_to_hex("hsl(188, 12%, 35%)"),
-    hover_bg=hsl_to_hex("hsla(188, 20%, 25%, 0.4)"),
+    hover_bg=hsl_to_hex("hsla(188, 12%, 40%, 0.95)"),
     hint=hsl_to_hex("hsl(188, 11%, 55%)"),
+    hint_strong=hsl_to_hex("hsl(188, 11%, 72%)"),
     hint_gentle=hsl_to_hex("hsla(188, 11%, 55%, 0.2)"),
     tooltip_bg=hsl_to_hex("hsla(188, 6%, 20%, 0.9)"),
     popover_bg=hsl_to_hex("hsla(188, 6%, 20%, 0.9)"),

kash/docs/markdown/topics/a1_what_is_kash.md

@@ -3,35 +3,64 @@
 > “*Simple should be simple.
 > Complex should be possible.*” —Alan Kay
 
-Kash (“Knowledge Agent SHell”) is an **interactive, AI-native command-line** shell for
-practical knowledge tasks.
+Kash (“Knowledge Agent SHell”) is an experiment in making software tasks more modular,
+exploratory, and flexible using Python and current AI tools.
+
+The philosophy behind kash is similar to Unix shell tools: simple commands that can be
+combined in flexible and powerful ways.
+It operates on "items" such as URLs, files, or Markdown notes within a workspace
+directory.
+
+You can use Kash as an **interactive, AI-native command-line** shell for practical
+knowledge tasks. It's also **a Python library** that lets you convert a simple Python
+function into a command and an MCP tool, so it integrates with other tools like
+Anthropic Desktop or Cursor.
+
+It's new and still has some rough edges, but it's now working well enough it is feeling
+quite powerful. It now serves as a replacement for my usual shell (previously bash or
+zsh). I use it routinely to remix, combine, and interactively explore and then gradually
+automate complex tasks by composing AI tools, APIs, and libraries.
+And last but not least, the same framework lets me build other tools (like
+[textpress](https://github.com/jlevy/textpress)).
 
-It's also **a Python library** that lets you convert a simple Python function into a
-command and an MCP tool, so it integrates with other tools like Anthropic Desktop or
-Cursor.
+And of course, kash can read its own functionality and enhance itself by writing new
+actions.
 
-You can think of it a kind of power-tool for technical users who want to use Python and
-APIs, a kind of hybrid between an AI assistant, a shell, and a developer tool like
-Cursor or Claude Code.
+### Kash Packages
 
-It's my attempt at finding a way to remix, combine, and interactively explore and then
-gradually automate complex tasks by composing AI tools, APIs, and libraries.
+The [kash-shell](https://github.com/jlevy/kash) package is the base package and includes
+the Python framework, a few core utilities, and the Kash command-line shell.
 
-And of course, kash can read its own functionality and enhance itself by writing new
-actions.
+Additional actions for handling more complex tasks like converting documents and
+transcribing, researching, or annotating videos, are in the
+[kash-docs](https://github.com/jlevy/kash-docs) and
+[kash-media](https://github.com/jlevy/kash-media) packages, all available on PyPI and
+quick to install via uv.
 
 ### Key Concepts
 
-- **Actions:** The core of Kash are **Kash actions**. By decorating a Python function,
-  you can turn it into an action, which makes it more flexible and powerful, able to
-  work with file inputs stored and outputs in a given directory, also called a
-  **workspace**.
+- **Actions:** The core of Kash are **actions**. By decorating a Python function with
+  `@kash_action`, you can turn it into an action, which makes it more flexible and
+  powerful. It can then be used like a command line command as well as a Python function
+  or an MCP tool.
+
+- **Workspaces:** A key element of Kash is that it does most nontrivial work in the
+  context of a **workspace**. A workspace is just a directory of files that have a few
+  conventions to make it easier to maintain context and perform actions.
+  A bit like how Git repos work, it has a `.kash/` directory that holds metadata and
+  cached content. The rest can be anything, but is typically directories of resources
+  (like .docx or .pdf or links to web pages) or content, typically Markdown files with
+  YAML frontmatter. All text files use
+  [frontmatter-format](https://github.com/jlevy/frontmatter-format) so have easy-to-read
+  YAML metadata that includes not just title or description, but also the names of the
+  actions that created it.
 
 - **Compositionality:** An action is composable with other actions simply as a Python
-  function, so complex (like transcribing and annotating a video) actions can be built
-  from simpler actions (like downloading and caching a YouTube video, identifying the
-  speakers in a transcript, etc.). The goal is to reduce the "interstitial complexity"
-  of combining tools, so it's easy for you (or an LLM!) to combine tools in flexible and
+  function, so complex operations (for example, transcribing and annotating a video and
+  publishing it on a website) actions can be built from simpler actions (say downloading
+  and caching a YouTube video, identifying the speakers in a transcript, formatting it
+  as pretty HTML, etc.). The goal is to reduce the "interstitial complexity" of
+  combining tools, so it's easy for you (or an LLM!) to combine tools in flexible and
   powerful ways.
 
 - **Command-line usage:** In addition to using the function in other libraries and
@@ -40,9 +69,6 @@ actions.
   video. In kash you have **smart tab completions**, **Python expressions**, and an **LLM
   assistant** built into the shell.
 
-- **MCP support:** Finally, an action is also an **MCP tool server** so you can use it
-  in any MCP client, like Anthropic Desktop or Cursor.
-
 - **Support for any API:** Kash is tool agnostic and runs locally, on file inputs in
   simple formats, so you own and manage your data and workspaces however you like.
   You can use it with any models or APIs you like, and is already set up to use the APIs
@@ -51,6 +77,9 @@ actions.
   **Perplexity**, **Firecrawl**, **Exa**, and any Python libraries.
   There is also some experimental support for **LlamaIndex** and **ChromaDB**.
 
+- **MCP support:** Finally, an action is also an **MCP tool server** so you can use it
+  in any MCP client, like Anthropic Desktop or Cursor.
+
 ### What Can Kash Do?
 
 You can use kash actions to do deep research, transcribe videos, summarize and organize

kash/docs/markdown/topics/a2_installation.md

@@ -2,9 +2,9 @@
 
 ### Running the Kash Shell
 
-Kash offers a shell environment based on [xonsh](https://xon.sh/) augmented with an LLM
-assistant and a variety of other enhanced commands and customizations.
-If you've used a bash or Python shell before, xonsh is very intuitive.
+Kash offers a shell environment based on [xonsh](https://xon.sh/) augmented with a bunch
+of enhanced commands and customizations.
+If you've used a bash or Python shell before, it should be very intuitive.
 
 Within the kash shell, you get a full environment with all actions and commands.
 You also get intelligent auto-complete, a built-in assistant to help you perform tasks,
@@ -38,30 +38,17 @@ These are for `kash-media` but you can use a `kash-shell` for a more basic setup
    Kash is easiest to use via [**uv**](https://docs.astral.sh/uv/), the new package
    manager for Python. `uv` replaces traditional use of `pyenv`, `pipx`, `poetry`, `pip`,
    etc. Installing `uv` also ensures you get a compatible version of Python.
-
-   If you don't have `uv` installed, a quick way to install it is:
-
+   See [uv's docs](https://docs.astral.sh/uv/getting-started/installation/) for other
+   installation methods and platforms.
+   Usually you just want to run:
    ```shell
    curl -LsSf https://astral.sh/uv/install.sh | sh
    ```
 
-   For macOS, you prefer [brew](https://brew.sh/) you can install or upgrade uv with:
-
-   ```shell
-   brew update
-   brew install uv
-   ```
-   See [uv's docs](https://docs.astral.sh/uv/getting-started/installation/) for other
-   installation methods and platforms.
-
 2. **Install additional command-line tools:**
 
    In addition to Python, it's highly recommended to install a few other dependencies to
-   make more tools and commands work: `ripgrep` (for search), `bat` (for prettier file
-   display), `eza` (a much improved version of `ls`), `hexyl` (a much improved hex
-   viewer), `imagemagick` (for image display in modern terminals), `libmagic` (for file
-   type detection), `ffmpeg` (for audio and video conversions)
-
+   make more tools and commands work.
    For macOS, you can again use brew:
 
    ```shell
@@ -82,22 +69,22 @@ These are for `kash-media` but you can use a `kash-shell` for a more basic setup
 
    For Windows or other platforms, see the uv instructions.
 
+   Kash auto-detects and uses `ripgrep` (for search), `bat` (for prettier file display),
+   `eza` (a much improved version of `ls`), `hexyl` (a much improved hex viewer),
+   `imagemagick` (for image display in modern terminals), `libmagic` (for file type
+   detection), `ffmpeg` (for audio and video conversions)
+
 3. **Install kash or a kash kit:**
 
+   For a more meaningful demo, use an enhanced version of kash that also has various
+   media tools (like yt-dlp and Deepgram support):
+
    ```shell
-   uv tool install kash-media --python=3.13
+   uv tool install kash-media --upgrade --python=3.13
    ```
 
    Other versions of Python should work but 3.13 is recommended.
-   For a setup without the media tools, use `kash-shell` instead.
-
-   If you've installed an older version and want to be sure you have the latest shell,
-   you may want to add `--upgrade --force` to be sure you get the latest version of the
-   kit.
-
-   ```shell
-   uv tool install kash-media --python=3.13 --upgrade --force 
-   ```
+   For a setup without the media tools, just install `kash-shell` instead.
 
 4. **Set up API keys:**
 

kash/docs/markdown/topics/a3_getting_started.md

@@ -15,25 +15,11 @@ Type `help` for the full documentation.
 The simplest way to illustrate how to use kash is by example.
 You can go through the commands below a few at a time, trying each one.
 
-This is a "real" example that uses a bunch of libraries.
+This is a "real" example that uses ffmpeg and a few other libraries.
 So to get it to work you must install not just the main shell but the kash "media kit"
 with extra dependencies.
-
-You need the following tools:
-
-```shell
-# On MacOS:
-brew install ripgrep bat eza hexyl imagemagick libmagic ffmpeg 
-# On Linux:
-apt install ripgrep bat eza hexyl imagemagick libmagic ffmpeg
-```
-
-Then install the `kash-media`, which includes kash-shell and many other libs like yt-dlp
-for YouTube handling:
-
-```shell
-uv tool install kash-media
-```
+This is discussed in [the installation instructions](#installation-steps).
+If you don't have these already installed, you can add these tools:
 
 Then run `kash` to start.
 
@@ -152,7 +138,7 @@ show_webpage
 show_webpage --help
 
 # And you can actually how this works by looking at its source:
-source_code show_webpage
+show_webpage --show_source
 
 # What if something isn't working right?
 # Sometimes we may want to browse more detailed system logs:
@@ -171,7 +157,7 @@ transcribe_format https://www.youtube.com/watch?v=_8djNYprRDI
 
 # Getting a little fancier, this one adds little paragraph annotations and
 # a nicer summary at the top:
-transcribe_annotate_summarize https://www.youtube.com/watch?v=_8djNYprRDI
+transcribe_annotate https://www.youtube.com/watch?v=_8djNYprRDI
 
 show_webpage
 ```

kash/exec/__init__.py

@@ -1,6 +1,7 @@
 from kash.exec.action_decorators import kash_action, kash_action_class
 from kash.exec.action_exec import SkipItem, prepare_action_input, run_action_with_shell_context
 from kash.exec.command_registry import kash_command
+from kash.exec.fetch_url_items import fetch_url_item, fetch_url_item_content
 from kash.exec.importing import import_and_register
 from kash.exec.llm_transforms import llm_transform_item, llm_transform_str
 from kash.exec.precondition_registry import kash_precondition
@@ -21,6 +22,8 @@ __all__ = [
     "prepare_action_input",
     "run_action_with_shell_context",
     "kash_command",
+    "fetch_url_item",
+    "fetch_url_item_content",
     "kash_runtime",
     "current_runtime_settings",
     "import_and_register",

kash/exec/action_exec.py

@@ -43,7 +43,7 @@ def prepare_action_input(*input_args: CommandArg, refetch: bool = False) -> Acti
     URL or file resources, either finding them in the workspace or importing them.
     Also fetches metadata for URLs if they don't already have title and description.
     """
-    from kash.exec.fetch_url_metadata import fetch_url_item_metadata
+    from kash.exec.fetch_url_items import fetch_url_item_content
 
     ws = current_ws()
 
@@ -55,7 +55,7 @@ def prepare_action_input(*input_args: CommandArg, refetch: bool = False) -> Acti
     if input_items:
         log.message("Assembling metadata for input items:\n%s", fmt_lines(input_items))
         input_items = [
-            fetch_url_item_metadata(item, refetch=refetch) if is_url_resource(item) else item
+            fetch_url_item_content(item, refetch=refetch) if is_url_resource(item) else item
             for item in input_items
         ]
 
@@ -102,7 +102,7 @@ def log_action(action: Action, action_input: ActionInput, operation: Operation):
     """
     PrintHooks.before_log_action_run()
     log.message("%s Action: `%s`", EMOJI_START, action.name)
-    log.message("Running: `%s`", operation.command_line(with_options=True))
+    log.info("Running: `%s`", operation.command_line(with_options=True))
     if len(action.param_value_summary()) > 0:
         log.message("Parameters:\n%s", action.param_value_summary_str())
     log.info("Operation is: %s", operation)

kash/exec/fetch_url_items.py

@@ -0,0 +1,109 @@
+from kash.config.logger import get_logger
+from kash.exec.preconditions import is_url_resource
+from kash.media_base.media_services import get_media_metadata
+from kash.model.items_model import Item, ItemType
+from kash.model.paths_model import StorePath
+from kash.utils.common.format_utils import fmt_loc
+from kash.utils.common.url import Url, is_url
+from kash.utils.common.url_slice import add_slice_to_url, parse_url_slice
+from kash.utils.errors import InvalidInput
+
+log = get_logger(__name__)
+
+
+def fetch_url_item(
+    locator: Url | StorePath, *, save_content: bool = True, refetch: bool = False
+) -> Item:
+    from kash.workspaces import current_ws
+
+    ws = current_ws()
+    if is_url(locator):
+        # Import or find URL as a resource in the current workspace.
+        store_path = ws.import_item(locator, as_type=ItemType.resource)
+        item = ws.load(store_path)
+    elif isinstance(locator, StorePath):
+        item = ws.load(locator)
+        if not is_url_resource(item):
+            raise InvalidInput(f"Not a URL resource: {fmt_loc(locator)}")
+    else:
+        raise InvalidInput(f"Not a URL or URL resource: {fmt_loc(locator)}")
+
+    return fetch_url_item_content(item, save_content=save_content, refetch=refetch)
+
+
+def fetch_url_item_content(item: Item, *, save_content: bool = True, refetch: bool = False) -> Item:
+    """
+    Fetch content and metadata for a URL using a media service if we
+    recognize the URL as a known media service. Otherwise, fetch and extract the
+    metadata and content from the web page and save it to the URL item.
+
+    If `save_content` is true, a copy of the content is also saved as
+    a resource item.
+
+    The content item is returned if content was saved. Otherwise, the updated
+    URL item is returned.
+    """
+    from kash.web_content.canon_url import canonicalize_url
+    from kash.web_content.web_extract import fetch_page_content
+    from kash.workspaces import current_ws
+
+    ws = current_ws()
+    if not refetch and item.title and item.description:
+        log.message(
+            "Already have title and description, will not fetch metadata: %s", item.fmt_loc()
+        )
+        return item
+
+    if not item.url:
+        raise InvalidInput(f"No URL for item: {item.fmt_loc()}")
+
+    url = canonicalize_url(item.url)
+    log.message("No metadata for URL, will fetch: %s", url)
+
+    # Prefer fetching metadata from media using the media service if possible.
+    # Data is cleaner and YouTube for example often blocks regular scraping.
+    media_metadata = get_media_metadata(url)
+    url_item: Item | None = None
+    content_item: Item | None = None
+    if media_metadata:
+        url_item = Item.from_media_metadata(media_metadata)
+        # Preserve and canonicalize any slice suffix on the URL.
+        _base_url, slice = parse_url_slice(item.url)
+        if slice:
+            new_url = add_slice_to_url(media_metadata.url, slice)
+            if new_url != item.url:
+                log.message("Updated URL from metadata and added slice: %s", new_url)
+            url_item.url = new_url
+
+        url_item = item.merged_copy(url_item)
+    else:
+        page_data = fetch_page_content(url, refetch=refetch, cache=save_content)
+        url_item = item.new_copy_with(
+            title=page_data.title or item.title,
+            description=page_data.description or item.description,
+            thumbnail_url=page_data.thumbnail_url or item.thumbnail_url,
+        )
+        if save_content:
+            assert page_data.saved_content
+            assert page_data.format_info
+            content_item = url_item.new_copy_with(
+                external_path=str(page_data.saved_content),
+                # Use the original filename, not the local cache filename (which has a hash suffix).
+                original_filename=item.get_filename(),
+                format=page_data.format_info.format,
+            )
+            ws.save(content_item)
+
+    if not url_item.title:
+        log.warning("Failed to fetch page data: title is missing: %s", item.url)
+
+    # Now save the updated URL item and also the content item if we have one.
+    ws.save(url_item)
+    assert url_item.store_path
+    log.debug("Saved URL item: %s", url_item.fmt_loc())
+    if content_item:
+        ws.save(content_item)
+        assert content_item.store_path
+        log.debug("Saved content item: %s", content_item.fmt_loc())
+
+    return content_item or url_item