kash-shell 0.3.18__py3-none-any.whl → 0.3.20__py3-none-any.whl

kash/actions/core/{markdownify.py → markdownify_html.py}

@@ -11,13 +11,10 @@ from kash.web_content.web_extract_readabilipy import extract_text_readabilipy
 log = get_logger(__name__)
 
 
-@kash_action(
-    precondition=is_url_resource | has_html_body,
-    mcp_tool=True,
-)
-def markdownify(item: Item) -> Item:
+@kash_action(precondition=is_url_resource | has_html_body, mcp_tool=True)
+def markdownify_html(item: Item) -> Item:
     """
-    Converts a URL or raw HTML item to Markdown, fetching with the content
+    Converts raw HTML or the URL of an HTML page to Markdown, fetching with the content
     cache if needed. Also uses readability to clean up the HTML.
     """
 

kash/commands/workspace/workspace_commands.py

@@ -23,14 +23,12 @@ from kash.exec import (
     resolve_locator_arg,
 )
 from kash.exec.action_registry import get_all_actions_defaults
-from kash.exec.fetch_url_metadata import fetch_url_metadata
+from kash.exec.fetch_url_items import fetch_url_item
 from kash.exec.precondition_checks import actions_matching_paths
 from kash.exec.precondition_registry import get_all_preconditions
-from kash.exec.preconditions import is_url_resource
 from kash.exec_model.shell_model import ShellResult
 from kash.local_server.local_url_formatters import local_url_formatter
 from kash.media_base import media_tools
-from kash.media_base.media_services import is_media_url
 from kash.model.items_model import Item, ItemType
 from kash.model.params_model import GLOBAL_PARAMS
 from kash.model.paths_model import StorePath, fmt_store_path
@@ -54,12 +52,11 @@ from kash.utils.common.format_utils import fmt_loc
 from kash.utils.common.obj_replace import remove_values
 from kash.utils.common.parse_key_vals import parse_key_value
 from kash.utils.common.type_utils import not_none
-from kash.utils.common.url import Url, is_url, parse_http_url
+from kash.utils.common.url import Url
 from kash.utils.errors import InvalidInput
 from kash.utils.file_formats.chat_format import tail_chat_history
 from kash.utils.file_utils.dir_info import is_nonempty_dir
 from kash.utils.file_utils.file_formats_model import Format
-from kash.utils.text_handling.doc_normalization import can_normalize
 from kash.web_content.file_cache_utils import cache_file
 from kash.workspaces import (
     current_ws,
@@ -189,85 +186,6 @@ def cache_content(*urls_or_paths: str, refetch: bool = False) -> None:
         PrintHooks.spacer()
 
 
-@kash_command
-def download(*urls_or_paths: str, refetch: bool = False, no_format: bool = False) -> ShellResult:
-    """
-    Download a URL or resource. Uses cached content if available, unless `refetch` is true.
-    Inputs can be URLs or paths to URL resources.
-    Creates both resource and document versions for text content.
-
-    :param no_format: If true, do not also normalize Markdown content.
-    """
-    ws = current_ws()
-    saved_paths = []
-
-    for url_or_path in urls_or_paths:
-        locator = resolve_locator_arg(url_or_path)
-        url: Url | None = None
-
-        # Get the URL from the locator
-        if not isinstance(locator, Path) and is_url(locator):
-            url = Url(locator)
-        elif isinstance(locator, StorePath):
-            url_item = ws.load(locator)
-            if is_url_resource(url_item):
-                url = url_item.url
-
-        if not url:
-            raise InvalidInput(f"Not a URL or URL resource: {fmt_loc(locator)}")
-
-        # Handle media URLs differently
-        if is_media_url(url):
-            log.message(
-                "URL is a media URL, so adding as a resource and will cache media: %s", fmt_loc(url)
-            )
-            store_path = ws.import_item(url, as_type=ItemType.resource, reimport=refetch)
-            saved_paths.append(store_path)
-            media_tools.cache_media(url)
-        else:
-            # Cache the content first
-            expiration_sec = 0 if refetch else None
-            cache_result = cache_file(url, expiration_sec=expiration_sec)
-            original_filename = Path(parse_http_url(url).path).name
-            mime_type = cache_result.content.headers and cache_result.content.headers.mime_type
-
-            # Create a resource item
-            resource_item = Item.from_external_path(
-                cache_result.content.path,
-                ItemType.resource,
-                url=url,
-                mime_type=mime_type,
-                original_filename=original_filename,
-            )
-            # For initial content, do not format or add frontmatter.
-            store_path = ws.save(resource_item, no_frontmatter=True, no_format=True)
-            saved_paths.append(store_path)
-            select(store_path)
-
-            # Also create a doc version for text content if we want to normalize formatting.
-            if resource_item.format and can_normalize(resource_item.format) and not no_format:
-                doc_item = Item.from_external_path(
-                    cache_result.content.path,
-                    ItemType.doc,
-                    url=url,
-                    mime_type=mime_type,
-                    original_filename=original_filename,
-                )
-                # Now use default formatting and frontmatter.
-                doc_store_path = ws.save(doc_item)
-                saved_paths.append(doc_store_path)
-                select(doc_store_path)
-
-    print_status(
-        "Downloaded %s %s:\n%s",
-        len(saved_paths),
-        plural("item", len(saved_paths)),
-        fmt_lines(saved_paths),
-    )
-
-    return ShellResult(show_selection=True)
-
-
 @kash_command
 def history(max: int = 30, raw: bool = False) -> None:
     """
@@ -536,10 +454,14 @@ def save_clipboard(
 
 
 @kash_command
-def fetch_metadata(*files_or_urls: str, refetch: bool = False) -> ShellResult:
+def fetch_url(*files_or_urls: str, refetch: bool = False) -> ShellResult:
     """
-    Fetch metadata for the given URLs or resources. Imports new URLs and saves back
-    the fetched metadata for existing resources.
+    Fetch content and metadata for the given URLs or resources, saving to the
+    current workspace.
+
+    Imports new URLs and saves back the fetched metadata for existing resources.
+    Also saves a resource item with the content of the URL, either HTML, text, or
+    of any other type.
 
     Skips items that already have a title and description, unless `refetch` is true.
     Skips (with a warning) items that are not URL resources.
@@ -552,7 +474,7 @@ def fetch_metadata(*files_or_urls: str, refetch: bool = False) -> ShellResult:
     store_paths = []
     for locator in locators:
         try:
-            fetched_item = fetch_url_metadata(locator, refetch=refetch)
+            fetched_item = fetch_url_item(locator, refetch=refetch)
             store_paths.append(fetched_item.store_path)
         except InvalidInput as e:
             log.warning(

kash/docs/markdown/topics/a1_what_is_kash.md

@@ -34,7 +34,7 @@ the Python framework, a few core utilities, and the Kash command-line shell.
 Additional actions for handling more complex tasks like converting documents and
 transcribing, researching, or annotating videos, are in the
 [kash-docs](https://github.com/jlevy/kash-docs) and
-[kash-media](https://github.com/jlevy/kash-docs) packages, all available on PyPI and
+[kash-media](https://github.com/jlevy/kash-media) packages, all available on PyPI and
 quick to install via uv.
 
 ### Key Concepts

kash/exec/__init__.py

@@ -1,6 +1,7 @@
 from kash.exec.action_decorators import kash_action, kash_action_class
 from kash.exec.action_exec import SkipItem, prepare_action_input, run_action_with_shell_context
 from kash.exec.command_registry import kash_command
+from kash.exec.fetch_url_items import fetch_url_item, fetch_url_item_content
 from kash.exec.importing import import_and_register
 from kash.exec.llm_transforms import llm_transform_item, llm_transform_str
 from kash.exec.precondition_registry import kash_precondition
@@ -21,6 +22,8 @@ __all__ = [
     "prepare_action_input",
     "run_action_with_shell_context",
     "kash_command",
+    "fetch_url_item",
+    "fetch_url_item_content",
     "kash_runtime",
     "current_runtime_settings",
     "import_and_register",

kash/exec/action_exec.py

@@ -43,7 +43,7 @@ def prepare_action_input(*input_args: CommandArg, refetch: bool = False) -> Acti
     URL or file resources, either finding them in the workspace or importing them.
     Also fetches metadata for URLs if they don't already have title and description.
     """
-    from kash.exec.fetch_url_metadata import fetch_url_item_metadata
+    from kash.exec.fetch_url_items import fetch_url_item_content
 
     ws = current_ws()
 
@@ -55,7 +55,7 @@ def prepare_action_input(*input_args: CommandArg, refetch: bool = False) -> Acti
     if input_items:
         log.message("Assembling metadata for input items:\n%s", fmt_lines(input_items))
         input_items = [
-            fetch_url_item_metadata(item, refetch=refetch) if is_url_resource(item) else item
+            fetch_url_item_content(item, refetch=refetch) if is_url_resource(item) else item
             for item in input_items
         ]
 

kash/exec/{fetch_url_metadata.py → fetch_url_items.py}

@@ -11,7 +11,9 @@ from kash.utils.errors import InvalidInput
 log = get_logger(__name__)
 
 
-def fetch_url_metadata(locator: Url | StorePath, refetch: bool = False) -> Item:
+def fetch_url_item(
+    locator: Url | StorePath, *, save_content: bool = True, refetch: bool = False
+) -> Item:
     from kash.workspaces import current_ws
 
     ws = current_ws()
@@ -26,16 +28,23 @@ def fetch_url_metadata(locator: Url | StorePath, refetch: bool = False) -> Item:
     else:
         raise InvalidInput(f"Not a URL or URL resource: {fmt_loc(locator)}")
 
-    return fetch_url_item_metadata(item, refetch=refetch)
+    return fetch_url_item_content(item, save_content=save_content, refetch=refetch)
 
 
-def fetch_url_item_metadata(item: Item, refetch: bool = False) -> Item:
+def fetch_url_item_content(item: Item, *, save_content: bool = True, refetch: bool = False) -> Item:
     """
-    Fetch metadata for a URL using a media service if we recognize the URL,
-    and otherwise fetching and extracting it from the web page HTML.
+    Fetch content and metadata for a URL using a media service if we
+    recognize the URL as a known media service. Otherwise, fetch and extract the
+    metadata and content from the web page and save it to the URL item.
+
+    If `save_content` is true, a copy of the content is also saved as
+    a resource item.
+
+    The content item is returned if content was saved. Otherwise, the updated
+    URL item is returned.
     """
     from kash.web_content.canon_url import canonicalize_url
-    from kash.web_content.web_extract import fetch_extract
+    from kash.web_content.web_extract import fetch_page_content
     from kash.workspaces import current_ws
 
     ws = current_ws()
@@ -54,28 +63,47 @@ def fetch_url_item_metadata(item: Item, refetch: bool = False) -> Item:
     # Prefer fetching metadata from media using the media service if possible.
     # Data is cleaner and YouTube for example often blocks regular scraping.
     media_metadata = get_media_metadata(url)
+    url_item: Item | None = None
+    content_item: Item | None = None
     if media_metadata:
-        fetched_item = Item.from_media_metadata(media_metadata)
+        url_item = Item.from_media_metadata(media_metadata)
         # Preserve and canonicalize any slice suffix on the URL.
         _base_url, slice = parse_url_slice(item.url)
         if slice:
             new_url = add_slice_to_url(media_metadata.url, slice)
             if new_url != item.url:
                 log.message("Updated URL from metadata and added slice: %s", new_url)
-            fetched_item.url = new_url
+            url_item.url = new_url
 
-        fetched_item = item.merged_copy(fetched_item)
+        url_item = item.merged_copy(url_item)
     else:
-        page_data = fetch_extract(url, refetch=refetch)
-        fetched_item = item.new_copy_with(
+        page_data = fetch_page_content(url, refetch=refetch, cache=save_content)
+        url_item = item.new_copy_with(
             title=page_data.title or item.title,
             description=page_data.description or item.description,
             thumbnail_url=page_data.thumbnail_url or item.thumbnail_url,
         )
+        if save_content:
+            assert page_data.saved_content
+            assert page_data.format_info
+            content_item = url_item.new_copy_with(
+                external_path=str(page_data.saved_content),
+                # Use the original filename, not the local cache filename (which has a hash suffix).
+                original_filename=item.get_filename(),
+                format=page_data.format_info.format,
+            )
+            ws.save(content_item)
 
-    if not fetched_item.title:
+    if not url_item.title:
         log.warning("Failed to fetch page data: title is missing: %s", item.url)
 
-    ws.save(fetched_item)
+    # Now save the updated URL item and also the content item if we have one.
+    ws.save(url_item)
+    assert url_item.store_path
+    log.debug("Saved URL item: %s", url_item.fmt_loc())
+    if content_item:
+        ws.save(content_item)
+        assert content_item.store_path
+        log.debug("Saved content item: %s", content_item.fmt_loc())
 
-    return fetched_item
+    return content_item or url_item

kash/file_storage/file_store.py

@@ -405,6 +405,7 @@ class FileStore(Workspace):
         # If external path already exists and is within the workspace, the file was
         # already saved (e.g. by an action that wrote the item directly to the store).
         external_path = item.external_path and Path(item.external_path).resolve()
+        skipped_save = False
         if external_path and self._is_in_store(external_path):
             log.info("Item with external_path already saved: %s", fmt_loc(external_path))
             rel_path = external_path.relative_to(self.base_dir)
@@ -480,12 +481,17 @@ class FileStore(Workspace):
                     )
                     os.unlink(full_path)
                     store_path = old_store_path
+                    skipped_save = True
 
         # Update in-memory store_path only after successful save.
         item.store_path = str(store_path)
         self._id_index_item(store_path)
 
-        log.message("%s Saved item: %s", EMOJI_SAVED, fmt_loc(store_path))
+        if not skipped_save:
+            log.message("%s Saved item: %s", EMOJI_SAVED, fmt_loc(store_path))
+        else:
+            log.info("%s Already saved: %s", EMOJI_SAVED, fmt_loc(store_path))
+
         return store_path
 
     @log_calls(level="debug")

kash/file_storage/store_filenames.py

@@ -30,6 +30,10 @@ def folder_for_type(item_type: ItemType) -> Path:
 
 
 def join_suffix(base_slug: str, full_suffix: str) -> str:
+    """
+    Create a store filename by joining a base slug and a full suffix, i.e. a filename
+    extension with or without an item type (`.html` or `.resource.html`, for example).
+    """
     return f"{base_slug}.{full_suffix.lstrip('.')}"
 
 

kash/help/function_param_info.py

@@ -2,9 +2,9 @@ from collections.abc import Callable
 from dataclasses import replace
 from typing import Any
 
-from kash.help.docstring_utils import parse_docstring
 from kash.model.params_model import ALL_COMMON_PARAMS, Param
 from kash.utils.common.function_inspect import FuncParam, inspect_function_params
+from kash.utils.common.parse_docstring import parse_docstring
 
 
 def _look_up_param_docs(func: Callable[..., Any], kw_params: list[FuncParam]) -> list[Param]:

kash/help/help_pages.py

@@ -3,7 +3,6 @@ from rich.text import Text
 from kash.config.logger import get_logger
 from kash.config.text_styles import STYLE_HINT
 from kash.docs.all_docs import DocSelection, all_docs
-from kash.help.docstring_utils import parse_docstring
 from kash.shell.output.shell_formatting import format_name_and_value
 from kash.shell.output.shell_output import (
     PrintHooks,
@@ -12,6 +11,7 @@ from kash.shell.output.shell_output import (
     print_hrule,
     print_markdown,
 )
+from kash.utils.common.parse_docstring import parse_docstring
 
 log = get_logger(__name__)
 

kash/help/help_printing.py

@@ -6,7 +6,6 @@ from kash.docs.all_docs import DocSelection
 from kash.exec.action_registry import look_up_action_class
 from kash.exec.command_registry import CommandFunction, look_up_command
 from kash.help.assistant import assist_preamble, assistance_unstructured
-from kash.help.docstring_utils import parse_docstring
 from kash.help.function_param_info import annotate_param_info
 from kash.help.help_lookups import look_up_faq
 from kash.help.tldr_help import tldr_help
@@ -22,6 +21,7 @@ from kash.shell.output.shell_output import (
     print_help,
     print_markdown,
 )
+from kash.utils.common.parse_docstring import parse_docstring
 from kash.utils.errors import InvalidInput, NoMatch
 from kash.utils.file_formats.chat_format import ChatHistory, ChatMessage, ChatRole
 

kash/model/items_model.py

@@ -675,9 +675,21 @@ class Item:
             raise FileFormatError(f"Config item is not YAML: {self.format}: {self}")
         return from_yaml_string(self.body)
 
+    def get_filename(self) -> str | None:
+        """
+        Get the store or external path filename of the item, including the
+        file extension.
+        """
+        if self.store_path:
+            return Path(self.store_path).name
+        elif self.external_path:
+            return Path(self.external_path).name
+        else:
+            return None
+
     def get_file_ext(self) -> FileExt:
         """
-        Get or infer file extension.
+        Get or infer the base file extension for the item.
         """
         if self.file_ext:
             return self.file_ext
@@ -688,7 +700,8 @@ class Item:
 
     def get_full_suffix(self) -> str:
         """
-        Get the full file extension suffix (e.g. "note.md") for this item.
+        Assemble the full file extension suffix (e.g. "resource.yml") for this item.
+        Without a leading dot.
         """
         if self.type == ItemType.extension:
             # Python files cannot have more than one . in them.
@@ -892,12 +905,14 @@ class Item:
 
     def fmt_loc(self) -> str:
         """
-        Formatted store path, external path, or title. For error messages etc.
+        Formatted store path, external path, URL, or title. Use for logging etc.
         """
         if self.store_path:
             return fmt_store_path(self.store_path)
         elif self.external_path:
             return fmt_loc(self.external_path)
+        elif self.url:
+            return fmt_loc(self.url)
         else:
             return repr(self.pick_title())
 

kash/utils/common/parse_docstring.py

@@ -0,0 +1,347 @@
+import re
+from dataclasses import dataclass, field
+from textwrap import dedent
+
+
+@dataclass
+class Docstring:
+    """
+    A parsed docstring.
+    """
+
+    body: str = ""
+    param: dict[str, str] = field(default_factory=dict)
+    type: dict[str, str] = field(default_factory=dict)
+    returns: str = ""
+    rtype: str = ""
+
+
+def parse_docstring(docstring: str) -> Docstring:
+    """
+    Parse a docstring in either reStructuredText or Google style format.
+
+    Supports two formats:
+    - reStructuredText style: `:param name: description`, `:type name: type`, etc.
+    - Google style: `Args:` section with `name (type): description` format
+
+    The parser automatically detects which format is used based on the presence
+    of `:param` directives or `Args:` sections.
+    """
+    docstring = dedent(docstring).strip()
+
+    if not docstring:
+        return Docstring()
+
+    # Detect format based on content
+    if ":param " in docstring or ":type " in docstring or ":return" in docstring:
+        return _parse_rst_docstring(docstring)
+    elif re.search(r"\b(Args|Arguments|Returns?):", docstring):
+        return _parse_google_docstring(docstring)
+    else:
+        # No special formatting, just treat as body
+        return Docstring(body=docstring)
+
+
+def _parse_rst_docstring(docstring: str) -> Docstring:
+    """
+    Parse reStructuredText-style docstring with :param: and :type: directives.
+    """
+    lines = docstring.split("\n")
+
+    result = Docstring()
+    body_lines = []
+
+    for line in lines:
+        if line.strip().startswith(":"):
+            break
+        body_lines.append(line)
+
+    result.body = "\n".join(body_lines).strip()
+    _parse_rst_fields(lines[len(body_lines) :], result)
+    return result
+
+
+def _parse_google_docstring(docstring: str) -> Docstring:
+    """
+    Parse Google-style docstring with Args: and Returns: sections.
+    """
+    lines = docstring.split("\n")
+    result = Docstring()
+
+    # Find sections using regex
+    sections = {}
+    for i, line in enumerate(lines):
+        stripped = line.strip()
+        if re.match(r"^(Args|Arguments):\s*$", stripped, re.IGNORECASE):
+            sections["args"] = i
+        elif re.match(r"^Returns?:\s*$", stripped, re.IGNORECASE):
+            sections["returns"] = i
+
+    # Body is everything before the first section
+    body_end = min(sections.values()) if sections else len(lines)
+    result.body = "\n".join(lines[:body_end]).strip()
+
+    # Parse each section
+    if "args" in sections:
+        _parse_google_args_section(lines, sections["args"] + 1, result, sections)
+    if "returns" in sections:
+        _parse_google_returns_section(lines, sections["returns"] + 1, result, sections)
+
+    return result
+
+
+def _parse_google_args_section(
+    lines: list[str], start_idx: int, result: Docstring, sections: dict[str, int]
+) -> None:
+    """
+    Parse the Args: section of a Google-style docstring.
+    """
+    # Find the end of this section
+    end_idx = len(lines)
+    for section_start in sections.values():
+        if section_start > start_idx:
+            end_idx = min(end_idx, section_start)
+
+    # Determine base indentation from first non-empty line
+    base_indent = None
+    for i in range(start_idx, end_idx):
+        line = lines[i]
+        if line.strip():
+            base_indent = len(line) - len(line.lstrip())
+            break
+
+    if base_indent is None:
+        return
+
+    i = start_idx
+    while i < end_idx:
+        line = lines[i]
+
+        # Skip empty lines
+        if not line.strip():
+            i += 1
+            continue
+
+        # Check if this line is at the base indentation level (parameter line)
+        line_indent = len(line) - len(line.lstrip())
+        if line_indent == base_indent:
+            param_line = line.strip()
+
+            # More robust regex that allows underscores and handles various formats
+            # Match: name (type): description
+            match = re.match(r"([a-zA-Z_]\w*)\s*\(([^)]+)\)\s*:\s*(.*)", param_line)
+            if match:
+                name, param_type, description = match.groups()
+                result.param[name] = description.strip()
+                result.type[name] = param_type.strip()
+            else:
+                # Match: name: description
+                match = re.match(r"([a-zA-Z_]\w*)\s*:\s*(.*)", param_line)
+                if match:
+                    name, description = match.groups()
+                    result.param[name] = description.strip()
+
+            # Collect continuation lines (more indented than base)
+            i += 1
+            continuation_lines = []
+            while i < end_idx:
+                if not lines[i].strip():
+                    i += 1
+                    continue
+                next_indent = len(lines[i]) - len(lines[i].lstrip())
+                if next_indent > base_indent:
+                    continuation_lines.append(lines[i].strip())
+                    i += 1
+                else:
+                    break
+
+            # Add continuation to the last parameter
+            if continuation_lines and result.param:
+                last_param = list(result.param.keys())[-1]
+                result.param[last_param] += " " + " ".join(continuation_lines)
+        else:
+            i += 1
+
+
+def _parse_google_returns_section(
+    lines: list[str], start_idx: int, result: Docstring, sections: dict[str, int]
+) -> None:
+    """
+    Parse the Returns: section of a Google-style docstring.
+    """
+    # Find the end of this section
+    end_idx = len(lines)
+    for section_start in sections.values():
+        if section_start > start_idx:
+            end_idx = min(end_idx, section_start)
+
+    # Collect all content from this section
+    content_lines = []
+    for i in range(start_idx, end_idx):
+        line = lines[i]
+        if line.strip():
+            content_lines.append(line.strip())
+
+    if content_lines:
+        content = " ".join(content_lines).strip()
+
+        # Try to parse "type: description" format
+        if ":" in content and not content.startswith(":"):
+            parts = content.split(":", 1)
+            if len(parts) == 2 and parts[0].strip():
+                result.rtype = parts[0].strip()
+                result.returns = parts[1].strip()
+            else:
+                result.returns = content
+        else:
+            result.returns = content
+
+
+def _parse_rst_fields(lines: list[str], result: Docstring) -> None:
+    """Parse reStructuredText-style field directives."""
+    current_field = None
+    current_content = []
+
+    def save_current_field():
+        if current_field and current_content:
+            content = " ".join(current_content).strip()
+            if current_field.startswith("param "):
+                result.param[current_field[6:]] = content
+            elif current_field.startswith("type "):
+                result.type[current_field[5:]] = content
+            elif current_field == "return":
+                result.returns = content
+            elif current_field == "rtype":
+                result.rtype = content
+
+    for line in lines:
+        if line.strip().startswith(":"):
+            save_current_field()
+            current_field, _, content = line.strip()[1:].partition(":")
+            current_content = [content.strip()]
+        else:
+            current_content.append(line.strip())
+
+    save_current_field()
+
+
+## Tests
+
+
+def test_parse_rst_docstring():
+    rst_docstring = """
+    Search for a string in files at the given paths and return their store paths.
+    Useful to find all docs or resources matching a string or regex.
+
+    :param sort: How to sort results. Can be `path` or `score`.
+    :param ignore_case: Ignore case when searching.
+    :type sort: str
+    :type ignore_case: bool
+    :return: The search results.
+    :rtype: CommandOutput
+    """
+
+    parsed = parse_docstring(rst_docstring)
+
+    assert (
+        parsed.body
+        == "Search for a string in files at the given paths and return their store paths.\nUseful to find all docs or resources matching a string or regex."
+    )
+    assert parsed.param == {
+        "sort": "How to sort results. Can be `path` or `score`.",
+        "ignore_case": "Ignore case when searching.",
+    }
+    assert parsed.type == {"sort": "str", "ignore_case": "bool"}
+    assert parsed.returns == "The search results."
+    assert parsed.rtype == "CommandOutput"
+
+
+def test_parse_google_docstring_with_types():
+    google_docstring = """
+    Search for a string in files at the given paths and return their store paths.
+    Useful to find all docs or resources matching a string or regex.
+
+    Args:
+        sort (str): How to sort results. Can be `path` or `score`.
+        ignore_case (bool): Ignore case when searching.
+        
+    Returns:
+        CommandOutput: The search results.
+    """
+
+    parsed = parse_docstring(google_docstring)
+
+    assert (
+        parsed.body
+        == "Search for a string in files at the given paths and return their store paths.\nUseful to find all docs or resources matching a string or regex."
+    )
+    assert parsed.param == {
+        "sort": "How to sort results. Can be `path` or `score`.",
+        "ignore_case": "Ignore case when searching.",
+    }
+    assert parsed.type == {"sort": "str", "ignore_case": "bool"}
+    assert parsed.returns == "The search results."
+    assert parsed.rtype == "CommandOutput"
+
+
+def test_parse_google_docstring_without_types():
+    google_no_types = """
+    Process the data.
+
+    Args:
+        data: The input data to process.
+        verbose: Whether to print verbose output.
+
+    Returns:
+        The processed result.
+    """
+
+    parsed = parse_docstring(google_no_types)
+
+    assert parsed.body == "Process the data."
+    assert parsed.param == {
+        "data": "The input data to process.",
+        "verbose": "Whether to print verbose output.",
+    }
+    assert parsed.type == {}
+    assert parsed.returns == "The processed result."
+    assert parsed.rtype == ""
+
+
+def test_parse_simple_docstring():
+    simple_docstring = """Some text."""
+    parsed = parse_docstring(simple_docstring)
+
+    assert parsed.body == "Some text."
+    assert parsed.param == {}
+    assert parsed.type == {}
+    assert parsed.returns == ""
+    assert parsed.rtype == ""
+
+
+def test_parse_docstring_with_underscores():
+    docstring = """
+    Test function.
+
+    Args:
+        some_param (str): A parameter with underscores.
+        another_param_name: Another parameter without type.
+    """
+
+    parsed = parse_docstring(docstring)
+
+    assert parsed.param == {
+        "some_param": "A parameter with underscores.",
+        "another_param_name": "Another parameter without type.",
+    }
+    assert parsed.type == {"some_param": "str"}
+
+
+def test_parse_empty_docstring():
+    """Test empty docstring handling."""
+    parsed = parse_docstring("")
+    assert parsed.body == ""
+    assert parsed.param == {}
+    assert parsed.type == {}
+    assert parsed.returns == ""
+    assert parsed.rtype == ""

kash/web_content/web_extract.py

@@ -1,38 +1,57 @@
 from funlog import log_calls
 
 from kash.utils.common.url import Url
+from kash.utils.file_utils.file_formats_model import file_format_info
 from kash.web_content.canon_url import thumbnail_url
 from kash.web_content.file_cache_utils import cache_file
 from kash.web_content.web_extract_justext import extract_text_justext
-from kash.web_content.web_fetch import fetch_url
 from kash.web_content.web_page_model import PageExtractor, WebPageData
 
 
 @log_calls(level="message")
-def fetch_extract(
+def fetch_page_content(
     url: Url,
+    *,
     refetch: bool = False,
-    use_cache: bool = True,
-    extractor: PageExtractor = extract_text_justext,
+    cache: bool = True,
+    text_extractor: PageExtractor = extract_text_justext,
 ) -> WebPageData:
     """
     Fetches a URL and extracts the title, description, and content.
-    By default, uses the content cache if available. Can force re-fetching and
-    updating the cache by setting `refetch` to true.
+    Always uses the content cache, at least temporarily.
+
+    Force re-fetching and updating the cache by setting `refetch` to true.
+    Cached file path is returned in the content, unless `cache` is false,
+    in case the cached content is deleted.
+
+    For HTML and other text files, uses the `text_extractor` to extract
+    clean text and page metadata.
     """
     expiration_sec = 0 if refetch else None
-    if use_cache:
-        path = cache_file(url, expiration_sec=expiration_sec).content.path
-        with open(path, "rb") as file:
-            content = file.read()
-        page_data = extractor(url, content)
+
+    path = cache_file(url, expiration_sec=expiration_sec).content.path
+    format_info = file_format_info(path)
+
+    content = None
+    if format_info.format and format_info.format.is_text:
+        content = path.read_bytes()
+        page_data = text_extractor(url, content)
     else:
-        response = fetch_url(url)
-        page_data = extractor(url, response.content)
+        page_data = WebPageData(url)
 
-    # Add a thumbnail, if available.
+    # Add file format info (for both HTML/text and all other file types).
+
+    page_data.format_info = format_info
+
+    # Add a thumbnail, if known for this URL.
     page_data.thumbnail_url = thumbnail_url(url)
 
+    # Return the local cache path if we will be keeping it.
+    if cache:
+        page_data.saved_content = path
+    else:
+        path.unlink()
+
     return page_data
 
 
@@ -53,5 +72,5 @@ if __name__ == "__main__":
 
     for url in sample_urls:
         print(f"URL: {url}")
-        print(fetch_extract(Url(url)))
+        print(fetch_page_content(Url(url)))
         print()

kash/web_content/web_page_model.py

@@ -5,12 +5,19 @@ from prettyfmt import abbrev_obj
 from pydantic.dataclasses import dataclass
 
 from kash.utils.common.url import Url
+from kash.utils.file_utils.file_formats_model import FileFormatInfo
 
 
 @dataclass
 class WebPageData:
     """
-    Data about a web page, including URL, title and optionally description and extracted content.
+    Data about a web page, including URL, title and optionally description and
+    extracted content.
+
+    The `text` field should be a clean text version of the page, if available.
+    The `clean_html` field should be a clean HTML version of the page, if available.
+    The `saved_content` is optional but can be used to reference the original content,
+    especially for large or non-text content.
     """
 
     locator: Url | Path
@@ -19,6 +26,8 @@ class WebPageData:
     description: str | None = None
     text: str | None = None
     clean_html: str | None = None
+    saved_content: Path | None = None
+    format_info: FileFormatInfo | None = None
     thumbnail_url: Url | None = None
 
     def __repr__(self):

{kash_shell-0.3.18.dist-info → kash_shell-0.3.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kash-shell
-Version: 0.3.18
+Version: 0.3.20
 Summary: The knowledge agent shell (core)
 Project-URL: Repository, https://github.com/jlevy/kash-shell
 Author-email: Joshua Levy <joshua@cal.berkeley.edu>
@@ -115,7 +115,7 @@ the Python framework, a few core utilities, and the Kash command-line shell.
 Additional actions for handling more complex tasks like converting documents and
 transcribing, researching, or annotating videos, are in the
 [kash-docs](https://github.com/jlevy/kash-docs) and
-[kash-media](https://github.com/jlevy/kash-docs) packages, all available on PyPI and
+[kash-media](https://github.com/jlevy/kash-media) packages, all available on PyPI and
 quick to install via uv.
 
 ### Key Concepts

{kash_shell-0.3.18.dist-info → kash_shell-0.3.20.dist-info}/RECORD

@@ -4,7 +4,7 @@ kash/actions/__init__.py,sha256=a4pQw8O-Y3q5N4Qg2jUV0xEZLX6d164FQhZ6zizY9fE,1357
 kash/actions/core/assistant_chat.py,sha256=28G20cSr7Z94cltouTPve5TXY3km0lACrRvpLE27fK8,1837
 kash/actions/core/chat.py,sha256=yCannBFa0cSpR_in-XSSuMm1x2ZZQUCKmlqzhsUfpOo,2696
 kash/actions/core/format_markdown_template.py,sha256=ZJbtyTSypPo2ewLiGRSyIpVf711vQMhI_-Ng-FgCs80,2991
-kash/actions/core/markdownify.py,sha256=KjdUeY4c9EhZ5geQrn22IoBv0P_p62q4zyyOYE0NRHM,1270
+kash/actions/core/markdownify_html.py,sha256=RXsC59rhDoLssRujkS32PETN1zzncIofFO7J3qTrnJc,1277
 kash/actions/core/minify_html.py,sha256=99r3SjpI2NQP7e5MnMixAiT5lxPx7t2nyJvJi6Yps6w,1365
 kash/actions/core/readability.py,sha256=ljdB2rOpzfKU2FpEJ2UELIzcdOAWvdUjFsxoHRTE3xo,989
 kash/actions/core/render_as_html.py,sha256=CIPGKCjUEVNsnXmpqHCUnjGwTfEfOyCXxlYFUN8mahY,1870
@@ -35,7 +35,7 @@ kash/commands/help/help_commands.py,sha256=eJTpIhXck123PAUq2k-D3Q6UL6IQ8atOVYurL
 kash/commands/help/logo.py,sha256=W8SUach9FjoTqpHZwTGS582ry4ZluxbBp86ZCiAtDkY,3505
 kash/commands/help/welcome.py,sha256=F4QBgj3e1dM9Pf0H4TSzCrkVfXQVKUIl0b6Qmofbdo4,905
 kash/commands/workspace/selection_commands.py,sha256=nZzA-H7Pk8kqSJVRlX7j1m6cZX-e0X8isOryDU41vqU,8156
-kash/commands/workspace/workspace_commands.py,sha256=ZJ3aPsnQ0FOkaA6stpV4YPEOQRCOKTazbMCIQkk9Cmk,25119
+kash/commands/workspace/workspace_commands.py,sha256=_2TcthGOu-nU9E_-jjf4kba9ldLRA6qe6Do6zV06EKc,21960
 kash/config/__init__.py,sha256=ytly9Typ1mWV4CXfV9G3CIPtPQ02u2rpZ304L3GlFro,148
 kash/config/capture_output.py,sha256=ud3uUVNuDicHj3mI_nBUBO-VmOrxtBdA3z-I3D1lSCU,2398
 kash/config/colors.py,sha256=qUUUE-x8srSp1IdJePNAUtNQbOZKBLix8FeOe-Rxmgg,13421
@@ -61,7 +61,7 @@ kash/docs/markdown/assistant_instructions_template.md,sha256=Fzeledd_nr3bKhvQ1qZ
 kash/docs/markdown/readme_template.md,sha256=iGx9IjSni1t_9BuYD5d2GgkxkNIkqvE3k78IufHF6Yg,409
 kash/docs/markdown/warning.md,sha256=bG0T3UFqAkzF8217J8AbIbQ7ftJ_GKMRilIGq9eLdYc,162
 kash/docs/markdown/welcome.md,sha256=yp_tmGXGIb8043ZDIL97Q1uaKVys7-ArHCprgDlon7w,611
-kash/docs/markdown/topics/a1_what_is_kash.md,sha256=zR1UNXSXwbk9vMmSTSyYADRNTxMcuYU6C0mzF7KS8PM,6757
+kash/docs/markdown/topics/a1_what_is_kash.md,sha256=rgVrv6tRXEwdqQ54DAfHP3BSAuq8Ux4wCNeluTwpkhU,6758
 kash/docs/markdown/topics/a2_installation.md,sha256=DSzaniHjOYPC3soGLPTGOGDVvbiPTROtb3S8zYUCPEs,5736
 kash/docs/markdown/topics/a3_getting_started.md,sha256=xOMevEXMIpVJvTGuuwI9Cc9sun3tQM3OqCgynSgMpeM,9376
 kash/docs/markdown/topics/a4_elements.md,sha256=XNJRw-iqnytiIHOAshp1YnUpHM5KBgFAhuOdp_fekxQ,4615
@@ -81,14 +81,14 @@ kash/docs_base/recipes/tldr_standard_commands.sh,sha256=7nPES55aT45HF3eDhQRrEUiW
 kash/embeddings/cosine.py,sha256=QTWPWUHivXjxCM6APSqij_-4mywM2BVVm0xb0hu7FHA,1587
 kash/embeddings/embeddings.py,sha256=v6RmrEHsx5PuE3fPrY15RK4fgW0K_VlNWDTjCVr11zY,4451
 kash/embeddings/text_similarity.py,sha256=BOo9Vcs5oi2Zs5La56uTkPMHo65XSd4qz_yr6GTfUA4,1924
-kash/exec/__init__.py,sha256=rdSsKzTaXfSZmD5JvmUSSwmpfvl-moNv9PUgtE_WUpQ,1148
+kash/exec/__init__.py,sha256=Najls8No143yoj_KAaOQgo8ufC2LWCB_DwwEQ-8nDM0,1277
 kash/exec/action_decorators.py,sha256=VOSCnFiev2_DuFoSk0i_moejwM4wJ1j6QfsQd93uetI,16480
-kash/exec/action_exec.py,sha256=k0HtAvpfISzEN6GQE-iMU36EihmwCzuOa679bk7b8jQ,19022
+kash/exec/action_exec.py,sha256=O_4UB_Vt7QRxltviMeBwNIfw9ten06n4fQ39MregacE,19017
 kash/exec/action_registry.py,sha256=numU9pH_W5RgIrYmfi0iYMYy_kLJl6vup8PMrhxAfdc,2627
 kash/exec/combiners.py,sha256=AJ6wgPUHsmwanObsUw64B83XzU26yuh5t4l7igLn82I,4291
 kash/exec/command_exec.py,sha256=zc-gWm7kyB5J5Kp8xhULQ9Jj9AL927KkDPXXk-Yr1Bw,1292
 kash/exec/command_registry.py,sha256=1s2ogU8b8nqK_AEtslbr1eYrXCGDkeT30UrB7L0BRoM,2027
-kash/exec/fetch_url_metadata.py,sha256=-ojwEpOpDNIS5xBSVJ7oHSgoue_ABywlZ2Y2A5fh0Kg,3159
+kash/exec/fetch_url_items.py,sha256=UUj3wrP3adzZHV_Icx7zSB9zRtl0S7EC54Ios8fuvbg,4404
 kash/exec/history.py,sha256=l2XwHGBR1UgTGSFPSBE9mltmxvjR_5qFFO6d-Z008nc,1208
 kash/exec/importing.py,sha256=xunmBapeUMNc6Zox7y6e_DZkidyWeouiFZpphajwSzc,1843
 kash/exec/llm_transforms.py,sha256=p_aLp70VoIgheW4v8uoweeuEVWj06AzQekvn_jM3B-g,4378
@@ -104,22 +104,21 @@ kash/exec_model/commands_model.py,sha256=iM8QhzA0tAas5OwF5liUfHtm45XIH1LcvCviuh3
 kash/exec_model/script_model.py,sha256=1VG3LhkTmlKzHOYouZ92ZpOSKSCcsz3-tHNcFMQF788,5031
 kash/exec_model/shell_model.py,sha256=LUhQivbpXlerM-DUzNY7BtctNBbn08Wto8CSSxQDxRU,568
 kash/file_storage/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kash/file_storage/file_store.py,sha256=5Hkw_fE-WbJyu8OeZoA8hCpVOnNYclsWY0iotHSnv10,30690
+kash/file_storage/file_store.py,sha256=_OxcExZOQ-ef9Sm2sdD28BSMcQ64KsSikq3CJPUKYUU,30886
 kash/file_storage/item_file_format.py,sha256=_o2CjWstk_Z__qMr-Inct9wJm2VEUK0GZvF-fDZ8bcc,5377
 kash/file_storage/metadata_dirs.py,sha256=9AqO3S3SSY1dtvP2iLX--E4ui0VIzXttG8R040otfyg,3820
 kash/file_storage/persisted_yaml.py,sha256=4-4RkFqdlBUkTOwkdA4vRKUywEE9TaDo13OGaDUyU9M,1309
 kash/file_storage/store_cache_warmer.py,sha256=cQ_KwxkBPWT3lMmYOCTkXgo7CKaGINns2YzIH32ExSU,1013
-kash/file_storage/store_filenames.py,sha256=RmuZ3hHuo95bV9Jv5qtUqN8NdJU8qI_7SYqgc5B5UL4,1720
+kash/file_storage/store_filenames.py,sha256=zvmVQxQy8QOHkUeh-6iFKlBSk0GzqUt9xmmqkh45Zdo,1909
 kash/help/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kash/help/assistant.py,sha256=R0XHNi-h51QoQ7rGouD2chrDPGomYaPQUqJdvpjlCs8,11535
 kash/help/assistant_instructions.py,sha256=jW5XAsmLx8YZMKSDJgWnqo9Vwe7VuiTURQHjKBqr_L8,2549
 kash/help/assistant_output.py,sha256=9sM-OVLc6eMSOkxyovB88dNlsknFpf8Wz89Zp5PuEA8,1668
-kash/help/docstring_utils.py,sha256=80vcEGR05G1wwIfzaKhX5QzU38mkKna-gN6_wTNHjXg,3208
-kash/help/function_param_info.py,sha256=IVi6dtRjAWUCxpy9rZ1fRqKLCciK_kuMybTwCZ9XiYg,1692
+kash/help/function_param_info.py,sha256=yGuFLVZoDF1E1YsXqGwJhuY2uJB3R1B1EC77MEMSc-U,1700
 kash/help/help_embeddings.py,sha256=le7yqxGOtzIU_LUkl-Ef-7WQGuWa0wAOx9umZGva8zg,2787
 kash/help/help_lookups.py,sha256=0dtuLWEXncqhJCijC98IA9stBDNNcJewt1JYqMLkTx4,2029
-kash/help/help_pages.py,sha256=XyV3SN1cvGjpS8xAbZ_X_6Z9hTIH7vXdgK8DCX2cwNg,3886
-kash/help/help_printing.py,sha256=NAJa8qfaZGce5NKtUqdyDE8Ey0ag2ywjL2LP8ACT060,6078
+kash/help/help_pages.py,sha256=TaKsE26R-pZTrK4Pa593DK5osdJodFHaVm5pZpjqgaI,3894
+kash/help/help_printing.py,sha256=eZbZdyJC158JiXcEk2zvUmqYbYzbYOpHvxEhC1kIN-Q,6086
 kash/help/help_types.py,sha256=xo0AXfaNjlMrWp6dkGixo6P61n1tIhFhlvv9sWhNBrI,7909
 kash/help/recommended_commands.py,sha256=jqc3TjWFBqDJ-iSzXn8vTOplb4uHndwvdAGJfcUV_qs,2486
 kash/help/tldr_help.py,sha256=bcu__MIF4vYlZEeqQqieGIBcRhNCTK5u8jPV08ObzCI,9654
@@ -165,7 +164,7 @@ kash/model/compound_actions_model.py,sha256=HiDK5wwCu3WwZYHATZoLEguiqwR9V6V296wi
 kash/model/concept_model.py,sha256=we2qOcy9Mv1q7XPfkDLp_CyO_-8DwAUfUYlpgy_jrFs,1011
 kash/model/exec_model.py,sha256=IlfvtQyoFRRWhWju7vdXp9J-w_NGcGtL5DhDLy9gRd8,2250
 kash/model/graph_model.py,sha256=jnctrPiBZ0xwAR8D54JMAJPanA1yZdaxSFQoIpe8anA,2662
-kash/model/items_model.py,sha256=B2YYIppOiPX-n5dseMqdp_PPMBJfie-EZ1yZGWeq-iQ,35201
+kash/model/items_model.py,sha256=ZWAsqTHA4p6GSNz3QjPr46LRwY7pbv7d73_KqU3gsO0,35686
 kash/model/language_list.py,sha256=I3RIbxTseVmPdhExQimimEv18Gmy2ImMbpXe0-_t1Qw,450
 kash/model/llm_actions_model.py,sha256=a29uXVNfS2CiqvM7HPdC6H9A23rSQQihAideuBLMH8g,2110
 kash/model/media_model.py,sha256=ZnlZ-FkswbAIGpUAuNqLce1WDZK-WbnwHn2ipg8x7-0,3511
@@ -206,6 +205,7 @@ kash/utils/common/function_inspect.py,sha256=gczPhFlrF4NotkJKw8rDcl1DFlWfHdur_J4
 kash/utils/common/import_utils.py,sha256=zyCa5sG_vTxzgIgjOS98xAwqkSeCQzN-8UkM6k9ZZOI,4615
 kash/utils/common/lazyobject.py,sha256=9dmOfSheblOXgo2RRakMwgfPIKdTgtyrlm6dCKAze04,5157
 kash/utils/common/obj_replace.py,sha256=AuiXptUOnuDNcWDgAJ3jEHkLh89XIqCP_SOkgaVyFIQ,2075
+kash/utils/common/parse_docstring.py,sha256=oM1ecGGySRA3L_poddjReJ_qPY5506Le7E8_CDUrU8k,10922
 kash/utils/common/parse_key_vals.py,sha256=yZRZIa5GD9SlnBSn2YNZm8PRVKoSJMY8DCmdGujQj_I,2418
 kash/utils/common/parse_shell_args.py,sha256=UZXTZDbV5m5Jy39jdAQ6W8uilr1TNa0__RqnE8UmQ_M,10604
 kash/utils/common/stack_traces.py,sha256=a2NwlK_0xxnjMCDC4LrQu7ueFylF-OImFG3bAAHpPwY,1392
@@ -245,11 +245,11 @@ kash/web_content/dir_store.py,sha256=BJc-s-RL5CC-GwhFTC_lhLXSMWluPPnLVmVBx-66DiM
 kash/web_content/file_cache_utils.py,sha256=JRXUCAmrc83iAgdiICU2EYGWcoORflWNl6GAVq-O80I,5529
 kash/web_content/file_processing.py,sha256=cQC-MnJMM5qG9-y0S4yobkmRi6A75qhHjV6xTwbtYDY,1904
 kash/web_content/local_file_cache.py,sha256=PEDKU5VIwhCnSC-HXG4EkO2OzrOUDuuDBMuo3lP2EN0,9466
-kash/web_content/web_extract.py,sha256=LbuG4AFEeIiXyUrN9CAxX0ret41Fqu_iTJSjIWyk3Bg,2296
+kash/web_content/web_extract.py,sha256=FLn4LPAQHe79kx9LhSbGNnvoDM59X-AqeyBMkyLdyZo,2847
 kash/web_content/web_extract_justext.py,sha256=74HLJBKDGKatwxyRDX6za70bZG9LrVmtj9jLX7UJzg4,2540
 kash/web_content/web_extract_readabilipy.py,sha256=IT7ET5IoU2-Nf37-Neh6CkKMvLL3WTNVJjq7ZMOx6OM,808
 kash/web_content/web_fetch.py,sha256=J8DLFP1vzp7aScanFq0Bd7xCP6AVL4JgMMBqyRPtZjQ,4720
-kash/web_content/web_page_model.py,sha256=9bPuqZxXo6hSUB_llEcz8bs3W1lW0r-Y3Q7pZgknlQU,693
+kash/web_content/web_page_model.py,sha256=aPpgC1fH2z2LTzGJhEDvZgq_mYwgsQIZaDS3UE7v98w,1147
 kash/web_gen/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kash/web_gen/simple_webpage.py,sha256=ks_0ljxCeS2-gAAEaUc1JEnzY3JY0nzqGFiyyqyRuZs,1537
 kash/web_gen/tabbed_webpage.py,sha256=DiZV48TVvcjOf31g3nzTAtGKpH5Cek1Unksr7Cwcwog,4949
@@ -286,8 +286,8 @@ kash/xonsh_custom/xonsh_modern_tools.py,sha256=mj_b34LZXfE8MJe9EpDmp5JZ0tDM1biYN
 kash/xonsh_custom/xonsh_ranking_completer.py,sha256=ZRGiAfoEgqgnlq2-ReUVEaX5oOgW1DQ9WxIv2OJLuTo,5620
 kash/xontrib/fnm.py,sha256=V2tsOdmIDgbFbZSfMLpsvDIwwJJqiYnOkOySD1cXNXw,3700
 kash/xontrib/kash_extension.py,sha256=FLIMlgR3C_6A1fwKE-Ul0nmmpJSszVPbAriinUyQ8Zg,1896
-kash_shell-0.3.18.dist-info/METADATA,sha256=5Lnqsbx7FMfH3GPjaANtJ5cBdONspmqPUf4oxvfeoG0,32585
-kash_shell-0.3.18.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-kash_shell-0.3.18.dist-info/entry_points.txt,sha256=SQraWDAo8SqYpthLXThei0mf_hGGyhYBUO-Er_0HcwI,85
-kash_shell-0.3.18.dist-info/licenses/LICENSE,sha256=rCh2PsfYeiU6FK_0wb58kHGm_Fj5c43fdcHEexiVzIo,34562
-kash_shell-0.3.18.dist-info/RECORD,,
+kash_shell-0.3.20.dist-info/METADATA,sha256=aImcO_J_PXzx57HrdQOq66nSLoqCp5oBg6__ZzSwN8k,32586
+kash_shell-0.3.20.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+kash_shell-0.3.20.dist-info/entry_points.txt,sha256=SQraWDAo8SqYpthLXThei0mf_hGGyhYBUO-Er_0HcwI,85
+kash_shell-0.3.20.dist-info/licenses/LICENSE,sha256=rCh2PsfYeiU6FK_0wb58kHGm_Fj5c43fdcHEexiVzIo,34562
+kash_shell-0.3.20.dist-info/RECORD,,

kash/help/docstring_utils.py

@@ -1,111 +0,0 @@
-from dataclasses import field
-from textwrap import dedent
-
-from pydantic.dataclasses import dataclass
-
-
-@dataclass
-class Docstring:
-    body: str = ""
-    param: dict[str, str] = field(default_factory=dict)
-    type: dict[str, str] = field(default_factory=dict)
-    returns: str = ""
-    rtype: str = ""
-
-
-def parse_docstring(docstring: str) -> Docstring:
-    """
-    Parse a reStructuredText-style docstring.
-    """
-    # TODO: Support other standard docstring formats too.
-    docstring = dedent(docstring).strip()
-
-    lines = docstring.split("\n")
-
-    result = Docstring()
-    body_lines = []
-
-    for line in lines:
-        if line.strip().startswith(":"):
-            break
-        body_lines.append(line)
-
-    result.body = "\n".join(body_lines).strip()
-
-    parse_fields(lines[len(body_lines) :], result)
-
-    return result
-
-
-def parse_fields(lines: list[str], result: Docstring):
-    current_field = None
-    current_content = []
-
-    def save_current_field():
-        if current_field and current_content:
-            content = " ".join(current_content).strip()
-            if current_field.startswith("param "):
-                result.param[current_field[6:]] = content
-            elif current_field.startswith("type "):
-                result.type[current_field[5:]] = content
-            elif current_field == "return":
-                result.returns = content
-            elif current_field == "rtype":
-                result.rtype = content
-
-    for line in lines:
-        if line.strip().startswith(":"):
-            save_current_field()
-            current_field, _, content = line.strip()[1:].partition(":")
-            current_content = [content.strip()]
-        else:
-            current_content.append(line.strip())
-
-    save_current_field()
-
-
-## Tests
-
-
-def test_parse_docstring():
-    docstring1 = """
-    Search for a string in files at the given paths and return their store paths.
-    Useful to find all docs or resources matching a string or regex.
-
-    :param sort: How to sort results. Can be `path` or `score`.
-    :param ignore_case: Ignore case when searching.
-    :type sort: str
-    :type ignore_case: bool
-    :return: The search results.
-    :rtype: CommandOutput
-    """
-
-    parsed1 = parse_docstring(docstring1)
-
-    print(f"Body: {parsed1.body}")
-    print(f"Params: {parsed1.param}")
-    print(f"Types: {parsed1.type}")
-    print(f"Returns: {parsed1.returns}")
-    print(f"Return type: {parsed1.rtype}")
-
-    assert (
-        parsed1.body
-        == "Search for a string in files at the given paths and return their store paths.\nUseful to find all docs or resources matching a string or regex."
-    )
-    assert parsed1.param == {
-        "sort": "How to sort results. Can be `path` or `score`.",
-        "ignore_case": "Ignore case when searching.",
-    }
-    assert parsed1.type == {"sort": "str", "ignore_case": "bool"}
-    assert parsed1.returns == "The search results."
-    assert parsed1.rtype == "CommandOutput"
-
-    docstring2 = """Some text."""
-
-    parsed2 = parse_docstring(docstring2)
-
-    assert parsed2.body == "Some text."
-    assert parsed2.param == {}
-    assert parsed2.type == {}
-    assert parsed2.returns == ""
-    assert parsed2.rtype == ""