PyPI - kash-shell - Versions diffs - 0.3.8__py3-none-any.whl → 0.3.10__py3-none-any.whl - Mend

kash-shell 0.3.8py3-none-any.whl → 0.3.10py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (154) hide show

kash/actions/__init__.py +4 -4
kash/actions/core/markdownify.py +5 -2
kash/actions/core/readability.py +5 -2
kash/actions/core/render_as_html.py +18 -0
kash/actions/core/webpage_config.py +12 -4
kash/commands/__init__.py +8 -20
kash/commands/base/basic_file_commands.py +15 -0
kash/commands/base/debug_commands.py +15 -2
kash/commands/base/general_commands.py +27 -18
kash/commands/base/logs_commands.py +1 -4
kash/commands/base/model_commands.py +8 -8
kash/commands/base/search_command.py +3 -2
kash/commands/base/show_command.py +5 -3
kash/commands/extras/parse_uv_lock.py +186 -0
kash/commands/help/doc_commands.py +2 -31
kash/commands/help/welcome.py +33 -0
kash/commands/workspace/selection_commands.py +11 -6
kash/commands/workspace/workspace_commands.py +19 -16
kash/config/colors.py +2 -0
kash/config/env_settings.py +72 -0
kash/config/init.py +2 -2
kash/config/logger.py +61 -59
kash/config/logger_basic.py +12 -5
kash/config/server_config.py +6 -6
kash/config/settings.py +117 -67
kash/config/setup.py +35 -9
kash/config/suppress_warnings.py +30 -12
kash/config/text_styles.py +3 -13
kash/docs/load_api_docs.py +2 -1
kash/docs/markdown/topics/a2_installation.md +7 -3
kash/docs/markdown/topics/a3_getting_started.md +3 -2
kash/docs/markdown/warning.md +3 -8
kash/docs/markdown/welcome.md +4 -0
kash/docs_base/load_recipe_snippets.py +1 -1
kash/docs_base/recipes/{general_system_commands.ksh → general_system_commands.sh} +1 -1
kash/{concepts → embeddings}/cosine.py +2 -1
kash/embeddings/text_similarity.py +57 -0
kash/exec/__init__.py +20 -3
kash/exec/action_decorators.py +18 -4
kash/exec/action_exec.py +41 -23
kash/exec/action_registry.py +13 -48
kash/exec/command_registry.py +2 -1
kash/exec/fetch_url_metadata.py +4 -6
kash/exec/importing.py +56 -0
kash/exec/llm_transforms.py +6 -6
kash/exec/precondition_registry.py +2 -1
kash/exec/preconditions.py +16 -1
kash/exec/shell_callable_action.py +33 -19
kash/file_storage/file_store.py +23 -14
kash/file_storage/item_file_format.py +13 -3
kash/file_storage/metadata_dirs.py +11 -2
kash/help/assistant.py +2 -2
kash/help/assistant_instructions.py +2 -1
kash/help/help_embeddings.py +2 -2
kash/help/help_printing.py +14 -10
kash/help/tldr_help.py +5 -3
kash/llm_utils/clean_headings.py +1 -1
kash/llm_utils/llm_api_keys.py +4 -4
kash/llm_utils/llm_completion.py +2 -2
kash/llm_utils/llm_features.py +68 -0
kash/llm_utils/llm_messages.py +1 -2
kash/llm_utils/llm_names.py +1 -1
kash/llm_utils/llms.py +17 -12
kash/local_server/__init__.py +5 -2
kash/local_server/local_server.py +56 -46
kash/local_server/local_server_commands.py +15 -15
kash/local_server/local_server_routes.py +2 -2
kash/local_server/local_url_formatters.py +1 -1
kash/mcp/__init__.py +5 -2
kash/mcp/mcp_cli.py +54 -17
kash/mcp/mcp_server_commands.py +5 -6
kash/mcp/mcp_server_routes.py +14 -11
kash/mcp/mcp_server_sse.py +61 -34
kash/mcp/mcp_server_stdio.py +0 -8
kash/media_base/audio_processing.py +81 -7
kash/media_base/media_cache.py +18 -18
kash/media_base/media_services.py +1 -1
kash/media_base/media_tools.py +6 -6
kash/media_base/services/local_file_media.py +2 -2
kash/media_base/{speech_transcription.py → transcription_deepgram.py} +25 -109
kash/media_base/transcription_format.py +73 -0
kash/media_base/transcription_whisper.py +38 -0
kash/model/__init__.py +73 -5
kash/model/actions_model.py +38 -4
kash/model/concept_model.py +30 -0
kash/model/items_model.py +56 -13
kash/model/params_model.py +24 -0
kash/shell/completions/completion_scoring.py +37 -5
kash/shell/output/kerm_codes.py +1 -2
kash/shell/output/shell_formatting.py +14 -4
kash/shell/shell_main.py +2 -2
kash/shell/utils/exception_printing.py +6 -0
kash/shell/utils/native_utils.py +26 -20
kash/text_handling/custom_sliding_transforms.py +12 -4
kash/text_handling/doc_normalization.py +6 -2
kash/text_handling/markdown_render.py +117 -0
kash/text_handling/markdown_utils.py +204 -0
kash/utils/common/import_utils.py +12 -3
kash/utils/common/type_utils.py +0 -29
kash/utils/common/url.py +80 -28
kash/utils/errors.py +6 -0
kash/utils/file_utils/{dir_size.py → dir_info.py} +25 -4
kash/utils/file_utils/file_ext.py +2 -3
kash/utils/file_utils/file_formats.py +28 -2
kash/utils/file_utils/file_formats_model.py +50 -19
kash/utils/file_utils/filename_parsing.py +10 -4
kash/web_content/dir_store.py +1 -2
kash/web_content/file_cache_utils.py +37 -10
kash/web_content/file_processing.py +68 -0
kash/web_content/local_file_cache.py +12 -9
kash/web_content/web_extract.py +8 -3
kash/web_content/web_fetch.py +12 -4
kash/web_gen/tabbed_webpage.py +5 -2
kash/web_gen/templates/base_styles.css.jinja +120 -14
kash/web_gen/templates/base_webpage.html.jinja +60 -13
kash/web_gen/templates/content_styles.css.jinja +4 -2
kash/web_gen/templates/item_view.html.jinja +2 -2
kash/web_gen/templates/tabbed_webpage.html.jinja +1 -2
kash/workspaces/__init__.py +15 -2
kash/workspaces/selections.py +18 -3
kash/workspaces/source_items.py +4 -2
kash/workspaces/workspace_output.py +11 -4
kash/workspaces/workspaces.py +5 -11
kash/xonsh_custom/command_nl_utils.py +40 -19
kash/xonsh_custom/custom_shell.py +44 -12
kash/xonsh_custom/customize_prompt.py +39 -21
kash/xonsh_custom/load_into_xonsh.py +26 -27
kash/xonsh_custom/shell_load_commands.py +2 -2
kash/xonsh_custom/xonsh_completers.py +2 -249
kash/xonsh_custom/xonsh_keybindings.py +282 -0
kash/xonsh_custom/xonsh_modern_tools.py +3 -3
kash/xontrib/kash_extension.py +5 -6
{kash_shell-0.3.8.dist-info → kash_shell-0.3.10.dist-info}/METADATA +26 -12
{kash_shell-0.3.8.dist-info → kash_shell-0.3.10.dist-info}/RECORD +140 -140
{kash_shell-0.3.8.dist-info → kash_shell-0.3.10.dist-info}/entry_points.txt +1 -1
kash/concepts/concept_formats.py +0 -23
kash/concepts/text_similarity.py +0 -112
kash/shell/clideps/api_keys.py +0 -99
kash/shell/clideps/dotenv_setup.py +0 -114
kash/shell/clideps/dotenv_utils.py +0 -89
kash/shell/clideps/pkg_deps.py +0 -232
kash/shell/clideps/platforms.py +0 -11
kash/shell/clideps/terminal_features.py +0 -56
kash/shell/utils/osc_utils.py +0 -95
kash/shell/utils/terminal_images.py +0 -133
kash/text_handling/markdown_util.py +0 -167
kash/utils/common/atomic_var.py +0 -158
kash/utils/common/string_replace.py +0 -93
kash/utils/common/string_template.py +0 -101
/kash/docs_base/recipes/{python_dev_commands.ksh → python_dev_commands.sh} +0 -0
/kash/docs_base/recipes/{tldr_standard_commands.ksh → tldr_standard_commands.sh} +0 -0
/kash/{concepts → embeddings}/embeddings.py +0 -0
{kash_shell-0.3.8.dist-info → kash_shell-0.3.10.dist-info}/WHEEL +0 -0
{kash_shell-0.3.8.dist-info → kash_shell-0.3.10.dist-info}/licenses/LICENSE +0 -0

kash/utils/file_utils/file_formats.py CHANGED Viewed

@@ -1,11 +1,12 @@
 import re
+import tempfile
 from pathlib import Path
 from typing import NewType
 import regex
+from clideps.pkgs.pkg_check import pkg_check
 from kash.config.logger import get_logger
-from kash.shell.clideps.pkg_deps import Pkg, pkg_check
 log = get_logger(__name__)
@@ -77,13 +78,15 @@ def read_partial_text(
 MimeType = NewType("MimeType", str)
+MIME_EMPTY = MimeType("inode/x-empty")
 def detect_mime_type(filename: str | Path) -> MimeType | None:
     """
     Get the mime type of a file using libmagic heuristics plus more careful
     detection of HTML, Markdown, and multipart YAML.
     """
-    pkg_check().require(Pkg.libmagic)
+    pkg_check().require("libmagic")
     import magic
     mime = magic.Magic(mime=True)
@@ -132,3 +135,26 @@ def mime_type_is_text(mime_type: MimeType) -> bool:
             "application/rtf",
         }
     )
+## Tests
+def test_detect_mime_type():
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tmpdir_path = Path(tmpdir)
+        empty_file = tmpdir_path / "empty.txt"
+        empty_file.touch()
+        html_file = tmpdir_path / "example.html"
+        with open(html_file, "w") as f:
+            f.write("<!DOCTYPE html>\n<html><body><h1>Test</h1></body></html>")
+        text_file = tmpdir_path / "example.txt"
+        with open(text_file, "w") as f:
+            f.write("This is a simple text file with some content.")
+        assert detect_mime_type(empty_file) == MIME_EMPTY
+        assert detect_mime_type(html_file) == "text/html"
+        assert detect_mime_type(text_file) == "text/plain"

kash/utils/file_utils/file_formats_model.py CHANGED Viewed

@@ -6,7 +6,12 @@ from pathlib import Path
 from kash.utils.common.url import Url, is_file_url, parse_file_url
 from kash.utils.file_utils.file_ext import FileExt
-from kash.utils.file_utils.file_formats import MimeType, detect_mime_type, mime_type_is_text
+from kash.utils.file_utils.file_formats import (
+    MIME_EMPTY,
+    MimeType,
+    detect_mime_type,
+    mime_type_is_text,
+)
 from kash.utils.file_utils.filename_parsing import parse_file_ext
@@ -31,6 +36,8 @@ class Format(Enum):
     it is the format of the resource (url, media, etc.).
     """
+    # TODO: Be more thorough, pulling in relevant extensions and types from the `mimetypes` module.
     # Formats with no body (content is in frontmatter).
     url = "url"
@@ -44,8 +51,9 @@ class Format(Enum):
     yaml = "yaml"
     diff = "diff"
     python = "python"
-    kash_script = "kash_script"
-    """Our own format for kash scripts."""
+    shellscript = "shellscript"
+    """Covers sh, bash, and similar shell scripts."""
+    xonsh = "xonsh"
     json = "json"
     csv = "csv"
     npz = "npz"
@@ -56,6 +64,7 @@ class Format(Enum):
     docx = "docx"
     jpeg = "jpeg"
     png = "png"
+    gif = "gif"
     svg = "svg"
     mp3 = "mp3"
     m4a = "m4a"
@@ -85,7 +94,8 @@ class Format(Enum):
             self.diff,
             self.python,
             self.json,
-            self.kash_script,
+            self.shellscript,
+            self.xonsh,
             self.csv,
             self.log,
         ]
@@ -102,7 +112,7 @@ class Format(Enum):
     @property
     def is_image(self) -> bool:
-        return self in [self.jpeg, self.png, self.svg]
+        return self in [self.jpeg, self.png, self.gif, self.svg]
     @property
     def is_audio(self) -> bool:
@@ -114,7 +124,7 @@ class Format(Enum):
     @property
     def is_code(self) -> bool:
-        return self in [self.python, self.kash_script, self.json, self.yaml]
+        return self in [self.python, self.shellscript, self.xonsh, self.json, self.yaml]
     @property
     def is_data(self) -> bool:
@@ -138,10 +148,12 @@ class Format(Enum):
             self.markdown,
             self.md_html,
             self.html,
+            self.json,  # Not strictly true but we encourage use of comments.
             self.yaml,
             self.diff,
             self.python,
-            self.kash_script,
+            self.shellscript,
+            self.xonsh,
             self.csv,
             self.log,
         ]
@@ -157,13 +169,15 @@ class Format(Enum):
             Format.yaml: MediaType.text,
             Format.diff: MediaType.text,
             Format.python: MediaType.text,
-            Format.kash_script: MediaType.text,
+            Format.shellscript: MediaType.text,
+            Format.xonsh: MediaType.text,
             Format.json: MediaType.text,
             Format.csv: MediaType.text,
             Format.log: MediaType.text,
             Format.pdf: MediaType.text,
             Format.jpeg: MediaType.image,
             Format.png: MediaType.image,
+            Format.gif: MediaType.image,
             Format.svg: MediaType.image,
             Format.docx: MediaType.text,
             Format.mp3: MediaType.audio,
@@ -189,11 +203,13 @@ class Format(Enum):
             FileExt.npz.value: Format.npz,
             FileExt.log.value: Format.log,
             FileExt.py.value: Format.python,
-            FileExt.ksh.value: Format.kash_script,
+            FileExt.sh.value: Format.shellscript,
+            FileExt.xsh.value: Format.xonsh,
             FileExt.pdf.value: Format.pdf,
             FileExt.docx.value: Format.docx,
             FileExt.jpg.value: Format.jpeg,
             FileExt.png.value: Format.png,
+            FileExt.gif.value: Format.gif,
             FileExt.svg.value: Format.svg,
             FileExt.mp3.value: Format.mp3,
             FileExt.m4a.value: Format.m4a,
@@ -219,10 +235,13 @@ class Format(Enum):
             Format.npz: FileExt.npz,
             Format.log: FileExt.log,
             Format.python: FileExt.py,
+            Format.shellscript: FileExt.sh,
+            Format.xonsh: FileExt.xsh,
             Format.pdf: FileExt.pdf,
             Format.docx: FileExt.docx,
             Format.jpeg: FileExt.jpg,
             Format.png: FileExt.png,
+            Format.gif: FileExt.gif,
             Format.svg: FileExt.svg,
             Format.mp3: FileExt.mp3,
             Format.m4a: FileExt.m4a,
@@ -244,6 +263,10 @@ class Format(Enum):
             "application/yaml": Format.yaml,
             "application/x-yaml": Format.yaml,
             "text/x-python": Format.python,
+            "text/x-script.python": Format.python,
+            "text/x-sh": Format.shellscript,
+            "text/x-shellscript": Format.shellscript,
+            "text/x-xonsh": Format.xonsh,
             "application/json": Format.json,
             "text/csv": Format.csv,
             "application/x-npz": Format.npz,
@@ -251,6 +274,7 @@ class Format(Enum):
             "application/vnd.openxmlformats-officedocument.wordprocessingml.document": Format.docx,
             "image/jpeg": Format.jpeg,
             "image/png": Format.png,
+            "image/gif": Format.gif,
             "image/svg+xml": Format.svg,
             "audio/mpeg": Format.mp3,
             "audio/mp3": Format.mp3,
@@ -326,15 +350,15 @@ class FileFormatInfo:
             and self.mime_type.startswith("image")
         )
-    def as_str(self) -> str:
-        if self.format and self.mime_type:
-            return f"{self.format.value} ({self.mime_type})"
-        elif self.format:
+    def as_str(self, mime_only: bool = False) -> str:
+        if self.format and not mime_only:
             return self.format.value
+        elif self.mime_type == MIME_EMPTY:
+            return "empty"
         elif self.mime_type:
             return self.mime_type
         else:
-            return "unrecognized"
+            return "unrecognized format"
     def __str__(self) -> str:
         return self.as_str()
@@ -357,15 +381,22 @@ def guess_format_by_name(path: str | Path) -> Format | None:
     return Format.guess_by_file_ext(file_ext) if file_ext else None
-def file_format_info(path: str | Path) -> FileFormatInfo:
+def file_format_info(path: str | Path, always_check_content: bool = False) -> FileFormatInfo:
     """
-    Full info on the file format path and content (file extension and file content).
+    Get info on the file format path and content (file extension and file content).
+    Looks at the file extension first and then the file content if needed.
+    If `always_check_content` is True, look at the file content even if we
+    recognize the file extension.
     """
     path = Path(path)
     file_ext = parse_file_ext(path)
-    mime_type = detect_mime_type(path)
-    format = _guess_format(file_ext, mime_type)
-    final_mime_type = format.mime_type if format else mime_type
+    if always_check_content or not file_ext:
+        # Look at the file content.
+        detected_mime_type = detect_mime_type(path)
+    else:
+        detected_mime_type = None
+    format = _guess_format(file_ext, detected_mime_type)
+    final_mime_type = format.mime_type if format else detected_mime_type
     return FileFormatInfo(file_ext, format, final_mime_type)

kash/utils/file_utils/filename_parsing.py CHANGED Viewed

@@ -2,7 +2,7 @@ import os
 from pathlib import Path
 from kash.config.logger import get_logger
-from kash.utils.common.url import Url
+from kash.utils.common.url import Url, check_if_url
 from kash.utils.errors import InvalidFilename
 from kash.utils.file_utils.file_ext import FileExt, canonicalize_file_ext
@@ -48,11 +48,17 @@ def join_filename(dirname: str | Path, name: str, item_type: str | None, ext: st
 def parse_file_ext(url_or_path: str | Url | Path) -> FileExt | None:
     """
-    Parse a known, canonical file extension from a path, a URL, or even just a
-    raw file extension (like "csv" or ".csv").
+    Parse a known, canonical file extension from a path or URL. Also accepts
+    raw file extensions (like "csv" or ".csv").
     """
-    front, ext = os.path.splitext(str(url_or_path).split("/")[-1])
+    parsed_url = check_if_url(url_or_path)
+    if parsed_url:
+        path = parsed_url.path
+    else:
+        path = str(url_or_path)
+    front, ext = os.path.splitext(path.split("/")[-1])
     if not ext:
+        # Handle bare file extensions too.
         ext = front
     return FileExt.parse(canonicalize_file_ext(ext))

kash/web_content/dir_store.py CHANGED Viewed

@@ -87,8 +87,7 @@ class DirStore:
         self, keys: list[str | Path], folder: str | None = None, suffix: str | None = None
     ) -> dict[str | Path, Path | None]:
         """
-        Look up all existing cached results for the set of keys. This should work fine but could
-        be optimized for large batches.
+        Look up all existing cached results for the set of keys.
         """
         return {key: self.find(key, folder=folder, suffix=suffix) for key in keys}

kash/web_content/file_cache_utils.py CHANGED Viewed

@@ -1,4 +1,7 @@
+import json
+from collections.abc import Callable
 from pathlib import Path
+from typing import Any
 from prettyfmt import fmt_lines, fmt_path
@@ -35,18 +38,40 @@ def reset_content_cache_dir(path: Path):
             log.info("Using web cache: %s", fmt_path(path))
-def cache_file(source: Url | Path | Loadable, global_cache: bool = False) -> tuple[Path, bool]:
+def cache_file(
+    source: Url | Path | Loadable, global_cache: bool = False, expiration_sec: float | None = None
+) -> tuple[Path, bool]:
     """
     Return a local cached copy of the item. If it is an URL, content is fetched.
-    Raises requests.HTTPError if the URL is not reachable. If it is a Path or
-    a Loadable, a cached copy is returned.
+    If it is a Path or a Loadable, a cached copy is returned.
+    LocalFileCache uses httpx so httpx.HTTPError is raised for non-2xx responses.
+    Uses the current content cache unless there is no current cache or `global_cache` is True,
+    in which case the global cache is used.
     """
     cache = _global_content_cache if global_cache else _content_cache
-    path, was_cached = cache.cache(source)
+    path, was_cached = cache.cache(source, expiration_sec)
     return path, was_cached
-def cache_resource(item: Item) -> dict[MediaType, Path]:
+def cache_api_response(
+    url: Url,
+    global_cache: bool = False,
+    expiration_sec: float | None = None,
+    parser: Callable[[str], Any] = json.loads,
+) -> tuple[Any, bool]:
+    """
+    Cache an API response. By default parse the response as JSON.
+    """
+    cache = _global_content_cache if global_cache else _content_cache
+    path, was_cached = cache.cache(url, expiration_sec)
+    result = parser(path.read_text())
+    return result, was_cached
+def cache_resource(
+    item: Item, global_cache: bool = False, expiration_sec: float | None = None
+) -> dict[MediaType, Path]:
     """
     Cache a resource item for an external local path or a URL, fetching or
     copying as needed. For media this may yield more than one format.
@@ -64,17 +89,17 @@ def cache_resource(item: Item) -> dict[MediaType, Path]:
         if is_media_url(item.url):
             result = cache_media(item.url)
         else:
-            path, _was_cached = cache_file(item.url)
+            path, _was_cached = cache_file(item.url, global_cache, expiration_sec)
     elif item.external_path:
         path = Path(item.external_path)
         if not path.is_file():
             raise FileNotFound(f"External path not found: {path}")
-        path, _was_cached = cache_file(path)
+        path, _was_cached = cache_file(path, global_cache, expiration_sec)
     elif item.original_filename:
         path = Path(item.original_filename)
         if not path.is_file():
             raise FileNotFound(f"Original filename not found: {path}")
-        path, _was_cached = cache_file(path)
+        path, _was_cached = cache_file(path, global_cache, expiration_sec)
     else:
         raise ValueError(f"Item has no URL or external path: {item}")
@@ -94,7 +119,9 @@ def cache_resource(item: Item) -> dict[MediaType, Path]:
     return result
-def get_url_html(item: Item) -> tuple[Url, str]:
+def get_url_html(
+    item: Item, global_cache: bool = False, expiration_sec: float | None = None
+) -> tuple[Url, str]:
     """
     Returns the HTML content of an URL item, using the content cache,
     or the body of the item if it has a URL and HTML body.
@@ -106,7 +133,7 @@ def get_url_html(item: Item) -> tuple[Url, str]:
     url = Url(canonicalize_url(item.url))
     if is_url_item(item):
-        path, _was_cached = cache_file(url)
+        path, _was_cached = cache_file(url, global_cache, expiration_sec)
         with open(path) as file:
             html_content = file.read()
     else:

kash/web_content/file_processing.py ADDED Viewed

@@ -0,0 +1,68 @@
+from __future__ import annotations
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TypeAlias
+from kash.web_content.local_file_cache import read_mtime
+@dataclass(frozen=True)
+class OutputType:
+    """
+    A type of output file, represented by the filename suffix, e.g. '.mp3', '.txt', etc.
+    """
+    suffix: str
+    def output_path(self, src: Path) -> Path:
+        """
+        Resolve the output path. Will be next to the source file, e.g.
+        some-dir/video.mp4 -> some-dir/video.mp3
+        """
+        return src.with_suffix(self.suffix)
+Processor: TypeAlias = Callable[[Path, Mapping[OutputType, Path]], None]
+"""
+A function that takes a source file and a mapping with one or more output paths.
+"""
+@dataclass(frozen=True)
+class FileProcess:
+    """
+    Process a file and produce one or more outputs.
+    """
+    processor: Processor
+    outputs: list[OutputType]
+    def is_outdated(self, src: Path) -> bool:
+        """
+        True when any output is missing or older (earliest mtime) than `src`.
+        """
+        dests = {o.output_path(src) for o in self.outputs}
+        if any(not p.exists() for p in dests):
+            return True
+        earliest = min(read_mtime(p) for p in dests)
+        return read_mtime(src) > earliest
+    def run(self, src: Path) -> dict[OutputType, Path]:
+        """
+        Run unconditionally and return a mapping of outputs to paths.
+        """
+        dests = {o: o.output_path(src) for o in self.outputs}
+        self.processor(src, dests)
+        return dests
+    def run_if_needed(self, src: Path) -> dict[OutputType, Path]:
+        """
+        Run only if any output is missing or outdated.
+        """
+        return (
+            self.run(src)
+            if self.is_outdated(src)
+            else {o: o.output_path(src) for o in self.outputs}
+        )

kash/web_content/local_file_cache.py CHANGED Viewed

@@ -11,7 +11,7 @@ from prettyfmt import fmt_path
 from strif import atomic_output_file, copyfile_atomic
 from kash.utils.common.url import Url, is_file_url, is_url, normalize_url, parse_file_url
-from kash.utils.errors import FileNotFound, InvalidInput
+from kash.utils.errors import FileNotFound
 from kash.utils.file_utils.file_formats_model import choose_file_ext
 from kash.web_content.dir_store import DirStore
 from kash.web_content.web_fetch import download_url
@@ -56,19 +56,21 @@ class Loadable:
     key: str
     """
-    The unique identifier for the item. If it ends in a recognized file extension,
-    both the key and the extension will be used when creating unique cache filenames.
+    The unique identifier for the item. Used when creating unique cache filenames,
+    as is or with added suffixes.
     """
     save: Callable[[Path], None]
     """
     Method that saves the item to the given path. Caller will handle path selection
-    and atomicity of file creation.
+    and atomicity of file creation. Raise an exception if the item cannot be saved.
     """
 Cacheable = Url | Path | Loadable
-"""An item that can be cached as a file."""
+"""
+An item that can be cached as a file.
+"""
 def _suffix_for(cacheable: Cacheable) -> str | None:
@@ -151,9 +153,7 @@ class LocalFileCache(DirStore):
             if isinstance(url_or_path, Path):
                 file_path = url_or_path
             else:
-                parsed = parse_file_url(url_or_path)
-                if not parsed:
-                    raise InvalidInput(f"Not a file URL: {url_or_path}")
+                parsed = parse_file_url(url_or_path)  # Raises ValueError if not a file URL.
                 file_path = parsed
             if not file_path.exists():
                 raise FileNotFound(f"File not found: {file_path}")
@@ -173,7 +173,10 @@ class LocalFileCache(DirStore):
             ) as tmp_path:
                 source.save(tmp_path)
             if not cache_path.exists():
-                raise InvalidCacheState(f"Failed to save to cache: {source}: {cache_path}")
+                # The source should have raised an exception if it failed to save.
+                raise InvalidCacheState(
+                    f"Loadable source failed to save to cache: {source}: {cache_path}"
+                )
         else:
             raise ValueError(f"Invalid source: {source}")

kash/web_content/web_extract.py CHANGED Viewed

@@ -10,14 +10,19 @@ from kash.web_content.web_page_model import PageExtractor, WebPageData
 @log_calls(level="message")
 def fetch_extract(
-    url: Url, use_cache: bool = True, extractor: PageExtractor = extract_text_justext
+    url: Url,
+    refetch: bool = False,
+    use_cache: bool = True,
+    extractor: PageExtractor = extract_text_justext,
 ) -> WebPageData:
     """
     Fetches a URL and extracts the title, description, and content.
+    By default, uses the content cache if available. Can force re-fetching and
+    updating the cache by setting `refetch` to true.
     """
+    expiration_sec = 0 if refetch else None
     if use_cache:
-        path, _was_cached = cache_file(url)
+        path, _was_cached = cache_file(url, expiration_sec=expiration_sec)
         with open(path, "rb") as file:
             content = file.read()
         page_data = extractor(url, content)

kash/web_content/web_fetch.py CHANGED Viewed

@@ -7,17 +7,22 @@ import httpx
 from strif import atomic_output_file, copyfile_atomic
 from tqdm import tqdm
+from kash.config.env_settings import KashEnv
 from kash.utils.common.url import Url
 log = logging.getLogger(__name__)
-USER_AGENT = "Mozilla/5.0 (Compatible)"
 DEFAULT_TIMEOUT = 30
+DEFAULT_USER_AGENT = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:126.0) Gecko/20100101 Firefox/126.0"
+)
 def default_headers() -> dict[str, str]:
-    return {"User-Agent": USER_AGENT}
+    return {"User-Agent": KashEnv.KASH_USER_AGENT.read_str(default=DEFAULT_USER_AGENT)}
 def fetch_url(
@@ -36,6 +41,7 @@ def fetch_url(
         auth=auth,
         headers=headers or default_headers(),
     ) as client:
+        log.debug("fetch_url: using headers: %s", client.headers)
         response = client.get(url)
         log.info("Fetched: %s (%s bytes): %s", response.status_code, len(response.content), url)
         response.raise_for_status()
@@ -52,7 +58,7 @@ def download_url(
     headers: dict[str, str] | None = None,
 ) -> None:
     """
-    Download given file, optionally with progress bar.
+    Download given file, optionally with progress bar, streaming to a target file.
     Also handles file:// and s3:// URLs. Output file is created atomically.
     Raise httpx.HTTPError for non-2xx responses.
     """
@@ -73,13 +79,15 @@ def download_url(
         client = session or httpx.Client(follow_redirects=True, timeout=timeout)
         response: httpx.Response | None = None
         try:
+            headers = headers or default_headers()
+            log.debug("download_url: using headers: %s", headers)
             with client.stream(
                 "GET",
                 url,
                 follow_redirects=True,
                 timeout=timeout,
                 auth=auth,
-                headers=headers or default_headers(),
+                headers=headers,
             ) as response:
                 response.raise_for_status()
                 total_size = int(response.headers.get("content-length", "0"))

kash/web_gen/tabbed_webpage.py CHANGED Viewed

@@ -2,6 +2,7 @@ import os
 from dataclasses import asdict, dataclass
 from frontmatter_format import read_yaml_file, to_yaml_string, write_yaml_file
+from prettyfmt import sanitize_title
 from kash.config.logger import get_logger
 from kash.exec.preconditions import has_thumbnail_url
@@ -41,7 +42,7 @@ def _fill_in_ids(tabs: list[TabInfo]):
             tab.id = f"tab_{i}"
-def webpage_config(items: list[Item]) -> Item:
+def webpage_config(items: list[Item], clean_headings: bool = False) -> Item:
     """
     Get an item with the config for a tabbed web page.
     """
@@ -57,9 +58,11 @@ def webpage_config(items: list[Item]) -> Item:
             log.warning("Item has no thumbnail URL: %s", item)
             return None
+    clean = clean_heading if clean_headings else sanitize_title
     tabs = [
         TabInfo(
-            label=clean_heading(item.abbrev_title()),
+            label=clean(item.abbrev_title()),
             store_path=item.store_path,
             thumbnail_url=get_thumbnail_url(item),
         )

kash-shell 0.3.8__py3-none-any.whl → 0.3.10__py3-none-any.whl

kash-shell 0.3.8py3-none-any.whl → 0.3.10py3-none-any.whl