kash-shell 0.3.9__py3-none-any.whl → 0.3.11__py3-none-any.whl

kash/utils/file_utils/file_formats.py

@@ -4,9 +4,9 @@ from pathlib import Path
 from typing import NewType
 
 import regex
+from clideps.pkgs.pkg_check import pkg_check
 
 from kash.config.logger import get_logger
-from kash.shell.clideps.pkg_deps import Pkg, pkg_check
 
 log = get_logger(__name__)
 
@@ -86,7 +86,7 @@ def detect_mime_type(filename: str | Path) -> MimeType | None:
     Get the mime type of a file using libmagic heuristics plus more careful
     detection of HTML, Markdown, and multipart YAML.
     """
-    pkg_check().require(Pkg.libmagic)
+    pkg_check().require("libmagic")
     import magic
 
     mime = magic.Magic(mime=True)

kash/utils/file_utils/file_formats_model.py

@@ -36,6 +36,8 @@ class Format(Enum):
     it is the format of the resource (url, media, etc.).
     """
 
+    # TODO: Be more thorough, pulling in relevant extensions and types from the `mimetypes` module.
+
     # Formats with no body (content is in frontmatter).
     url = "url"
 
@@ -46,6 +48,7 @@ class Format(Enum):
     """`md_html` is Markdown with HTML, used for example when we structure Markdown with divs."""
     html = "html"
     """`markdown` should be simple and clean Markdown that we can use with LLMs."""
+    epub = "epub"
     yaml = "yaml"
     diff = "diff"
     python = "python"
@@ -54,12 +57,14 @@ class Format(Enum):
     xonsh = "xonsh"
     json = "json"
     csv = "csv"
+    xlsx = "xlsx"
     npz = "npz"
     log = "log"
 
     # Media formats.
     pdf = "pdf"
     docx = "docx"
+    pptx = "pptx"
     jpeg = "jpeg"
     png = "png"
     gif = "gif"
@@ -106,6 +111,7 @@ class Format(Enum):
             self.html,
             self.pdf,
             self.docx,
+            self.pptx,
         ]
 
     @property
@@ -126,7 +132,7 @@ class Format(Enum):
 
     @property
     def is_data(self) -> bool:
-        return self in [self.csv, self.npz]
+        return self in [self.csv, self.xlsx, self.npz]
 
     @property
     def is_binary(self) -> bool:
@@ -146,6 +152,7 @@ class Format(Enum):
             self.markdown,
             self.md_html,
             self.html,
+            self.json,  # Not strictly true but we encourage use of comments.
             self.yaml,
             self.diff,
             self.python,
@@ -163,6 +170,7 @@ class Format(Enum):
             Format.markdown: MediaType.text,
             Format.md_html: MediaType.text,
             Format.html: MediaType.webpage,
+            Format.epub: MediaType.text,
             Format.yaml: MediaType.text,
             Format.diff: MediaType.text,
             Format.python: MediaType.text,
@@ -172,11 +180,13 @@ class Format(Enum):
             Format.csv: MediaType.text,
             Format.log: MediaType.text,
             Format.pdf: MediaType.text,
+            Format.xlsx: MediaType.text,
             Format.jpeg: MediaType.image,
             Format.png: MediaType.image,
             Format.gif: MediaType.image,
             Format.svg: MediaType.image,
             Format.docx: MediaType.text,
+            Format.pptx: MediaType.text,
             Format.mp3: MediaType.audio,
             Format.m4a: MediaType.audio,
             Format.mp4: MediaType.video,
@@ -197,6 +207,7 @@ class Format(Enum):
             FileExt.diff.value: Format.diff,
             FileExt.json.value: Format.json,
             FileExt.csv.value: Format.csv,
+            FileExt.xlsx.value: Format.xlsx,
             FileExt.npz.value: Format.npz,
             FileExt.log.value: Format.log,
             FileExt.py.value: Format.python,
@@ -204,6 +215,7 @@ class Format(Enum):
             FileExt.xsh.value: Format.xonsh,
             FileExt.pdf.value: Format.pdf,
             FileExt.docx.value: Format.docx,
+            FileExt.pptx.value: Format.pptx,
             FileExt.jpg.value: Format.jpeg,
             FileExt.png.value: Format.png,
             FileExt.gif.value: Format.gif,
@@ -211,6 +223,7 @@ class Format(Enum):
             FileExt.mp3.value: Format.mp3,
             FileExt.m4a.value: Format.m4a,
             FileExt.mp4.value: Format.mp4,
+            FileExt.epub.value: Format.epub,
         }
         return ext_to_format.get(file_ext.value, None)
 
@@ -225,10 +238,12 @@ class Format(Enum):
             Format.md_html: FileExt.md,
             Format.html: FileExt.html,
             Format.plaintext: FileExt.txt,
+            Format.epub: FileExt.epub,
             Format.yaml: FileExt.yml,
             Format.diff: FileExt.diff,
             Format.json: FileExt.json,
             Format.csv: FileExt.csv,
+            Format.xlsx: FileExt.xlsx,
             Format.npz: FileExt.npz,
             Format.log: FileExt.log,
             Format.python: FileExt.py,
@@ -236,6 +251,7 @@ class Format(Enum):
             Format.xonsh: FileExt.xsh,
             Format.pdf: FileExt.pdf,
             Format.docx: FileExt.docx,
+            Format.pptx: FileExt.pptx,
             Format.jpeg: FileExt.jpg,
             Format.png: FileExt.png,
             Format.gif: FileExt.gif,
@@ -257,6 +273,7 @@ class Format(Enum):
             "text/html": Format.html,
             "text/diff": Format.diff,
             "text/x-diff": Format.diff,
+            "application/epub+zip": Format.epub,
             "application/yaml": Format.yaml,
             "application/x-yaml": Format.yaml,
             "text/x-python": Format.python,
@@ -266,9 +283,11 @@ class Format(Enum):
             "text/x-xonsh": Format.xonsh,
             "application/json": Format.json,
             "text/csv": Format.csv,
+            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": Format.xlsx,
             "application/x-npz": Format.npz,
             "application/pdf": Format.pdf,
             "application/vnd.openxmlformats-officedocument.wordprocessingml.document": Format.docx,
+            "application/vnd.openxmlformats-officedocument.presentationml.presentation": Format.pptx,
             "image/jpeg": Format.jpeg,
             "image/png": Format.png,
             "image/gif": Format.gif,

kash/web_content/dir_store.py

@@ -87,8 +87,7 @@ class DirStore:
         self, keys: list[str | Path], folder: str | None = None, suffix: str | None = None
     ) -> dict[str | Path, Path | None]:
         """
-        Look up all existing cached results for the set of keys. This should work fine but could
-        be optimized for large batches.
+        Look up all existing cached results for the set of keys.
         """
         return {key: self.find(key, folder=folder, suffix=suffix) for key in keys}
 

kash/web_content/file_cache_utils.py

@@ -1,4 +1,7 @@
+import json
+from collections.abc import Callable
 from pathlib import Path
+from typing import Any
 
 from prettyfmt import fmt_lines, fmt_path
 
@@ -35,18 +38,40 @@ def reset_content_cache_dir(path: Path):
             log.info("Using web cache: %s", fmt_path(path))
 
 
-def cache_file(source: Url | Path | Loadable, global_cache: bool = False) -> tuple[Path, bool]:
+def cache_file(
+    source: Url | Path | Loadable, global_cache: bool = False, expiration_sec: float | None = None
+) -> tuple[Path, bool]:
     """
     Return a local cached copy of the item. If it is an URL, content is fetched.
-    Raises requests.HTTPError if the URL is not reachable. If it is a Path or
-    a Loadable, a cached copy is returned.
+    If it is a Path or a Loadable, a cached copy is returned.
+    LocalFileCache uses httpx so httpx.HTTPError is raised for non-2xx responses.
+
+    Uses the current content cache unless there is no current cache or `global_cache` is True,
+    in which case the global cache is used.
     """
     cache = _global_content_cache if global_cache else _content_cache
-    path, was_cached = cache.cache(source)
+    path, was_cached = cache.cache(source, expiration_sec)
     return path, was_cached
 
 
-def cache_resource(item: Item) -> dict[MediaType, Path]:
+def cache_api_response(
+    url: Url,
+    global_cache: bool = False,
+    expiration_sec: float | None = None,
+    parser: Callable[[str], Any] = json.loads,
+) -> tuple[Any, bool]:
+    """
+    Cache an API response. By default parse the response as JSON.
+    """
+    cache = _global_content_cache if global_cache else _content_cache
+    path, was_cached = cache.cache(url, expiration_sec)
+    result = parser(path.read_text())
+    return result, was_cached
+
+
+def cache_resource(
+    item: Item, global_cache: bool = False, expiration_sec: float | None = None
+) -> dict[MediaType, Path]:
     """
     Cache a resource item for an external local path or a URL, fetching or
     copying as needed. For media this may yield more than one format.
@@ -64,17 +89,17 @@ def cache_resource(item: Item) -> dict[MediaType, Path]:
         if is_media_url(item.url):
             result = cache_media(item.url)
         else:
-            path, _was_cached = cache_file(item.url)
+            path, _was_cached = cache_file(item.url, global_cache, expiration_sec)
     elif item.external_path:
         path = Path(item.external_path)
         if not path.is_file():
             raise FileNotFound(f"External path not found: {path}")
-        path, _was_cached = cache_file(path)
+        path, _was_cached = cache_file(path, global_cache, expiration_sec)
     elif item.original_filename:
         path = Path(item.original_filename)
         if not path.is_file():
             raise FileNotFound(f"Original filename not found: {path}")
-        path, _was_cached = cache_file(path)
+        path, _was_cached = cache_file(path, global_cache, expiration_sec)
     else:
         raise ValueError(f"Item has no URL or external path: {item}")
 
@@ -94,7 +119,9 @@ def cache_resource(item: Item) -> dict[MediaType, Path]:
     return result
 
 
-def get_url_html(item: Item) -> tuple[Url, str]:
+def get_url_html(
+    item: Item, global_cache: bool = False, expiration_sec: float | None = None
+) -> tuple[Url, str]:
     """
     Returns the HTML content of an URL item, using the content cache,
     or the body of the item if it has a URL and HTML body.
@@ -106,7 +133,7 @@ def get_url_html(item: Item) -> tuple[Url, str]:
     url = Url(canonicalize_url(item.url))
 
     if is_url_item(item):
-        path, _was_cached = cache_file(url)
+        path, _was_cached = cache_file(url, global_cache, expiration_sec)
         with open(path) as file:
             html_content = file.read()
     else:

kash/web_content/file_processing.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TypeAlias
+
+from kash.web_content.local_file_cache import read_mtime
+
+
+@dataclass(frozen=True)
+class OutputType:
+    """
+    A type of output file, represented by the filename suffix, e.g. '.mp3', '.txt', etc.
+    """
+
+    suffix: str
+
+    def output_path(self, src: Path) -> Path:
+        """
+        Resolve the output path. Will be next to the source file, e.g.
+        some-dir/video.mp4 -> some-dir/video.mp3
+        """
+        return src.with_suffix(self.suffix)
+
+
+Processor: TypeAlias = Callable[[Path, Mapping[OutputType, Path]], None]
+"""
+A function that takes a source file and a mapping with one or more output paths.
+"""
+
+
+@dataclass(frozen=True)
+class FileProcess:
+    """
+    Process a file and produce one or more outputs.
+    """
+
+    processor: Processor
+    outputs: list[OutputType]
+
+    def is_outdated(self, src: Path) -> bool:
+        """
+        True when any output is missing or older (earliest mtime) than `src`.
+        """
+        dests = {o.output_path(src) for o in self.outputs}
+        if any(not p.exists() for p in dests):
+            return True
+        earliest = min(read_mtime(p) for p in dests)
+        return read_mtime(src) > earliest
+
+    def run(self, src: Path) -> dict[OutputType, Path]:
+        """
+        Run unconditionally and return a mapping of outputs to paths.
+        """
+        dests = {o: o.output_path(src) for o in self.outputs}
+        self.processor(src, dests)
+        return dests
+
+    def run_if_needed(self, src: Path) -> dict[OutputType, Path]:
+        """
+        Run only if any output is missing or outdated.
+        """
+        return (
+            self.run(src)
+            if self.is_outdated(src)
+            else {o: o.output_path(src) for o in self.outputs}
+        )

kash/web_content/local_file_cache.py

@@ -11,7 +11,7 @@ from prettyfmt import fmt_path
 from strif import atomic_output_file, copyfile_atomic
 
 from kash.utils.common.url import Url, is_file_url, is_url, normalize_url, parse_file_url
-from kash.utils.errors import FileNotFound, InvalidInput
+from kash.utils.errors import FileNotFound
 from kash.utils.file_utils.file_formats_model import choose_file_ext
 from kash.web_content.dir_store import DirStore
 from kash.web_content.web_fetch import download_url
@@ -56,19 +56,21 @@ class Loadable:
 
     key: str
     """
-    The unique identifier for the item. If it ends in a recognized file extension,
-    both the key and the extension will be used when creating unique cache filenames.
+    The unique identifier for the item. Used when creating unique cache filenames,
+    as is or with added suffixes.
     """
 
     save: Callable[[Path], None]
     """
     Method that saves the item to the given path. Caller will handle path selection
-    and atomicity of file creation.
+    and atomicity of file creation. Raise an exception if the item cannot be saved.
     """
 
 
 Cacheable = Url | Path | Loadable
-"""An item that can be cached as a file."""
+"""
+An item that can be cached as a file.
+"""
 
 
 def _suffix_for(cacheable: Cacheable) -> str | None:
@@ -151,9 +153,7 @@ class LocalFileCache(DirStore):
             if isinstance(url_or_path, Path):
                 file_path = url_or_path
             else:
-                parsed = parse_file_url(url_or_path)
-                if not parsed:
-                    raise InvalidInput(f"Not a file URL: {url_or_path}")
+                parsed = parse_file_url(url_or_path)  # Raises ValueError if not a file URL.
                 file_path = parsed
             if not file_path.exists():
                 raise FileNotFound(f"File not found: {file_path}")
@@ -173,7 +173,10 @@ class LocalFileCache(DirStore):
             ) as tmp_path:
                 source.save(tmp_path)
             if not cache_path.exists():
-                raise InvalidCacheState(f"Failed to save to cache: {source}: {cache_path}")
+                # The source should have raised an exception if it failed to save.
+                raise InvalidCacheState(
+                    f"Loadable source failed to save to cache: {source}: {cache_path}"
+                )
         else:
             raise ValueError(f"Invalid source: {source}")
 

kash/web_content/web_extract.py

@@ -10,14 +10,19 @@ from kash.web_content.web_page_model import PageExtractor, WebPageData
 
 @log_calls(level="message")
 def fetch_extract(
-    url: Url, use_cache: bool = True, extractor: PageExtractor = extract_text_justext
+    url: Url,
+    refetch: bool = False,
+    use_cache: bool = True,
+    extractor: PageExtractor = extract_text_justext,
 ) -> WebPageData:
     """
     Fetches a URL and extracts the title, description, and content.
+    By default, uses the content cache if available. Can force re-fetching and
+    updating the cache by setting `refetch` to true.
     """
-
+    expiration_sec = 0 if refetch else None
     if use_cache:
-        path, _was_cached = cache_file(url)
+        path, _was_cached = cache_file(url, expiration_sec=expiration_sec)
         with open(path, "rb") as file:
             content = file.read()
         page_data = extractor(url, content)

kash/web_content/web_fetch.py

@@ -7,17 +7,22 @@ import httpx
 from strif import atomic_output_file, copyfile_atomic
 from tqdm import tqdm
 
+from kash.config.env_settings import KashEnv
 from kash.utils.common.url import Url
 
 log = logging.getLogger(__name__)
 
-USER_AGENT = "Mozilla/5.0 (Compatible)"
 
 DEFAULT_TIMEOUT = 30
 
 
+DEFAULT_USER_AGENT = (
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:126.0) Gecko/20100101 Firefox/126.0"
+)
+
+
 def default_headers() -> dict[str, str]:
-    return {"User-Agent": USER_AGENT}
+    return {"User-Agent": KashEnv.KASH_USER_AGENT.read_str(default=DEFAULT_USER_AGENT)}
 
 
 def fetch_url(
@@ -36,6 +41,7 @@ def fetch_url(
         auth=auth,
         headers=headers or default_headers(),
     ) as client:
+        log.debug("fetch_url: using headers: %s", client.headers)
         response = client.get(url)
         log.info("Fetched: %s (%s bytes): %s", response.status_code, len(response.content), url)
         response.raise_for_status()
@@ -52,7 +58,7 @@ def download_url(
     headers: dict[str, str] | None = None,
 ) -> None:
     """
-    Download given file, optionally with progress bar.
+    Download given file, optionally with progress bar, streaming to a target file.
     Also handles file:// and s3:// URLs. Output file is created atomically.
     Raise httpx.HTTPError for non-2xx responses.
     """
@@ -73,13 +79,15 @@ def download_url(
         client = session or httpx.Client(follow_redirects=True, timeout=timeout)
         response: httpx.Response | None = None
         try:
+            headers = headers or default_headers()
+            log.debug("download_url: using headers: %s", headers)
             with client.stream(
                 "GET",
                 url,
                 follow_redirects=True,
                 timeout=timeout,
                 auth=auth,
-                headers=headers or default_headers(),
+                headers=headers,
             ) as response:
                 response.raise_for_status()
                 total_size = int(response.headers.get("content-length", "0"))

kash/web_gen/__init__.py

@@ -1,4 +0,0 @@
-from pathlib import Path
-
-base_templates_dir = Path(__file__).parent / "templates"
-"""Common base web page templates."""

kash/web_gen/simple_webpage.py

@@ -0,0 +1,52 @@
+from kash.model.items_model import Item
+from kash.utils.file_utils.file_formats_model import Format
+from kash.web_gen.template_render import render_web_template
+
+
+def simple_webpage_render(
+    item: Item,
+    page_template: str = "simple_webpage.html.jinja",
+    add_title_h1: bool = True,
+) -> str:
+    """
+    Generate a simple web page from a single item.
+    If `add_title_h1` is True, the title will be inserted as an h1 heading above the body.
+    """
+    return render_web_template(
+        template_filename=page_template,
+        data={
+            "title": item.title,
+            "add_title_h1": add_title_h1,
+            "content_html": item.body_as_html(),
+            "thumbnail_url": item.thumbnail_url,
+        },
+    )
+
+
+## Tests
+
+
+def test_render():
+    import os
+
+    from kash.model.items_model import ItemType
+
+    # Create a test item
+    item = Item(
+        type=ItemType.doc,
+        format=Format.html,
+        title="A Simple Web Page",
+        body="<p>This is a simple web page with <b>HTML content</b>.</p>",
+    )
+
+    # Generate HTML
+    html = simple_webpage_render(item)
+
+    os.makedirs("tmp", exist_ok=True)
+    with open("tmp/simple_webpage.html", "w") as f:
+        f.write(html)
+    print("Rendered simple webpage to tmp/simple_webpage.html")
+
+    # Basic validation
+    assert item.title and item.title in html
+    assert "<b>HTML content</b>" in html

kash/web_gen/tabbed_webpage.py

@@ -2,6 +2,7 @@ import os
 from dataclasses import asdict, dataclass
 
 from frontmatter_format import read_yaml_file, to_yaml_string, write_yaml_file
+from prettyfmt import abbrev_on_words, sanitize_title
 
 from kash.config.logger import get_logger
 from kash.exec.preconditions import has_thumbnail_url
@@ -11,7 +12,6 @@ from kash.model.paths_model import StorePath
 from kash.utils.common.type_utils import as_dataclass, not_none
 from kash.utils.errors import NoMatch
 from kash.utils.file_utils.file_formats_model import Format
-from kash.web_gen import base_templates_dir
 from kash.web_gen.template_render import render_web_template
 from kash.workspaces import current_ws
 from kash.workspaces.source_items import find_upstream_item
@@ -33,6 +33,7 @@ class TabbedWebpage:
     title: str
     tabs: list[TabInfo]
     show_tabs: bool = True
+    add_title_h1: bool = True
 
 
 def _fill_in_ids(tabs: list[TabInfo]):
@@ -41,7 +42,9 @@ def _fill_in_ids(tabs: list[TabInfo]):
             tab.id = f"tab_{i}"
 
 
-def webpage_config(items: list[Item]) -> Item:
+def tabbed_webpage_config(
+    items: list[Item], clean_headings: bool = False, add_title_h1: bool = True
+) -> Item:
     """
     Get an item with the config for a tabbed web page.
     """
@@ -57,9 +60,15 @@ def webpage_config(items: list[Item]) -> Item:
             log.warning("Item has no thumbnail URL: %s", item)
             return None
 
+    def clean_label(label: str) -> str:
+        if clean_headings:
+            return clean_heading(label)
+        else:
+            return abbrev_on_words(sanitize_title(label), max_len=40)
+
     tabs = [
         TabInfo(
-            label=clean_heading(item.abbrev_title()),
+            label=clean_label(item.abbrev_title()),
             store_path=item.store_path,
             thumbnail_url=get_thumbnail_url(item),
         )
@@ -67,7 +76,9 @@ def webpage_config(items: list[Item]) -> Item:
     ]
     _fill_in_ids(tabs)
     title = summary_heading([item.abbrev_title() for item in items])
-    config = TabbedWebpage(title=title, tabs=tabs, show_tabs=len(tabs) > 1)
+    config = TabbedWebpage(
+        title=title, tabs=tabs, show_tabs=len(tabs) > 1, add_title_h1=add_title_h1
+    )
 
     config_item = Item(
         title=f"{title} (config)",
@@ -88,7 +99,9 @@ def _load_tab_content(config: TabbedWebpage):
         tab.content_html = html
 
 
-def webpage_generate(config_item: Item) -> str:
+def tabbed_webpage_generate(
+    config_item: Item, page_template: str = "base_webpage.html.jinja", add_title_h1: bool = True
+) -> str:
     """
     Generate a web page using the supplied config.
     """
@@ -98,14 +111,15 @@ def webpage_generate(config_item: Item) -> str:
     _load_tab_content(tabbed_webpage)
 
     content = render_web_template(
-        base_templates_dir, "tabbed_webpage.html.jinja", asdict(tabbed_webpage)
+        template_filename="tabbed_webpage.html.jinja",
+        data=asdict(tabbed_webpage),
     )
 
     return render_web_template(
-        base_templates_dir,
-        "base_webpage.html.jinja",
-        {
+        page_template,
+        data={
             "title": tabbed_webpage.title,
+            "add_title_h1": add_title_h1,
             "content": content,
         },
     )
@@ -135,11 +149,7 @@ def test_render():
     new_config = as_dataclass(read_yaml_file("tmp/webpage_config.yaml"), TabbedWebpage)
     assert new_config == config
 
-    html = render_web_template(
-        base_templates_dir,
-        "tabbed_webpage.html.jinja",
-        asdict(config),
-    )
+    html = render_web_template(template_filename="tabbed_webpage.html.jinja", data=asdict(config))
     with open("tmp/webpage.html", "w") as f:
         f.write(html)
     print("Rendered tabbed webpage to tmp/webpage.html")