kash-shell 0.3.9__py3-none-any.whl → 0.3.10__py3-none-any.whl

kash/text_handling/markdown_render.py

@@ -0,0 +1,117 @@
+from textwrap import dedent
+
+import marko
+import regex
+from marko.block import HTMLBlock
+from marko.ext.gfm import GFM
+from marko.helpers import MarkoExtension
+
+
+# When we use divs in Markdown we usually want them to be standalone paragraphs,
+# so it doesn't break other wrapping with flowmark etc. This handles that.
+class CustomHTMLBlockMixin:
+    div_pattern = regex.compile(r"^\s*<div\b", regex.IGNORECASE)
+
+    def render_html_block(self, element: HTMLBlock) -> str:
+        # Apply GFM filtering first via the next renderer in the MRO.
+        filtered_body = super().render_html_block(element)  # pyright: ignore
+
+        # Check if the original block was a div.
+        if self.div_pattern.match(element.body.strip()):
+            # If it was a div, wrap the *filtered* result in newlines.
+            return f"\n{filtered_body.strip()}\n"
+        else:
+            # Otherwise, return the GFM-filtered body directly.
+            return filtered_body
+
+
+# GFM first, adding our custom override as an extension to handle divs our way.
+# Extensions later in this list are earlier in MRO.
+MARKO_GFM = marko.Markdown(
+    extensions=["footnote", GFM, MarkoExtension(renderer_mixins=[CustomHTMLBlockMixin])]
+)
+
+
+FOOTNOTE_UP_ARROW = "&nbsp;↑&nbsp;"
+
+
+def html_postprocess(html: str) -> str:
+    """
+    Final tweaks to the HTML.
+    """
+    html = html.replace(
+        """class="footnote">&#8617;</a>""", f"""class="footnote">{FOOTNOTE_UP_ARROW}</a>"""
+    )
+    return html
+
+
+def markdown_to_html(markdown: str, converter: marko.Markdown = MARKO_GFM) -> str:
+    """
+    Convert Markdown to HTML.
+
+    Wraps div blocks with newlines for better Markdown compatibility.
+
+    Output passes through raw HTML! Note per GFM, unsafe script tags etc
+    are [allowed in some cases](https://github.github.com/gfm/#example-140) so
+    additional sanitization is needed if input isn't trusted.
+    """
+    html = converter.convert(markdown)
+    return html_postprocess(html)
+    return html
+
+
+## Tests
+
+
+def test_markdown_to_html():
+    markdown = dedent(
+        """
+        # Heading
+
+        This is a paragraph and a [link](https://example.com).
+
+        - Item 1
+        - Item 2
+
+        ## Subheading
+
+        This is a paragraph with a <span>span</span> tag.
+        This is a paragraph with a <div>div</div> tag.
+        This is a paragraph with an <a href='https://example.com'>example link</a>.
+
+        <div class="div1">This is a div.</div>
+
+        <div class="div2">This is a second div.
+        <iframe src="https://example.com">Inline iframe, note this is sanitized</iframe>
+        </div>
+
+        <!-- Script tag in a block, note this isn't sanitized -->
+        <script>console.log("Javascript block!");</script>
+        """
+    )
+    print(markdown_to_html(markdown))
+
+    expected_html = dedent(
+        """
+        <h1>Heading</h1>
+        <p>This is a paragraph and a <a href="https://example.com">link</a>.</p>
+        <ul>
+        <li>Item 1</li>
+        <li>Item 2</li>
+        </ul>
+        <h2>Subheading</h2>
+        <p>This is a paragraph with a <span>span</span> tag.
+        This is a paragraph with a <div>div</div> tag.
+        This is a paragraph with an <a href='https://example.com'>example link</a>.</p>
+
+        <div class="div1">This is a div.</div>
+
+        <div class="div2">This is a second div.
+        &lt;iframe src="https://example.com">Inline iframe, note this is sanitized</iframe>
+        </div>
+        <!-- Script tag in a block, note this isn't sanitized -->
+        <script>console.log("Javascript block!");</script>
+        """
+    )
+
+    assert markdown_to_html(markdown).strip() == expected_html.strip()

kash/text_handling/markdown_utils.py

@@ -0,0 +1,204 @@
+import re
+from typing import Any
+
+import marko
+import regex
+from marko.block import Heading, ListItem
+from marko.inline import Link
+
+from kash.config.logger import get_logger
+from kash.utils.common.url import Url
+
+log = get_logger(__name__)
+
+# Characters that commonly need escaping in Markdown inline text.
+MARKDOWN_ESCAPE_CHARS = r"([\\`*_{}\[\]()#+.!-])"
+MARKDOWN_ESCAPE_RE = re.compile(MARKDOWN_ESCAPE_CHARS)
+
+
+def escape_markdown(text: str) -> str:
+    """
+    Escape characters with special meaning in Markdown.
+    """
+    return MARKDOWN_ESCAPE_RE.sub(r"\\\1", text)
+
+
+def as_bullet_points(values: list[Any]) -> str:
+    """
+    Convert a list of values to a Markdown bullet-point list. If a value is a string,
+    it is treated like Markdown. If it's something else it's converted to a string
+    and also escaped for Markdown.
+    """
+    points: list[str] = []
+    for value in values:
+        value = value.replace("\n", " ").strip()
+        if isinstance(value, str):
+            points.append(value)
+        else:
+            points.append(escape_markdown(str(value)))
+
+    return "\n\n".join(f"- {point}" for point in points)
+
+
+def markdown_link(text: str, url: str | Url) -> str:
+    """
+    Create a Markdown link.
+    """
+    text = text.replace("[", "\\[").replace("]", "\\]")
+    return f"[{text}]({url})"
+
+
+def is_markdown_header(markdown: str) -> bool:
+    """
+    Is the start of this content a Markdown header?
+    """
+    return regex.match(r"^#+ ", markdown) is not None
+
+
+def _tree_links(element, include_internal=False):
+    links = []
+
+    def _find_links(element):
+        match element:
+            case Link():
+                if include_internal or not element.dest.startswith("#"):
+                    links.append(element.dest)
+            case _:
+                if hasattr(element, "children"):
+                    for child in element.children:
+                        _find_links(child)
+
+    _find_links(element)
+    return links
+
+
+def extract_links(file_path: str, include_internal=False) -> list[str]:
+    """
+    Extract all links from a Markdown file. Future: Include textual and section context.
+    """
+
+    with open(file_path) as file:
+        content = file.read()
+        document = marko.parse(content)
+        return _tree_links(document, include_internal)
+
+
+def _extract_text(element: Any) -> str:
+    if isinstance(element, str):
+        return element
+    elif hasattr(element, "children"):
+        return "".join(_extract_text(child) for child in element.children)
+    else:
+        return ""
+
+
+def _tree_bullet_points(element: marko.block.Document) -> list[str]:
+    bullet_points: list[str] = []
+
+    def _find_bullet_points(element):
+        if isinstance(element, ListItem):
+            bullet_points.append(_extract_text(element).strip())
+        elif hasattr(element, "children"):
+            for child in element.children:
+                _find_bullet_points(child)
+
+    _find_bullet_points(element)
+    return bullet_points
+
+
+def extract_bullet_points(content: str) -> list[str]:
+    """
+    Extract list item values from a Markdown file.
+    """
+
+    document = marko.parse(content)
+    return _tree_bullet_points(document)
+
+
+def _type_from_heading(heading: Heading) -> str:
+    if heading.level in [1, 2, 3, 4, 5, 6]:
+        return f"h{heading.level}"
+    else:
+        raise ValueError(f"Unsupported heading: {heading}: level {heading.level}")
+
+
+def _last_unescaped_bracket(text: str, index: int) -> str | None:
+    escaped = False
+    for i in range(index - 1, -1, -1):
+        ch = text[i]
+        if ch == "\\":
+            escaped = not escaped  # Toggle escaping chain
+            continue
+        if ch in "[]":
+            if not escaped:
+                return ch
+        # Reset escape status after any non‑backslash char
+        escaped = False
+    return None
+
+
+def find_markdown_text(
+    pattern: re.Pattern[str], text: str, *, start_pos: int = 0
+) -> re.Match[str] | None:
+    """
+    Return first regex `pattern` match in `text` not inside an existing link.
+
+    A match is considered inside a link when the most recent unescaped square
+    bracket preceding the match start is an opening bracket "[".
+    """
+
+    pos = start_pos
+    while True:
+        match = pattern.search(text, pos)
+        if match is None:
+            return None
+
+        last_bracket = _last_unescaped_bracket(text, match.start())
+        if last_bracket != "[":
+            return match
+
+        # Skip this match and continue searching
+        pos = match.end()
+
+
+## Tests
+
+
+def test_escape_markdown() -> None:
+    assert escape_markdown("") == ""
+    assert escape_markdown("Hello world") == "Hello world"
+    assert escape_markdown("`code`") == "\\`code\\`"
+    assert escape_markdown("*italic*") == "\\*italic\\*"
+    assert escape_markdown("_bold_") == "\\_bold\\_"
+    assert escape_markdown("{braces}") == "\\{braces\\}"
+    assert escape_markdown("# header") == "\\# header"
+    assert escape_markdown("1. item") == "1\\. item"
+    assert escape_markdown("line+break") == "line\\+break"
+    assert escape_markdown("dash-") == "dash\\-"
+    assert escape_markdown("!bang") == "\\!bang"
+    assert escape_markdown("backslash\\") == "backslash\\\\"
+    assert escape_markdown("Multiple *special* chars [here](#anchor).") == (
+        "Multiple \\*special\\* chars \\[here\\]\\(\\#anchor\\)\\."
+    )
+
+
+def test_find_markdown_text() -> None:  # pragma: no cover
+    # Match is returned when the term is not inside a link.
+    text = "Foo bar baz"
+    pattern = re.compile("Foo Bar", re.IGNORECASE)
+    match = find_markdown_text(pattern, text)
+    assert match is not None and match.group(0) == "Foo bar"
+
+    # Skips occurrence inside link and returns the first one outside.
+    text = "[Foo](http://example.com) something Foo"
+    pattern = re.compile("Foo", re.IGNORECASE)
+    match = find_markdown_text(pattern, text)
+    assert match is not None
+    assert match.start() > text.index(") ")
+    assert text[match.start() : match.end()] == "Foo"
+
+    # Returns None when the only occurrences are inside links.
+    text = "prefix [bar](http://example.com) suffix"
+    pattern = re.compile("bar", re.IGNORECASE)
+    match = find_markdown_text(pattern, text)
+    assert match is None

kash/utils/common/import_utils.py

@@ -15,21 +15,30 @@ Tallies: TypeAlias = dict[str, int]
 def import_subdirs(
     parent_package_name: str,
     parent_dir: Path,
-    subdir_names: list[str],
+    subdir_names: list[str] | None = None,
     tallies: Tallies | None = None,
 ):
     """
     Import all files in the given subdirectories of a single parent directory.
+    Wraps `pkgutil.iter_modules` to iterate over all modules in the subdirectories.
+    If `subdir_names` is `None`, will import all subdirectories.
     """
     if tallies is None:
         tallies = {}
+    if not subdir_names:
+        subdir_names = ["."]
 
     for subdir_name in subdir_names:
-        full_path = parent_dir / subdir_name
+        if subdir_name == ".":
+            full_path = parent_dir
+            package_name = parent_package_name
+        else:
+            full_path = parent_dir / subdir_name
+            package_name = f"{parent_package_name}.{subdir_name}"
+
         if not full_path.is_dir():
             raise FileNotFoundError(f"Subdirectory not found: {full_path}")
 
-        package_name = f"{parent_package_name}.{subdir_name}"
         for _module_finder, module_name, _is_pkg in pkgutil.iter_modules(path=[str(full_path)]):
             importlib.import_module(f"{package_name}.{module_name}")  # Propagate import errors
             tallies[package_name] = tallies.get(package_name, 0) + 1

kash/utils/common/type_utils.py

@@ -15,35 +15,6 @@ def not_none(value: T | None, message: str | None = None) -> T:
     return value
 
 
-def is_truthy(value: Any, strict: bool = True) -> bool:
-    """
-    True for all common string and non-string values for true. Useful for parsing
-    string values or command line arguments.
-    """
-    truthy_values = {"true", "1", "yes", "on", "y"}
-    falsy_values = {"false", "0", "no", "off", "n", ""}
-
-    if value is None:
-        return False
-    elif isinstance(value, str):
-        value = value.strip().lower()
-        if value in truthy_values:
-            return True
-        elif value in falsy_values:
-            return False
-    elif isinstance(value, (int, float)):
-        return value != 0
-    elif isinstance(value, bool):
-        return value
-    elif isinstance(value, (list, tuple, set, dict)):
-        return len(value) > 0
-
-    if strict:
-        raise ValueError(f"Could not convert type {type(value)} to boolean: {repr(value)}")
-
-    return bool(value)
-
-
 def as_dataclass(dict_data: dict[str, Any], dataclass_type: type[T]) -> T:
     """
     Convert a dict recursively to dataclass object, raising an error if the data does

kash/utils/common/url.py

@@ -67,15 +67,39 @@ def is_file_url(url: str | Url) -> bool:
     return url.startswith("file://")
 
 
-def parse_file_url(url: str | Url) -> Path | None:
+def parse_http_url(url: str | Url) -> ParseResult:
     """
-    Parse a file URL and return the path, or None if not a file URL.
+    Parse an http/https URL and return the parsed result, raising ValueError if
+    not an http/https URL.
+    """
+    parsed_url = urlparse(url)
+    if parsed_url.scheme in ("http", "https"):
+        return parsed_url
+    else:
+        raise ValueError(f"Not an http/https URL: {url}")
+
+
+def parse_file_url(url: str | Url) -> Path:
+    """
+    Parse a file URL and return the path, raising ValueError if not a file URL.
     """
     parsed_url = urlparse(url)
     if parsed_url.scheme == "file":
         return Path(parsed_url.path)
     else:
-        return None
+        raise ValueError(f"Not a file URL: {url}")
+
+
+def parse_s3_url(url: str | Url) -> tuple[str, str]:
+    """
+    Parse an S3 URL and return the bucket and key, raising ValueError if not an
+    S3 URL.
+    """
+    parsed_url = urlparse(url)
+    if parsed_url.scheme == "s3":
+        return parsed_url.netloc, parsed_url.path.lstrip("/")
+    else:
+        raise ValueError(f"Not an S3 URL: {url}")
 
 
 def as_file_url(path: str | Path) -> Url:

kash/utils/errors.py

@@ -139,6 +139,12 @@ class FileFormatError(ContentError):
     pass
 
 
+class ApiError(KashRuntimeError):
+    """Raised when an API call returns something unexpected."""
+
+    pass
+
+
 def _nonfatal_exceptions() -> tuple[type[Exception], ...]:
     exceptions: list[type[Exception]] = [SelfExplanatoryError, FileNotFoundError, IOError]
     try:

kash/utils/file_utils/file_formats.py

@@ -4,9 +4,9 @@ from pathlib import Path
 from typing import NewType
 
 import regex
+from clideps.pkgs.pkg_check import pkg_check
 
 from kash.config.logger import get_logger
-from kash.shell.clideps.pkg_deps import Pkg, pkg_check
 
 log = get_logger(__name__)
 
@@ -86,7 +86,7 @@ def detect_mime_type(filename: str | Path) -> MimeType | None:
     Get the mime type of a file using libmagic heuristics plus more careful
     detection of HTML, Markdown, and multipart YAML.
     """
-    pkg_check().require(Pkg.libmagic)
+    pkg_check().require("libmagic")
     import magic
 
     mime = magic.Magic(mime=True)

kash/utils/file_utils/file_formats_model.py

@@ -36,6 +36,8 @@ class Format(Enum):
     it is the format of the resource (url, media, etc.).
     """
 
+    # TODO: Be more thorough, pulling in relevant extensions and types from the `mimetypes` module.
+
     # Formats with no body (content is in frontmatter).
     url = "url"
 
@@ -146,6 +148,7 @@ class Format(Enum):
             self.markdown,
             self.md_html,
             self.html,
+            self.json,  # Not strictly true but we encourage use of comments.
             self.yaml,
             self.diff,
             self.python,

kash/web_content/dir_store.py

@@ -87,8 +87,7 @@ class DirStore:
         self, keys: list[str | Path], folder: str | None = None, suffix: str | None = None
     ) -> dict[str | Path, Path | None]:
         """
-        Look up all existing cached results for the set of keys. This should work fine but could
-        be optimized for large batches.
+        Look up all existing cached results for the set of keys.
         """
         return {key: self.find(key, folder=folder, suffix=suffix) for key in keys}
 

kash/web_content/file_cache_utils.py

@@ -1,4 +1,7 @@
+import json
+from collections.abc import Callable
 from pathlib import Path
+from typing import Any
 
 from prettyfmt import fmt_lines, fmt_path
 
@@ -35,18 +38,40 @@ def reset_content_cache_dir(path: Path):
             log.info("Using web cache: %s", fmt_path(path))
 
 
-def cache_file(source: Url | Path | Loadable, global_cache: bool = False) -> tuple[Path, bool]:
+def cache_file(
+    source: Url | Path | Loadable, global_cache: bool = False, expiration_sec: float | None = None
+) -> tuple[Path, bool]:
     """
     Return a local cached copy of the item. If it is an URL, content is fetched.
-    Raises requests.HTTPError if the URL is not reachable. If it is a Path or
-    a Loadable, a cached copy is returned.
+    If it is a Path or a Loadable, a cached copy is returned.
+    LocalFileCache uses httpx so httpx.HTTPError is raised for non-2xx responses.
+
+    Uses the current content cache unless there is no current cache or `global_cache` is True,
+    in which case the global cache is used.
     """
     cache = _global_content_cache if global_cache else _content_cache
-    path, was_cached = cache.cache(source)
+    path, was_cached = cache.cache(source, expiration_sec)
     return path, was_cached
 
 
-def cache_resource(item: Item) -> dict[MediaType, Path]:
+def cache_api_response(
+    url: Url,
+    global_cache: bool = False,
+    expiration_sec: float | None = None,
+    parser: Callable[[str], Any] = json.loads,
+) -> tuple[Any, bool]:
+    """
+    Cache an API response. By default parse the response as JSON.
+    """
+    cache = _global_content_cache if global_cache else _content_cache
+    path, was_cached = cache.cache(url, expiration_sec)
+    result = parser(path.read_text())
+    return result, was_cached
+
+
+def cache_resource(
+    item: Item, global_cache: bool = False, expiration_sec: float | None = None
+) -> dict[MediaType, Path]:
     """
     Cache a resource item for an external local path or a URL, fetching or
     copying as needed. For media this may yield more than one format.
@@ -64,17 +89,17 @@ def cache_resource(item: Item) -> dict[MediaType, Path]:
         if is_media_url(item.url):
             result = cache_media(item.url)
         else:
-            path, _was_cached = cache_file(item.url)
+            path, _was_cached = cache_file(item.url, global_cache, expiration_sec)
     elif item.external_path:
         path = Path(item.external_path)
         if not path.is_file():
             raise FileNotFound(f"External path not found: {path}")
-        path, _was_cached = cache_file(path)
+        path, _was_cached = cache_file(path, global_cache, expiration_sec)
     elif item.original_filename:
         path = Path(item.original_filename)
         if not path.is_file():
             raise FileNotFound(f"Original filename not found: {path}")
-        path, _was_cached = cache_file(path)
+        path, _was_cached = cache_file(path, global_cache, expiration_sec)
     else:
         raise ValueError(f"Item has no URL or external path: {item}")
 
@@ -94,7 +119,9 @@ def cache_resource(item: Item) -> dict[MediaType, Path]:
     return result
 
 
-def get_url_html(item: Item) -> tuple[Url, str]:
+def get_url_html(
+    item: Item, global_cache: bool = False, expiration_sec: float | None = None
+) -> tuple[Url, str]:
     """
     Returns the HTML content of an URL item, using the content cache,
     or the body of the item if it has a URL and HTML body.
@@ -106,7 +133,7 @@ def get_url_html(item: Item) -> tuple[Url, str]:
     url = Url(canonicalize_url(item.url))
 
     if is_url_item(item):
-        path, _was_cached = cache_file(url)
+        path, _was_cached = cache_file(url, global_cache, expiration_sec)
         with open(path) as file:
             html_content = file.read()
     else:

kash/web_content/file_processing.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TypeAlias
+
+from kash.web_content.local_file_cache import read_mtime
+
+
+@dataclass(frozen=True)
+class OutputType:
+    """
+    A type of output file, represented by the filename suffix, e.g. '.mp3', '.txt', etc.
+    """
+
+    suffix: str
+
+    def output_path(self, src: Path) -> Path:
+        """
+        Resolve the output path. Will be next to the source file, e.g.
+        some-dir/video.mp4 -> some-dir/video.mp3
+        """
+        return src.with_suffix(self.suffix)
+
+
+Processor: TypeAlias = Callable[[Path, Mapping[OutputType, Path]], None]
+"""
+A function that takes a source file and a mapping with one or more output paths.
+"""
+
+
+@dataclass(frozen=True)
+class FileProcess:
+    """
+    Process a file and produce one or more outputs.
+    """
+
+    processor: Processor
+    outputs: list[OutputType]
+
+    def is_outdated(self, src: Path) -> bool:
+        """
+        True when any output is missing or older (earliest mtime) than `src`.
+        """
+        dests = {o.output_path(src) for o in self.outputs}
+        if any(not p.exists() for p in dests):
+            return True
+        earliest = min(read_mtime(p) for p in dests)
+        return read_mtime(src) > earliest
+
+    def run(self, src: Path) -> dict[OutputType, Path]:
+        """
+        Run unconditionally and return a mapping of outputs to paths.
+        """
+        dests = {o: o.output_path(src) for o in self.outputs}
+        self.processor(src, dests)
+        return dests
+
+    def run_if_needed(self, src: Path) -> dict[OutputType, Path]:
+        """
+        Run only if any output is missing or outdated.
+        """
+        return (
+            self.run(src)
+            if self.is_outdated(src)
+            else {o: o.output_path(src) for o in self.outputs}
+        )