kash-shell 0.3.28__py3-none-any.whl → 0.3.33__py3-none-any.whl

kash/utils/common/s3_utils.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import shutil
+import subprocess
+from pathlib import Path
+
+from sidematter_format.sidematter_format import Sidematter
+
+from kash.utils.common.url import Url, is_s3_url, parse_s3_url
+
+
+def check_aws_cli() -> None:
+    """
+    Check if the AWS CLI is installed and available.
+    """
+    if shutil.which("aws") is None:
+        raise RuntimeError(
+            "AWS CLI not found in PATH. Please install 'awscli' and ensure 'aws' is available."
+        )
+
+
+def get_s3_parent_folder(url: Url) -> Url | None:
+    """
+    Get the parent folder of an S3 URL, or None if not an S3 URL.
+    """
+    if is_s3_url(url):
+        s3_bucket, s3_key = parse_s3_url(url)
+        s3_parent_folder = Path(s3_key).parent
+
+        return Url(f"s3://{s3_bucket}/{s3_parent_folder}")
+
+    else:
+        return None
+
+
+def s3_sync_to_folder(
+    src_path: str | Path,
+    s3_dest_parent: Url,
+    *,
+    include_sidematter: bool = False,
+) -> list[Url]:
+    """
+    Sync a local file or directory to an S3 "parent" folder using the AWS CLI.
+    Set `include_sidematter` to include sidematter files alongside the source files.
+
+    Returns a list of S3 URLs that were the top-level sync targets:
+    - For a single file: the file URL (and sidematter file/dir URLs if included).
+    - For a directory: the destination parent prefix URL (non-recursive reporting).
+    """
+
+    src_path = Path(src_path)
+    if not src_path.exists():
+        raise ValueError(f"Source path does not exist: {src_path}")
+    if not is_s3_url(s3_dest_parent):
+        raise ValueError(f"Destination must be an s3:// URL: {s3_dest_parent}")
+
+    check_aws_cli()
+
+    dest_prefix = str(s3_dest_parent).rstrip("/") + "/"
+    targets: list[Url] = []
+
+    if src_path.is_file():
+        # Build the list of paths to sync using Sidematter's resolved path_list if requested.
+        sync_paths: list[Path]
+        if include_sidematter:
+            resolved = Sidematter(src_path).resolve(parse_meta=False, use_frontmatter=False)
+            sync_paths = resolved.path_list
+        else:
+            sync_paths = [src_path]
+
+        for p in sync_paths:
+            if p.is_file():
+                # Use sync with include/exclude to leverage default short-circuiting
+                subprocess.run(
+                    [
+                        "aws",
+                        "s3",
+                        "sync",
+                        str(p.parent),
+                        dest_prefix,
+                        "--exclude",
+                        "*",
+                        "--include",
+                        p.name,
+                    ],
+                    check=True,
+                )
+                targets.append(Url(dest_prefix + p.name))
+            elif p.is_dir():
+                dest_dir = dest_prefix + p.name + "/"
+                subprocess.run(["aws", "s3", "sync", str(p), dest_dir], check=True)
+                targets.append(Url(dest_dir))
+
+        return targets
+    else:
+        # Directory mode: sync whole directory.
+        subprocess.run(
+            [
+                "aws",
+                "s3",
+                "sync",
+                str(src_path),
+                dest_prefix,
+            ],
+            check=True,
+        )
+        targets.append(Url(dest_prefix))
+        return targets

kash/utils/common/url.py

@@ -26,6 +26,7 @@ A string that may not be resolved to a URL or path.
 
 HTTP_ONLY = ["http", "https"]
 HTTP_OR_FILE = HTTP_ONLY + ["file"]
+HTTP_OR_FILE_OR_S3 = HTTP_OR_FILE + ["s3"]
 
 
 def check_if_url(
@@ -36,7 +37,8 @@ def check_if_url(
     the `urlparse.ParseResult`.
 
     Also returns false for Paths, so that it's easy to use local paths and URLs
-    (`Locator`s) interchangeably. Can provide `HTTP_ONLY` or `HTTP_OR_FILE` to
+    (`Locator`s) interchangeably. Can provide `HTTP_ONLY` or `HTTP_OR_FILE`
+    or `HTTP_OR_FILE_OR_S3` to restrict to only certain schemes.
     restrict to only certain schemes.
     """
     if isinstance(text, Path):
@@ -69,6 +71,13 @@ def is_file_url(url: str | Url) -> bool:
     return url.startswith("file://")
 
 
+def is_s3_url(url: str | Url) -> bool:
+    """
+    Is URL an S3 URL?
+    """
+    return url.startswith("s3://")
+
+
 def parse_http_url(url: str | Url) -> ParseResult:
     """
     Parse an http/https URL and return the parsed result, raising ValueError if
@@ -118,7 +127,7 @@ def as_file_url(path: str | Path) -> Url:
 
 def normalize_url(
     url: Url,
-    check_schemes: list[str] | None = HTTP_OR_FILE,
+    check_schemes: list[str] | None = HTTP_OR_FILE_OR_S3,
     drop_fragment: bool = True,
     resolve_local_paths: bool = True,
 ) -> Url:
@@ -238,7 +247,10 @@ def test_normalize_url():
         normalize_url(url=Url("/not/a/URL"))
         raise AssertionError()
     except ValueError as e:
-        assert str(e) == "Scheme '' not in allowed schemes: ['http', 'https', 'file']: /not/a/URL"
+        assert (
+            str(e)
+            == "Scheme '' not in allowed schemes: ['http', 'https', 'file', 's3']: /not/a/URL"
+        )
 
     try:
         normalize_url(Url("ftp://example.com"))
@@ -246,7 +258,7 @@ def test_normalize_url():
     except ValueError as e:
         assert (
             str(e)
-            == "Scheme 'ftp' not in allowed schemes: ['http', 'https', 'file']: ftp://example.com"
+            == "Scheme 'ftp' not in allowed schemes: ['http', 'https', 'file', 's3']: ftp://example.com"
         )
 
 

kash/utils/file_formats/chat_format.py

@@ -93,7 +93,6 @@ content: |
 
 from __future__ import annotations
 
-import json
 from dataclasses import field
 from enum import Enum
 from io import StringIO
@@ -104,6 +103,7 @@ from typing import Any
 from frontmatter_format import from_yaml_string, new_yaml, to_yaml_string
 from prettyfmt import abbrev_obj, custom_key_sort, fmt_size_human
 from pydantic.dataclasses import dataclass
+from sidematter_format import to_json_string
 
 
 class ChatRole(str, Enum):
@@ -161,9 +161,12 @@ class ChatMessage:
         Convert to a format that can be used as a standard chat completion, with
         the content field holding JSON-serialized data if it is structured.
         """
+
         return {
             "role": self.role.value,
-            "content": json.dumps(self.content) if isinstance(self.content, dict) else self.content,
+            "content": to_json_string(self.content)
+            if isinstance(self.content, dict)
+            else self.content,
         }
 
     @classmethod
@@ -174,7 +177,7 @@ class ChatMessage:
         return to_yaml_string(self.as_dict(), key_sort=_custom_key_sort)
 
     def to_json(self) -> str:
-        return json.dumps(self.as_dict())
+        return to_json_string(self.as_dict())
 
     def as_str(self) -> str:
         return self.to_yaml()
@@ -222,7 +225,7 @@ class ChatHistory:
         return stream.getvalue()
 
     def to_json(self) -> str:
-        return json.dumps([message.as_dict() for message in self.messages])
+        return to_json_string([message.as_dict() for message in self.messages], indent=None)
 
     def size_summary(self) -> str:
         role_counts = {}

kash/utils/file_utils/file_ext.py

@@ -37,6 +37,7 @@ class FileExt(Enum):
     mp4 = "mp4"
     pptx = "pptx"
     epub = "epub"
+    zip = "zip"
 
     @property
     def dot_ext(self) -> str:

kash/utils/file_utils/file_formats.py

@@ -16,7 +16,7 @@ def is_fullpage_html(content: str) -> bool:
     A full HTML document that is a full page (headers, footers, etc.) and
     so probably best rendered in a browser.
     """
-    return bool(re.search(r"<!DOCTYPE html>|<html>|<body>|<head>", content, re.IGNORECASE))
+    return bool(re.search(r"<!DOCTYPE html>|<html.*?>|<body>|<head>", content, re.IGNORECASE))
 
 
 _yaml_header_pattern = re.compile(r"^---\n\w+:", re.MULTILINE)
@@ -35,7 +35,9 @@ def is_html(content: str) -> bool:
     """
     return bool(
         re.search(
-            r"<!DOCTYPE html>|<html>|<body>|<head>|<div>|<p>|<img |<a href", content, re.IGNORECASE
+            r"<!DOCTYPE html>|<html.*?>|<body>|<head>|<div>|<p>|<img |<a href",
+            content,
+            re.IGNORECASE,
         )
     )
 

kash/utils/file_utils/file_formats_model.py

@@ -72,6 +72,9 @@ class Format(Enum):
     mp3 = "mp3"
     m4a = "m4a"
     mp4 = "mp4"
+
+    # Binary formats.
+    zip = "zip"
     binary = "binary"
     """Catch-all format for binary files that are unrecognized."""
 
@@ -167,6 +170,10 @@ class Format(Enum):
     def is_data(self) -> bool:
         return self in [self.csv, self.xlsx, self.npz]
 
+    @property
+    def is_zip(self) -> bool:
+        return self in [self.zip]
+
     @property
     def is_binary(self) -> bool:
         return self.has_body and not self.is_text
@@ -257,6 +264,7 @@ class Format(Enum):
             FileExt.m4a.value: Format.m4a,
             FileExt.mp4.value: Format.mp4,
             FileExt.epub.value: Format.epub,
+            FileExt.zip.value: Format.zip,
         }
         return ext_to_format.get(file_ext.value, None)
 
@@ -292,6 +300,7 @@ class Format(Enum):
             Format.mp3: FileExt.mp3,
             Format.m4a: FileExt.m4a,
             Format.mp4: FileExt.mp4,
+            Format.zip: FileExt.zip,
         }
 
         return format_to_file_ext.get(self, None)
@@ -329,6 +338,9 @@ class Format(Enum):
             "audio/mp3": Format.mp3,
             "audio/mp4": Format.m4a,
             "video/mp4": Format.mp4,
+            "application/zip": Format.zip,
+            "application/x-zip": Format.zip,
+            "application/x-zip-compressed": Format.zip,
             "application/octet-stream": Format.binary,
         }
 

kash/utils/text_handling/doc_normalization.py

@@ -75,7 +75,7 @@ def normalize_text_file(
 
 def test_osc8_link():
     from clideps.terminal.osc_utils import osc8_link
-    from flowmark.text_wrapping import wrap_paragraph
+    from flowmark import wrap_paragraph
 
     link = osc8_link("https://example.com/" + "x" * 50, "Example")
     assert ansi_cell_len(link) == 7

kash/utils/text_handling/markdown_footnotes.py

@@ -0,0 +1,224 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+from flowmark import flowmark_markdown, line_wrap_by_sentence
+from marko import Markdown
+from marko.ext import footnote
+
+from kash.utils.text_handling.markdown_utils import comprehensive_transform_tree
+
+
+def _normalize_footnotes_in_markdown(content: str) -> str:
+    """
+    Ensure blank lines between consecutive footnote definitions.
+
+    Marko has a bug where consecutive footnotes without blank lines are parsed
+    as a single footnote. This adds blank lines where needed.
+    """
+    lines = content.split("\n")
+    result = []
+    i = 0
+
+    while i < len(lines):
+        line = lines[i]
+        result.append(line)
+
+        # Check if this is a footnote definition
+        if re.match(r"^\[\^[^\]]+\]:", line):
+            # Look ahead to see if the next non-empty line is also a footnote
+            j = i + 1
+            while j < len(lines) and not lines[j].strip():
+                result.append(lines[j])
+                j += 1
+
+            if j < len(lines) and re.match(r"^\[\^[^\]]+\]:", lines[j]):
+                # Next non-empty line is also a footnote, add blank line
+                result.append("")
+
+            i = j
+        else:
+            i += 1
+
+    return "\n".join(result)
+
+
+@dataclass
+class FootnoteInfo:
+    """
+    Information about a single footnote definition.
+    """
+
+    footnote_id: str  # The footnote ID with caret (e.g., "^123", "^foo")
+    content: str  # The rendered markdown content of the footnote
+    raw_element: footnote.FootnoteDef  # The original marko element
+
+
+@dataclass
+class MarkdownFootnotes:
+    """
+    Container for all footnotes in a markdown document with fast lookup.
+
+    Provides efficient access to footnote definitions by their IDs.
+    IDs are stored with the leading caret (^) to avoid collisions.
+    """
+
+    footnotes: dict[str, FootnoteInfo] = field(default_factory=dict)
+    """Dictionary mapping footnote IDs (with ^) to FootnoteInfo objects."""
+
+    @staticmethod
+    def from_markdown(content: str, markdown_parser: Markdown | None = None) -> MarkdownFootnotes:
+        """
+        Extract all footnotes from markdown content.
+
+        Args:
+            content: The markdown content to parse
+            markdown_parser: Optional custom markdown parser. If None, uses default flowmark setup.
+
+        Returns:
+            MarkdownFootnotes instance with all footnotes indexed by ID
+        """
+        if markdown_parser is None:
+            markdown_parser = flowmark_markdown(line_wrap_by_sentence(is_markdown=True))
+
+        # Normalize to work around marko bug with consecutive footnotes
+        normalized_content = _normalize_footnotes_in_markdown(content)
+        document = markdown_parser.parse(normalized_content)
+        return MarkdownFootnotes.from_document(document, markdown_parser)
+
+    @staticmethod
+    def from_document(document: Any, markdown_parser: Markdown | None = None) -> MarkdownFootnotes:
+        """
+        Extract all footnotes from a parsed markdown document.
+
+        Args:
+            document: A parsed marko document object
+            markdown_parser: The markdown parser used (needed for rendering).
+                           If None, uses default flowmark setup.
+
+        Returns:
+            MarkdownFootnotes instance with all footnotes indexed by ID
+        """
+        if markdown_parser is None:
+            markdown_parser = flowmark_markdown(line_wrap_by_sentence(is_markdown=True))
+
+        footnotes_dict: dict[str, FootnoteInfo] = {}
+
+        def collect_footnote(element: Any) -> None:
+            if isinstance(element, footnote.FootnoteDef):
+                content_parts = []
+                if hasattr(element, "children") and element.children:
+                    for child in element.children:
+                        rendered = markdown_parser.renderer.render(child)
+                        content_parts.append(rendered)
+
+                rendered_content = "".join(content_parts).strip()
+
+                footnote_id = f"^{element.label}"
+                footnotes_dict[footnote_id] = FootnoteInfo(
+                    footnote_id=footnote_id,
+                    content=rendered_content,
+                    raw_element=element,
+                )
+
+        comprehensive_transform_tree(document, collect_footnote)
+
+        return MarkdownFootnotes(footnotes=footnotes_dict)
+
+    def get(self, footnote_id: str, default: FootnoteInfo | None = None) -> FootnoteInfo | None:
+        """
+        Get a footnote by its ID.
+
+        Args:
+            footnote_id: The footnote ID (with or without leading ^)
+            default: Default value if footnote not found
+
+        Returns:
+            FootnoteInfo if found, otherwise default value
+        """
+        if not footnote_id.startswith("^"):
+            footnote_id = f"^{footnote_id}"
+        return self.footnotes.get(footnote_id, default)
+
+    def __getitem__(self, footnote_id: str) -> FootnoteInfo:
+        """
+        Get a footnote by its ID using dictionary-style access.
+
+        Args:
+            footnote_id: The footnote ID (with or without leading ^)
+
+        Returns:
+            FootnoteInfo for the ID
+
+        Raises:
+            KeyError: If the footnote ID is not found
+        """
+        if not footnote_id.startswith("^"):
+            footnote_id = f"^{footnote_id}"
+        return self.footnotes[footnote_id]
+
+    def __contains__(self, footnote_id: str) -> bool:
+        """
+        Check if a footnote exists.
+
+        Args:
+            footnote_id: The footnote ID (with or without leading ^)
+        """
+        if not footnote_id.startswith("^"):
+            footnote_id = f"^{footnote_id}"
+        return footnote_id in self.footnotes
+
+    def __len__(self) -> int:
+        """Return the number of footnotes."""
+        return len(self.footnotes)
+
+    def __iter__(self):
+        """Iterate over footnote IDs (with carets)."""
+        return iter(self.footnotes)
+
+    def items(self):
+        """Return (footnote_id, FootnoteInfo) pairs."""
+        return self.footnotes.items()
+
+    def values(self):
+        """Return FootnoteInfo objects."""
+        return self.footnotes.values()
+
+    def keys(self):
+        """Return footnote IDs (with carets)."""
+        return self.footnotes.keys()
+
+
+def extract_footnote_references(content: str, markdown_parser: Markdown | None = None) -> list[str]:
+    """
+    Extract all footnote reference IDs used in the content.
+
+    This finds all FootnoteRef elements (e.g., [^123] in the text) as opposed
+    to FootnoteDef elements which are the definitions.
+
+    Args:
+        content: The markdown content to parse
+        markdown_parser: Optional custom markdown parser
+
+    Returns:
+        List of unique footnote IDs that are referenced (with the ^)
+    """
+    if markdown_parser is None:
+        markdown_parser = flowmark_markdown(line_wrap_by_sentence(is_markdown=True))
+
+    normalized_content = _normalize_footnotes_in_markdown(content)
+    document = markdown_parser.parse(normalized_content)
+    references: list[str] = []
+    seen: set[str] = set()
+
+    def collect_references(element: Any) -> None:
+        if isinstance(element, footnote.FootnoteRef):
+            footnote_id = f"^{element.label}"
+            if footnote_id not in seen:
+                seen.add(footnote_id)
+                references.append(footnote_id)
+
+    comprehensive_transform_tree(document, collect_references)
+    return references