kash-shell 0.3.30__py3-none-any.whl → 0.3.34__py3-none-any.whl

kash/utils/common/s3_utils.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import shutil
+import subprocess
+from pathlib import Path
+
+from sidematter_format.sidematter_format import Sidematter
+
+from kash.utils.common.url import Url, is_s3_url, parse_s3_url
+
+
+def check_aws_cli() -> None:
+    """
+    Check if the AWS CLI is installed and available.
+    """
+    if shutil.which("aws") is None:
+        raise RuntimeError(
+            "AWS CLI not found in PATH. Please install 'awscli' and ensure 'aws' is available."
+        )
+
+
+def get_s3_parent_folder(url: Url) -> Url | None:
+    """
+    Get the parent folder of an S3 URL, or None if not an S3 URL.
+    """
+    if is_s3_url(url):
+        s3_bucket, s3_key = parse_s3_url(url)
+        s3_parent_folder = Path(s3_key).parent
+
+        return Url(f"s3://{s3_bucket}/{s3_parent_folder}")
+
+    else:
+        return None
+
+
+def s3_sync_to_folder(
+    src_path: str | Path,
+    s3_dest_parent: Url,
+    *,
+    include_sidematter: bool = False,
+) -> list[Url]:
+    """
+    Sync a local file or directory to an S3 "parent" folder using the AWS CLI.
+    Set `include_sidematter` to include sidematter files alongside the source files.
+
+    Returns a list of S3 URLs that were the top-level sync targets:
+    - For a single file: the file URL (and sidematter file/dir URLs if included).
+    - For a directory: the destination parent prefix URL (non-recursive reporting).
+    """
+
+    src_path = Path(src_path)
+    if not src_path.exists():
+        raise ValueError(f"Source path does not exist: {src_path}")
+    if not is_s3_url(s3_dest_parent):
+        raise ValueError(f"Destination must be an s3:// URL: {s3_dest_parent}")
+
+    check_aws_cli()
+
+    dest_prefix = str(s3_dest_parent).rstrip("/") + "/"
+    targets: list[Url] = []
+
+    if src_path.is_file():
+        # Build the list of paths to sync using Sidematter's resolved path_list if requested.
+        sync_paths: list[Path]
+        if include_sidematter:
+            resolved = Sidematter(src_path).resolve(parse_meta=False, use_frontmatter=False)
+            sync_paths = resolved.path_list
+        else:
+            sync_paths = [src_path]
+
+        for p in sync_paths:
+            if p.is_file():
+                # Use sync with include/exclude to leverage default short-circuiting
+                subprocess.run(
+                    [
+                        "aws",
+                        "s3",
+                        "sync",
+                        str(p.parent),
+                        dest_prefix,
+                        "--exclude",
+                        "*",
+                        "--include",
+                        p.name,
+                    ],
+                    check=True,
+                )
+                targets.append(Url(dest_prefix + p.name))
+            elif p.is_dir():
+                dest_dir = dest_prefix + p.name + "/"
+                subprocess.run(["aws", "s3", "sync", str(p), dest_dir], check=True)
+                targets.append(Url(dest_dir))
+
+        return targets
+    else:
+        # Directory mode: sync whole directory.
+        subprocess.run(
+            [
+                "aws",
+                "s3",
+                "sync",
+                str(src_path),
+                dest_prefix,
+            ],
+            check=True,
+        )
+        targets.append(Url(dest_prefix))
+        return targets

kash/utils/common/url.py

@@ -26,6 +26,7 @@ A string that may not be resolved to a URL or path.
 
 HTTP_ONLY = ["http", "https"]
 HTTP_OR_FILE = HTTP_ONLY + ["file"]
+HTTP_OR_FILE_OR_S3 = HTTP_OR_FILE + ["s3"]
 
 
 def check_if_url(
@@ -36,7 +37,8 @@ def check_if_url(
     the `urlparse.ParseResult`.
 
     Also returns false for Paths, so that it's easy to use local paths and URLs
-    (`Locator`s) interchangeably. Can provide `HTTP_ONLY` or `HTTP_OR_FILE` to
+    (`Locator`s) interchangeably. Can provide `HTTP_ONLY` or `HTTP_OR_FILE`
+    or `HTTP_OR_FILE_OR_S3` to restrict to only certain schemes.
     restrict to only certain schemes.
     """
     if isinstance(text, Path):
@@ -69,6 +71,13 @@ def is_file_url(url: str | Url) -> bool:
     return url.startswith("file://")
 
 
+def is_s3_url(url: str | Url) -> bool:
+    """
+    Is URL an S3 URL?
+    """
+    return url.startswith("s3://")
+
+
 def parse_http_url(url: str | Url) -> ParseResult:
     """
     Parse an http/https URL and return the parsed result, raising ValueError if
@@ -118,7 +127,7 @@ def as_file_url(path: str | Path) -> Url:
 
 def normalize_url(
     url: Url,
-    check_schemes: list[str] | None = HTTP_OR_FILE,
+    check_schemes: list[str] | None = HTTP_OR_FILE_OR_S3,
     drop_fragment: bool = True,
     resolve_local_paths: bool = True,
 ) -> Url:
@@ -238,7 +247,10 @@ def test_normalize_url():
         normalize_url(url=Url("/not/a/URL"))
         raise AssertionError()
     except ValueError as e:
-        assert str(e) == "Scheme '' not in allowed schemes: ['http', 'https', 'file']: /not/a/URL"
+        assert (
+            str(e)
+            == "Scheme '' not in allowed schemes: ['http', 'https', 'file', 's3']: /not/a/URL"
+        )
 
     try:
         normalize_url(Url("ftp://example.com"))
@@ -246,7 +258,7 @@ def test_normalize_url():
     except ValueError as e:
         assert (
             str(e)
-            == "Scheme 'ftp' not in allowed schemes: ['http', 'https', 'file']: ftp://example.com"
+            == "Scheme 'ftp' not in allowed schemes: ['http', 'https', 'file', 's3']: ftp://example.com"
         )
 
 

kash/utils/rich_custom/multitask_status.py

@@ -72,6 +72,8 @@ RUNNING_SYMBOL = ""
 DEFAULT_LABEL_WIDTH = 40
 DEFAULT_PROGRESS_WIDTH = 20
 
+MAX_DISPLAY_TASKS = 20
+
 
 # Calculate spinner width to maintain column alignment
 def _get_spinner_width(spinner_name: str) -> int:
@@ -101,6 +103,9 @@ class StatusSettings:
     transient: bool = True
     refresh_per_second: float = 10
     styles: StatusStyles = DEFAULT_STYLES
+    # Maximum number of tasks to keep visible in the live display.
+    # Older completed/skipped/failed tasks beyond this cap will be removed from the live view.
+    max_display_tasks: int = MAX_DISPLAY_TASKS
 
 
 class SpinnerStatusColumn(ProgressColumn):
@@ -298,6 +303,10 @@ class MultiTaskStatus(AbstractAsyncContextManager):
         self._task_info: dict[int, TaskInfo] = {}
         self._next_id: int = 1
         self._rich_task_ids: dict[int, TaskID] = {}  # Map our IDs to Rich Progress IDs
+        # Track order of tasks added to the Progress so we can prune oldest completed ones
+        self._displayed_task_order: list[int] = []
+        # Track tasks pruned from the live display so we don't re-add them later
+        self._pruned_task_ids: set[int] = set()
 
         # Unified live integration
         self._unified_live: Any | None = None  # Reference to the global unified live
@@ -442,6 +451,10 @@ class MultiTaskStatus(AbstractAsyncContextManager):
                 progress_display=None,
             )
             self._rich_task_ids[task_id] = rich_task_id
+            self._displayed_task_order.append(task_id)
+
+            # Prune if too many tasks are visible (prefer removing completed ones)
+            self._prune_completed_tasks_if_needed()
 
     async def set_progress_display(self, task_id: int, display: RenderableType) -> None:
         """
@@ -536,18 +549,31 @@ class MultiTaskStatus(AbstractAsyncContextManager):
 
             # Complete the progress bar and stop spinner
             if rich_task_id is not None:
-                total = self._progress.tasks[rich_task_id].total or 1
+                # Safely find the Task by id; Progress.tasks is a list, not a dict
+                task_obj = next((t for t in self._progress.tasks if t.id == rich_task_id), None)
+                if task_obj is not None and task_obj.total is not None:
+                    total = task_obj.total
+                else:
+                    total = task_info.steps_total or 1
                 self._progress.update(rich_task_id, completed=total, task_info=task_info)
             else:
-                # Task was never started, but we still need to add it to show completion
-                rich_task_id = self._progress.add_task(
-                    "",
-                    total=task_info.steps_total,
-                    label=task_info.label,
-                    completed=task_info.steps_total,
-                    task_info=task_info,
-                )
-                self._rich_task_ids[task_id] = rich_task_id
+                # If this task was pruned from the live display, skip re-adding it
+                if task_id in self._pruned_task_ids:
+                    pass
+                else:
+                    # Task was never started; add a completed row so it appears once
+                    rich_task_id = self._progress.add_task(
+                        "",
+                        total=task_info.steps_total,
+                        label=task_info.label,
+                        completed=task_info.steps_total,
+                        task_info=task_info,
+                    )
+                    self._rich_task_ids[task_id] = rich_task_id
+                    self._displayed_task_order.append(task_id)
+
+            # After finishing, prune completed tasks to respect max visible cap
+            self._prune_completed_tasks_if_needed()
 
     def get_task_info(self, task_id: int) -> TaskInfo | None:
         """Get additional task information."""
@@ -567,6 +593,54 @@ class MultiTaskStatus(AbstractAsyncContextManager):
         """Get console instance for additional output above progress."""
         return self._progress.console
 
+    def _prune_completed_tasks_if_needed(self) -> None:
+        """
+        Ensure at most `max_display_tasks` tasks are visible by removing the oldest
+        completed/skipped/failed tasks first. Running or waiting tasks are never
+        removed by this method.
+        Note: This method assumes it's called under self._lock.
+        """
+        max_visible = self.settings.max_display_tasks
+
+        # Nothing to prune or unlimited
+        if max_visible <= 0:
+            return
+
+        # Count visible tasks (those with a Rich task id present)
+        visible_task_ids = [tid for tid in self._displayed_task_order if tid in self._rich_task_ids]
+        excess = len(visible_task_ids) - max_visible
+        if excess <= 0:
+            return
+
+        # Build list of terminal tasks that can be pruned (oldest first)
+        terminal_tasks = []
+        for tid in self._displayed_task_order:
+            if tid not in self._rich_task_ids:
+                continue
+            info = self._task_info.get(tid)
+            if info and info.state in (
+                TaskState.COMPLETED,
+                TaskState.FAILED,
+                TaskState.SKIPPED,
+            ):
+                terminal_tasks.append(tid)
+
+        # Remove the oldest terminal tasks up to the excess count
+        tasks_to_remove = terminal_tasks[:excess]
+
+        for tid in tasks_to_remove:
+            rich_tid = self._rich_task_ids.pop(tid, None)
+            if rich_tid is not None:
+                # Remove from Rich progress display
+                self._progress.remove_task(rich_tid)
+            # Mark as pruned so we don't re-add on finish
+            self._pruned_task_ids.add(tid)
+
+        # Efficiently rebuild the displayed task order without the removed tasks
+        self._displayed_task_order = [
+            tid for tid in self._displayed_task_order if tid not in tasks_to_remove
+        ]
+
 
 ## Tests
 

kash/utils/text_handling/markdown_footnotes.py

@@ -1,48 +1,19 @@
 from __future__ import annotations
 
-import re
 from dataclasses import dataclass, field
 from typing import Any
 
-from flowmark import flowmark_markdown, line_wrap_by_sentence
 from marko import Markdown
+from marko.block import Document
 from marko.ext import footnote
 
-from kash.utils.text_handling.markdown_utils import comprehensive_transform_tree
-
-
-def _normalize_footnotes_in_markdown(content: str) -> str:
-    """
-    Ensure blank lines between consecutive footnote definitions.
-
-    Marko has a bug where consecutive footnotes without blank lines are parsed
-    as a single footnote. This adds blank lines where needed.
-    """
-    lines = content.split("\n")
-    result = []
-    i = 0
-
-    while i < len(lines):
-        line = lines[i]
-        result.append(line)
-
-        # Check if this is a footnote definition
-        if re.match(r"^\[\^[^\]]+\]:", line):
-            # Look ahead to see if the next non-empty line is also a footnote
-            j = i + 1
-            while j < len(lines) and not lines[j].strip():
-                result.append(lines[j])
-                j += 1
-
-            if j < len(lines) and re.match(r"^\[\^[^\]]+\]:", lines[j]):
-                # Next non-empty line is also a footnote, add blank line
-                result.append("")
-
-            i = j
-        else:
-            i += 1
-
-    return "\n".join(result)
+from kash.utils.text_handling.markdown_utils import (
+    MARKDOWN as DEFAULT_MARKDOWN,
+)
+from kash.utils.text_handling.markdown_utils import (
+    comprehensive_transform_tree,
+    normalize_footnotes_in_markdown,
+)
 
 
 @dataclass
@@ -81,15 +52,17 @@ class MarkdownFootnotes:
             MarkdownFootnotes instance with all footnotes indexed by ID
         """
         if markdown_parser is None:
-            markdown_parser = flowmark_markdown(line_wrap_by_sentence(is_markdown=True))
+            markdown_parser = DEFAULT_MARKDOWN
 
         # Normalize to work around marko bug with consecutive footnotes
-        normalized_content = _normalize_footnotes_in_markdown(content)
+        normalized_content = normalize_footnotes_in_markdown(content)
         document = markdown_parser.parse(normalized_content)
         return MarkdownFootnotes.from_document(document, markdown_parser)
 
     @staticmethod
-    def from_document(document: Any, markdown_parser: Markdown | None = None) -> MarkdownFootnotes:
+    def from_document(
+        document: Document, markdown_parser: Markdown | None = None
+    ) -> MarkdownFootnotes:
         """
         Extract all footnotes from a parsed markdown document.
 
@@ -102,7 +75,7 @@ class MarkdownFootnotes:
             MarkdownFootnotes instance with all footnotes indexed by ID
         """
         if markdown_parser is None:
-            markdown_parser = flowmark_markdown(line_wrap_by_sentence(is_markdown=True))
+            markdown_parser = DEFAULT_MARKDOWN
 
         footnotes_dict: dict[str, FootnoteInfo] = {}
 
@@ -206,9 +179,9 @@ def extract_footnote_references(content: str, markdown_parser: Markdown | None =
         List of unique footnote IDs that are referenced (with the ^)
     """
     if markdown_parser is None:
-        markdown_parser = flowmark_markdown(line_wrap_by_sentence(is_markdown=True))
+        markdown_parser = DEFAULT_MARKDOWN
 
-    normalized_content = _normalize_footnotes_in_markdown(content)
+    normalized_content = normalize_footnotes_in_markdown(content)
     document = markdown_parser.parse(normalized_content)
     references: list[str] = []
     seen: set[str] = set()