kash-shell 0.3.22__py3-none-any.whl → 0.3.24__py3-none-any.whl

kash/utils/api_utils/http_utils.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+
+def extract_http_status_code(exception: Exception) -> int | None:
+    """
+    Extract HTTP status code from various exception types.
+
+    Args:
+        exception: The exception to extract status code from
+
+    Returns:
+        HTTP status code or None if not found
+    """
+    # Check for httpx.HTTPStatusError and requests.HTTPError
+    if hasattr(exception, "response"):
+        response = getattr(exception, "response", None)
+        if response and hasattr(response, "status_code"):
+            return getattr(response, "status_code", None)
+
+    # Check for aiohttp errors
+    if hasattr(exception, "status"):
+        return getattr(exception, "status", None)
+
+    # Parse from exception message as fallback
+    exception_str = str(exception)
+
+    # Try to find status code patterns in the message
+    import re
+
+    # Pattern for "403 Forbidden", "HTTP 429", etc.
+    status_patterns = [
+        r"\b(\d{3})\s+(?:Forbidden|Unauthorized|Not Found|Too Many Requests|Internal Server Error|Bad Gateway|Service Unavailable|Gateway Timeout)\b",
+        r"\bHTTP\s+(\d{3})\b",
+        r"\b(\d{3})\s+error\b",
+        r"status\s*(?:code)?:\s*(\d{3})\b",
+    ]
+
+    for pattern in status_patterns:
+        match = re.search(pattern, exception_str, re.IGNORECASE)
+        if match:
+            try:
+                return int(match.group(1))
+            except (ValueError, IndexError):
+                continue
+
+    return None

kash/utils/api_utils/progress_protocol.py

@@ -6,7 +6,7 @@ from enum import Enum
 from typing import Any, Protocol, TypeAlias, TypeVar
 
 T = TypeVar("T")
-TaskID = TypeVar("TaskID")
+TaskId = TypeVar("TaskId")
 
 # Generic task spec types for labeler functions
 TaskSpec = TypeVar("TaskSpec")
@@ -16,14 +16,16 @@ Labeler: TypeAlias = Callable[[int, TaskSpec], str]
 EMOJI_SUCCESS = "[✔︎]"
 EMOJI_FAILURE = "[✘]"
 EMOJI_SKIP = "[-]"
-EMOJI_WARN = "[∆]"
-EMOJI_RETRY = "▵"
+EMOJI_WAITING = " ⧖ "
+EMOJI_WARN = "∆"
+EMOJI_RETRY = "⟲"
 
 
 class TaskState(Enum):
     """Task execution states."""
 
     QUEUED = "queued"
+    WAITING = "waiting"
     RUNNING = "running"
     COMPLETED = "completed"
     FAILED = "failed"
@@ -38,7 +40,7 @@ class TaskInfo:
     retry_count: int = 0
     failures: list[str] = field(default_factory=list)
     label: str = ""
-    total: int = 1
+    steps_total: int = 1
 
 
 @dataclass(frozen=True)
@@ -47,30 +49,9 @@ class TaskSummary:
 
     task_states: list[TaskState]
 
-    @property
-    def queued(self) -> int:
-        """Number of queued tasks."""
-        return sum(1 for state in self.task_states if state == TaskState.QUEUED)
-
-    @property
-    def running(self) -> int:
-        """Number of running tasks."""
-        return sum(1 for state in self.task_states if state == TaskState.RUNNING)
-
-    @property
-    def completed(self) -> int:
-        """Number of completed tasks."""
-        return sum(1 for state in self.task_states if state == TaskState.COMPLETED)
-
-    @property
-    def failed(self) -> int:
-        """Number of failed tasks."""
-        return sum(1 for state in self.task_states if state == TaskState.FAILED)
-
-    @property
-    def skipped(self) -> int:
-        """Number of skipped tasks."""
-        return sum(1 for state in self.task_states if state == TaskState.SKIPPED)
+    def count(self, state: TaskState) -> int:
+        """Count the number of tasks in a given state."""
+        return sum(1 for s in self.task_states if s == state)
 
     @property
     def total(self) -> int:
@@ -84,30 +65,35 @@ class TaskSummary:
         if not self.task_states:
             return "No tasks to process"
 
-        if self.completed == self.total:
-            return f"All tasks successful: {self.completed}/{self.total} completed"
-        elif self.completed + self.skipped == self.total:
-            return f"All tasks successful: {self.completed}/{self.total} completed, {self.skipped} skipped"
-        elif self.failed == self.total:
-            return f"All tasks failed: {self.failed}/{self.total} failed"
+        completed = self.count(TaskState.COMPLETED)
+        skipped = self.count(TaskState.SKIPPED)
+        failed = self.count(TaskState.FAILED)
+        queued = self.count(TaskState.QUEUED)
+
+        if completed == self.total:
+            return f"All tasks successful: {completed}/{self.total} completed"
+        elif completed + skipped == self.total:
+            return f"All tasks successful: {completed}/{self.total} completed, {skipped} skipped"
+        elif failed == self.total:
+            return f"All tasks failed: {failed}/{self.total} failed"
         else:
             parts = []
-            if self.completed > 0:
-                parts.append(f"{self.completed}/{self.total} tasks completed")
-            if self.failed > 0:
-                parts.append(f"{self.failed} tasks failed")
-            if self.skipped > 0:
-                parts.append(f"{self.skipped} tasks skipped")
-            if self.queued > 0:
-                parts.append(f"{self.queued} tasks not yet run")
-
-            if self.queued > 0:
+            if completed > 0:
+                parts.append(f"{completed}/{self.total} tasks completed")
+            if failed > 0:
+                parts.append(f"{failed} tasks failed")
+            if skipped > 0:
+                parts.append(f"{skipped} tasks skipped")
+            if queued > 0:
+                parts.append(f"{queued} tasks not yet run")
+
+            if queued > 0:
                 return "Tasks were interrupted: " + ", ".join(parts)
             else:
                 return "Tasks had errors: " + ", ".join(parts)
 
 
-class ProgressTracker(Protocol[TaskID]):
+class ProgressTracker(Protocol[TaskId]):
     """
     Protocol for progress tracking that gather_limited can depend on.
 
@@ -123,19 +109,20 @@ class ProgressTracker(Protocol[TaskID]):
         """
         ...
 
-    async def add(self, label: str, total: int = 1) -> TaskID:
+    async def add(self, label: str, steps_total: int = 1) -> TaskId:
         """Add a new task to track."""
         ...
 
-    async def start(self, task_id: TaskID) -> None:
+    async def start(self, task_id: TaskId) -> None:
         """Mark task as started (after rate limiting/queuing)."""
         ...
 
     async def update(
         self,
-        task_id: TaskID,
+        task_id: TaskId,
+        state: TaskState | None = None,
         *,
-        progress: int | None = None,
+        steps_done: int | None = None,
         label: str | None = None,
         error_msg: str | None = None,
     ) -> None:
@@ -144,7 +131,8 @@ class ProgressTracker(Protocol[TaskID]):
 
         Args:
             task_id: Task ID from add()
-            progress: Steps to advance (None = no change)
+            state: New task state (None = no change)
+            steps_done: Steps to advance (None = no change)
             label: New label (None = no change)
             error_msg: Error message to record as retry (None = no retry)
         """
@@ -152,7 +140,7 @@ class ProgressTracker(Protocol[TaskID]):
 
     async def finish(
         self,
-        task_id: TaskID,
+        task_id: TaskId,
         state: TaskState,
         message: str = "",
     ) -> None:
@@ -167,10 +155,10 @@ class ProgressTracker(Protocol[TaskID]):
         ...
 
 
-class AsyncProgressContext(Protocol[TaskID]):
+class AsyncProgressContext(Protocol[TaskId]):
     """Protocol for async context manager progress trackers."""
 
-    async def __aenter__(self) -> ProgressTracker[TaskID]:
+    async def __aenter__(self) -> ProgressTracker[TaskId]:
         """Start progress tracking."""
         ...
 
@@ -198,11 +186,11 @@ class SimpleProgressTracker:
         """Console-based tracker works with standard logging."""
         return False
 
-    async def add(self, label: str, total: int = 1) -> int:  # pyright: ignore[reportUnusedParameter]
+    async def add(self, label: str, steps_total: int = 1) -> int:  # pyright: ignore[reportUnusedParameter]
         task_id = self._next_id
         self._next_id += 1
 
-        self._tasks[task_id] = TaskInfo(label=label)
+        self._tasks[task_id] = TaskInfo(label=label, steps_total=steps_total)
 
         if self.verbose:
             self.print_fn(f"Queued: {label}")
@@ -223,8 +211,9 @@ class SimpleProgressTracker:
     async def update(
         self,
         task_id: int,
+        state: TaskState | None = None,
         *,
-        progress: int | None = None,  # pyright: ignore[reportUnusedParameter]
+        steps_done: int | None = None,  # pyright: ignore[reportUnusedParameter]
         label: str | None = None,
         error_msg: str | None = None,
     ) -> None:
@@ -232,6 +221,10 @@ class SimpleProgressTracker:
         if not task_info:
             return
 
+        # Update state if provided
+        if state is not None:
+            task_info.state = state
+
         # Update label if provided
         if label is not None:
             task_info.label = label

kash/utils/rich_custom/multitask_status.py

@@ -22,6 +22,7 @@ from kash.utils.api_utils.progress_protocol import (
     EMOJI_RETRY,
     EMOJI_SKIP,
     EMOJI_SUCCESS,
+    EMOJI_WAITING,
     TaskInfo,
     TaskState,
     TaskSummary,
@@ -30,7 +31,7 @@ from kash.utils.api_utils.progress_protocol import (
 T = TypeVar("T")
 
 # Spinner configuration
-SPINNER_NAME = "dots12"
+SPINNER_NAME = "dots8Bit"
 
 
 @dataclass(frozen=True)
@@ -45,6 +46,7 @@ class StatusStyles:
     failure_symbol: str = EMOJI_FAILURE
     skip_symbol: str = EMOJI_SKIP
     retry_symbol: str = EMOJI_RETRY
+    wait_symbol: str = EMOJI_WAITING
 
     # Status styles
     retry_style: str = "red"
@@ -52,6 +54,7 @@ class StatusStyles:
     failure_style: str = "red"
     skip_style: str = "yellow"
     running_style: str = "blue"
+    waiting_style: str = "yellow"
     error_style: str = "dim red"
 
     # Progress bar styles
@@ -114,12 +117,13 @@ class SpinnerStatusColumn(ProgressColumn):
         self.spinner: Spinner = Spinner(spinner_name)
         self.styles = styles
 
-        # Calculate fixed width for consistent column sizing
+        # Calculate fixed width for consistent column sizing, adding 2 for padding (space on each side)
         self.column_width: int = max(
-            _get_spinner_width(spinner_name),
+            _get_spinner_width(spinner_name) + 2,
             len(styles.success_symbol),
             len(styles.failure_symbol),
             len(styles.skip_symbol),
+            len(styles.wait_symbol),
         )
 
     @override
@@ -136,13 +140,18 @@ class SpinnerStatusColumn(ProgressColumn):
             text = Text(self.styles.failure_symbol, style=self.styles.failure_style)
         elif task_info.state == TaskState.SKIPPED:
             text = Text(self.styles.skip_symbol, style=self.styles.skip_style)
-        else:
-            # Running: show spinner
+        elif task_info.state == TaskState.WAITING:
+            text = Text(self.styles.wait_symbol, style=self.styles.waiting_style)
+        elif task_info.state == TaskState.RUNNING:
+            # Running: show spinner with padding
             spinner_result = self.spinner.render(task.get_time())
             if isinstance(spinner_result, Text):
-                text = spinner_result
+                text = Text(" ") + spinner_result + Text(" ")
             else:
-                text = Text(str(spinner_result))
+                text = Text(" " + str(spinner_result) + " ")
+        else:
+            # Should not happen, but return empty space
+            return Text(" " * self.column_width)
 
         # Ensure consistent width
         current_len = len(text.plain)
@@ -376,13 +385,13 @@ class MultiTaskStatus(AbstractAsyncContextManager):
             summary = self.get_summary()
             self.console.print(summary)
 
-    async def add(self, label: str, total: int | None = None) -> int:
+    async def add(self, label: str, steps_total: int | None = None) -> int:
         """
         Add a new task to the display. Task won't appear until start() is called.
 
         Args:
             label: Human-readable task description
-            total: Total steps for progress bar (None for no default bar)
+            steps_total: Total steps for progress bar (None for no default bar)
 
         Returns:
             Task ID for subsequent updates
@@ -392,7 +401,7 @@ class MultiTaskStatus(AbstractAsyncContextManager):
             task_id: int = self._next_id
             self._next_id += 1
 
-            task_info = TaskInfo(label=label, total=total or 1)
+            task_info = TaskInfo(label=label, steps_total=steps_total or 1)
             self._task_info[task_id] = task_info
             return task_id
 
@@ -413,7 +422,7 @@ class MultiTaskStatus(AbstractAsyncContextManager):
             # Now add to Rich Progress display
             rich_task_id = self._progress.add_task(
                 "",
-                total=task_info.total,
+                total=task_info.steps_total,
                 label=task_info.label,
                 task_info=task_info,
                 progress_display=None,
@@ -440,8 +449,9 @@ class MultiTaskStatus(AbstractAsyncContextManager):
     async def update(
         self,
         task_id: int,
+        state: TaskState | None = None,
         *,
-        progress: int | None = None,
+        steps_done: int | None = None,
         label: str | None = None,
         error_msg: str | None = None,
     ) -> None:
@@ -450,7 +460,8 @@ class MultiTaskStatus(AbstractAsyncContextManager):
 
         Args:
             task_id: Task ID from add()
-            progress: Steps to advance (None = no change)
+            state: New task state (None = no change)
+            steps_done: Steps to advance (None = no change)
             label: New label (None = no change)
             error_msg: Error message to record as retry (None = no retry)
         """
@@ -461,6 +472,12 @@ class MultiTaskStatus(AbstractAsyncContextManager):
             task_info = self._task_info[task_id]
             rich_task_id = self._rich_task_ids.get(task_id)
 
+            # Update state if provided
+            if state is not None:
+                task_info.state = state
+                if rich_task_id is not None:
+                    self._progress.update(rich_task_id, task_info=task_info)
+
             # Update label if provided
             if label is not None:
                 task_info.label = label
@@ -468,8 +485,8 @@ class MultiTaskStatus(AbstractAsyncContextManager):
                     self._progress.update(rich_task_id, label=label, task_info=task_info)
 
             # Advance progress if provided
-            if progress is not None and rich_task_id is not None:
-                self._progress.advance(rich_task_id, advance=progress)
+            if steps_done is not None and rich_task_id is not None:
+                self._progress.advance(rich_task_id, advance=steps_done)
 
             # Record retry if error message provided
             if error_msg is not None:
@@ -511,9 +528,9 @@ class MultiTaskStatus(AbstractAsyncContextManager):
                 # Task was never started, but we still need to add it to show completion
                 rich_task_id = self._progress.add_task(
                     "",
-                    total=task_info.total,
+                    total=task_info.steps_total,
                     label=task_info.label,
-                    completed=task_info.total,
+                    completed=task_info.steps_total,
                     task_info=task_info,
                 )
                 self._rich_task_ids[task_id] = rich_task_id
@@ -573,9 +590,9 @@ def test_task_status_with_progress():
             settings=StatusSettings(show_progress=True),
         ) as status:
             # Traditional progress bar
-            download_task = await status.add("Downloading", total=100)
+            download_task = await status.add("Downloading", steps_total=100)
             for i in range(0, 101, 10):
-                await status.update(download_task, progress=10)
+                await status.update(download_task, steps_done=10)
                 await asyncio.sleep(0.1)
             await status.finish(download_task, TaskState.COMPLETED)
 
@@ -608,14 +625,14 @@ def test_task_status_mixed():
             settings=StatusSettings(show_progress=True, transient=True),
         ) as status:
             # Multiple concurrent tasks
-            install_task = await status.add("Installing packages", total=50)
+            install_task = await status.add("Installing packages", steps_total=50)
             test_task = await status.add("Running tests")
             build_task = await status.add("Building project")
             optional_task = await status.add("Optional feature")
 
             # Simulate concurrent work
             for i in range(5):
-                await status.update(install_task, progress=10)
+                await status.update(install_task, steps_done=10)
                 await status.set_progress_display(test_task, f"Test {i + 1}/10")
                 await status.set_progress_display(build_task, Text(f"Step {i + 1}", style="blue"))
                 await asyncio.sleep(0.2)
@@ -629,3 +646,35 @@ def test_task_status_mixed():
             await status.finish(optional_task, TaskState.SKIPPED, "Feature disabled in config")
 
     asyncio.run(_test_impl())
+
+
+def test_task_status_retry_states():
+    """Test TaskStatus with retry wait states."""
+    print("Testing TaskStatus with retry wait states...")
+
+    async def _test_impl():
+        async with MultiTaskStatus(
+            settings=StatusSettings(show_progress=False, transient=True),
+        ) as status:
+            # Task that will demonstrate retry wait state
+            retry_task = await status.add("API call with retries")
+            await status.start(retry_task)
+
+            # Simulate retry cycle
+            await status.update(retry_task, error_msg="Connection timeout", state=TaskState.WAITING)
+            await asyncio.sleep(1.0)  # Simulate backoff
+
+            await status.update(retry_task, state=TaskState.RUNNING)
+            await asyncio.sleep(0.5)  # Simulate execution
+
+            await status.update(
+                retry_task, error_msg="Rate limit exceeded", state=TaskState.WAITING
+            )
+            await asyncio.sleep(1.0)  # Simulate longer backoff
+
+            await status.update(retry_task, state=TaskState.RUNNING)
+            await asyncio.sleep(0.5)  # Simulate final execution
+
+            await status.finish(retry_task, TaskState.COMPLETED)
+
+    asyncio.run(_test_impl())

kash/utils/text_handling/markdown_utils.py

@@ -78,13 +78,23 @@ def _tree_links(element, include_internal=False):
 
 def extract_links(content: str, include_internal=False) -> list[str]:
     """
-    Extract all links from Markdown content.
+    Extract all links from Markdown content. Deduplicates and
+    preserves order.
 
     Raises:
         marko.ParseError: If the markdown content contains invalid syntax that cannot be parsed.
     """
     document = marko.parse(content)
-    return _tree_links(document, include_internal)
+    all_links = _tree_links(document, include_internal)
+
+    # Deduplicate while preserving order
+    seen: dict[str, None] = {}
+    result = []
+    for link in all_links:
+        if link not in seen:
+            seen[link] = None
+            result.append(link)
+    return result
 
 
 def extract_file_links(file_path: Path, include_internal=False) -> list[str]:
@@ -813,11 +823,12 @@ def test_extract_reference_style_links() -> None:
     assert len(result) == 2
 
 
-def test_extract_links_with_internal_fragments() -> None:
+def test_extract_links_and_dups() -> None:
     """Test that internal fragment links are excluded by default but included when requested."""
     content = dedent("""
         See [this section](#introduction) and [external link](https://example.com).
         Also check [another section](#conclusion) here.
+        Adding a [duplicate](https://example.com).
         """)
 
     # Default behavior: exclude internal links

kash/web_content/web_extract.py

@@ -8,7 +8,7 @@ from kash.web_content.web_extract_justext import extract_text_justext
 from kash.web_content.web_page_model import PageExtractor, WebPageData
 
 
-@log_calls(level="message")
+@log_calls(level="info")
 def fetch_page_content(
     url: Url,
     *,
@@ -17,19 +17,22 @@ def fetch_page_content(
     text_extractor: PageExtractor = extract_text_justext,
 ) -> WebPageData:
     """
-    Fetches a URL and extracts the title, description, and content.
-    Always uses the content cache, at least temporarily.
+    Fetches a URL and extracts the title, description, and content,
+    with optional caching.
+
+    Always uses the content cache for fetching. Cached file path is
+    returned in the content, unless `cache` is false, in which case
+    the cached content is deleted.
 
     Force re-fetching and updating the cache by setting `refetch` to true.
-    Cached file path is returned in the content, unless `cache` is false,
-    in case the cached content is deleted.
+
 
     For HTML and other text files, uses the `text_extractor` to extract
     clean text and page metadata.
     """
     expiration_sec = 0 if refetch else None
-
-    path = cache_file(url, expiration_sec=expiration_sec).content.path
+    cache_result = cache_file(url, expiration_sec=expiration_sec)
+    path = cache_result.content.path
     format_info = file_format_info(path)
 
     content = None
@@ -40,13 +43,14 @@ def fetch_page_content(
         page_data = WebPageData(url)
 
     # Add file format info (for both HTML/text and all other file types).
-
     page_data.format_info = format_info
 
     # Add a thumbnail, if known for this URL.
     page_data.thumbnail_url = thumbnail_url(url)
 
-    # Return the local cache path if we will be keeping it.
+    # Return whether this is from cache and the local cache path
+    # if we will be keeping it.
+    page_data.cache_result = cache_result
     if cache:
         page_data.saved_content = path
     else: