kash-shell 0.3.9__py3-none-any.whl → 0.3.10__py3-none-any.whl

kash/mcp/mcp_server_commands.py

@@ -3,8 +3,7 @@ from pathlib import Path
 
 from kash.config.logger import get_logger
 from kash.config.settings import (
-    get_system_logs_dir,
-    local_server_log_path,
+    global_settings,
 )
 from kash.exec import kash_command
 from kash.mcp import mcp_server_routes
@@ -58,13 +57,14 @@ def mcp_logs(follow: bool = False, all: bool = False) -> None:
     :param follow: Follow the file as it grows.
     :param all: Show all logs, not just the server logs, including Claude Desktop logs if found.
     """
+    settings = global_settings()
     if all:
-        global_log_base = get_system_logs_dir()
+        global_log_base = settings.system_logs_dir
         claude_log_base = Path("~/Library/Logs/Claude").expanduser()
         log_paths = []
         did_log = False
         while len(log_paths) == 0:
-            log_paths = [local_server_log_path(), MCP_CLI_LOG_PATH]
+            log_paths = [settings.local_server_log_path, MCP_CLI_LOG_PATH]
             claude_logs = list(claude_log_base.glob("mcp*.log"))
             if claude_logs:
                 log.message("Found Claude Desktop logs, will also tail them: %s", claude_logs)
@@ -81,7 +81,7 @@ def mcp_logs(follow: bool = False, all: bool = False) -> None:
                     did_log = True
                 time.sleep(1)
     else:
-        server_log_path = local_server_log_path()  # MCP logs shared with local server logs.
+        server_log_path = settings.local_server_log_path  # MCP logs shared with local server logs.
         if not server_log_path.exists():
             raise InvalidState(
                 f"MCP server log not found (forgot to run `start_mcp_server`?): {server_log_path}"

kash/mcp/mcp_server_routes.py

@@ -8,16 +8,16 @@ from funlog import log_calls
 from mcp.server.lowlevel import Server
 from mcp.types import Prompt, Resource, TextContent, Tool
 from prettyfmt import fmt_path
+from strif import AtomicVar
 
 from kash.config.capture_output import CapturedOutput, captured_output
 from kash.config.logger import get_logger
-from kash.config.settings import get_mcp_ws_dir
+from kash.config.settings import global_settings
 from kash.exec.action_exec import prepare_action_input, run_action_with_caching
 from kash.exec.action_registry import get_all_actions_defaults, look_up_action_class
 from kash.model.actions_model import Action, ActionResult, ExecContext
 from kash.model.params_model import TypedParamValues
 from kash.model.paths_model import StorePath
-from kash.utils.common.atomic_var import AtomicVar
 from kash.workspaces.workspaces import current_ws, get_ws
 
 log = get_logger(__name__)
@@ -213,7 +213,7 @@ def run_mcp_tool(action_name: str, arguments: dict) -> list[TextContent]:
             # XXX For now, unless the user has overridden the MCP workspace, we use the
             # current workspace, which could be changed by the user by changing working
             # directories. Maybe confusing?
-            explicit_mcp_ws = get_mcp_ws_dir()
+            explicit_mcp_ws = global_settings().mcp_ws_dir
             ws = get_ws(explicit_mcp_ws) if explicit_mcp_ws else current_ws()
 
             with ws:
@@ -231,8 +231,8 @@ def run_mcp_tool(action_name: str, arguments: dict) -> list[TextContent]:
                 context = ExecContext(
                     action=action,
                     workspace_dir=ws.base_dir,
-                    # Enabling rerun always for now, seems good for tools.
-                    rerun=True,
+                    rerun=True,  # Enabling rerun always for now, seems good for tools.
+                    refetch=False,  # Using the file caches.
                     # Keeping all transient files for now, but maybe make transient?
                     override_state=None,
                 )

kash/mcp/mcp_server_sse.py

@@ -18,7 +18,7 @@ if TYPE_CHECKING:
 
 from kash.config.logger import get_logger
 from kash.config.server_config import create_server_config
-from kash.config.settings import global_settings, local_server_log_path
+from kash.config.settings import global_settings
 from kash.local_server.port_tools import find_available_local_port
 from kash.mcp import mcp_server_routes
 from kash.utils.errors import InvalidState
@@ -154,7 +154,9 @@ class MCPServerSSE:
 
 
 # Singleton instance
-_mcp_sse_server = MCPServerSSE(MCP_SERVER_NAME, MCP_SERVER_HOST, local_server_log_path())
+_mcp_sse_server = MCPServerSSE(
+    MCP_SERVER_NAME, MCP_SERVER_HOST, global_settings().local_server_log_path
+)
 
 
 def start_mcp_server_sse():

kash/media_base/media_cache.py

@@ -11,7 +11,7 @@ from kash.media_base.media_services import (
     download_media_by_service,
     get_media_services,
 )
-from kash.media_base.speech_transcription import deepgram_transcribe_audio
+from kash.media_base.transcription_deepgram import deepgram_transcribe_audio
 from kash.utils.common.format_utils import fmt_loc
 from kash.utils.common.url import Url, as_file_url, is_url
 from kash.utils.errors import FileNotFound, InvalidInput, UnexpectedError
@@ -88,19 +88,19 @@ class MediaCache(DirStore):
         return transcript
 
     def cache(
-        self, url: Url, no_cache=False, media_types: list[MediaType] | None = None
+        self, url: Url, refetch=False, media_types: list[MediaType] | None = None
     ) -> dict[MediaType, Path]:
         """
         Cache the media files for the given media URL. Returns paths to cached copies
         for each media type (video or audio). Returns cached copies if available,
-        unless `no_cache` is True.
+        unless `refetch` is True.
         """
         cached_paths: dict[MediaType, Path] = {}
 
         if not media_types:
             media_types = [MediaType.audio, MediaType.video]
 
-        if not no_cache:
+        if not refetch:
             if MediaType.audio in media_types:
                 audio_file = self.find(url, suffix=SUFFIX_MP3)
                 if audio_file:
@@ -141,11 +141,11 @@ class MediaCache(DirStore):
         return cached_paths
 
     def transcribe(
-        self, url_or_path: Url | Path, no_cache=False, language: str | None = None
+        self, url_or_path: Url | Path, refetch=False, language: str | None = None
     ) -> str:
         """
         Transcribe the audio file, caching audio, downsampled audio, and the transcription.
-        Return the cached transcript if available, unless `no_cache` is True.
+        Return the cached transcript if available, unless `refetch` is True.
         """
         if not isinstance(url_or_path, Path) and is_url(url_or_path):
             # If it is a URL, cache it locally.
@@ -156,12 +156,12 @@ class MediaCache(DirStore):
                 raise InvalidInput(
                     "Unrecognized media URL (is this media service configured?): %s" % url_or_path
                 )
-            if not no_cache:
+            if not refetch:
                 transcript = self._read_transcript(url)
                 if transcript:
                     return transcript
             # Cache all formats since we usually will want them.
-            self.cache(url, no_cache)
+            self.cache(url, refetch)
         elif isinstance(url_or_path, Path):
             # Treat local media files as file:// URLs.
             # Don't need to cache originals but we still will cache audio and transcriptions.

kash/media_base/media_services.py

@@ -2,10 +2,10 @@ import logging
 from pathlib import Path
 
 from funlog import log_calls
+from strif import AtomicVar
 
 from kash.media_base.services.local_file_media import LocalFileMedia
 from kash.model.media_model import MediaMetadata, MediaService
-from kash.utils.common.atomic_var import AtomicVar
 from kash.utils.common.url import Url
 from kash.utils.errors import InvalidInput
 from kash.utils.file_utils.file_formats_model import MediaType

kash/media_base/media_tools.py

@@ -28,20 +28,20 @@ def reset_media_cache_dir(path: Path):
 
 
 def cache_and_transcribe(
-    url_or_path: Url | Path, no_cache=False, language: str | None = None
+    url_or_path: Url | Path, refetch=False, language: str | None = None
 ) -> str:
     """
-    Download and transcribe audio or video, saving in cache. If no_cache is
+    Download and transcribe audio or video, saving in cache. If `refetch` is
     True, force fresh download.
     """
-    return _media_cache.transcribe(url_or_path, no_cache=no_cache, language=language)
+    return _media_cache.transcribe(url_or_path, refetch=refetch, language=language)
 
 
 def cache_media(
-    url: Url, no_cache=False, media_types: list[MediaType] | None = None
+    url: Url, refetch=False, media_types: list[MediaType] | None = None
 ) -> dict[MediaType, Path]:
     """
-    Download audio and video (if available), saving in cache. If no_cache is
+    Download audio and video (if available), saving in cache. If refetch is
     True, force fresh download.
     """
-    return _media_cache.cache(url, no_cache, media_types)
+    return _media_cache.cache(url, refetch, media_types)

kash/media_base/services/local_file_media.py

@@ -4,13 +4,13 @@ import subprocess  # Add this import
 from pathlib import Path
 from urllib.parse import urlparse
 
+from clideps.pkgs.pkg_check import pkg_check
 from strif import copyfile_atomic
 from typing_extensions import override
 
 from kash.config.logger import get_log_file_stream, get_logger
 from kash.file_storage.store_filenames import parse_item_filename
 from kash.model.media_model import MediaMetadata, MediaService, MediaUrlType
-from kash.shell.clideps.pkg_deps import Pkg, pkg_check
 from kash.utils.common.format_utils import fmt_loc
 from kash.utils.common.url import Url
 from kash.utils.errors import FileNotFound, InvalidInput
@@ -20,7 +20,7 @@ log = get_logger(__name__)
 
 
 def _run_ffmpeg(cmdline: list[str]) -> None:
-    pkg_check().require(Pkg.ffmpeg)
+    pkg_check().require("ffmpeg")
     log.message("Running: %s", " ".join([shlex.quote(arg) for arg in cmdline]))
     subprocess.run(
         cmdline,

kash/media_base/{speech_transcription.py → transcription_deepgram.py}

@@ -1,61 +1,23 @@
 from os.path import getsize
 from pathlib import Path
-from typing import NamedTuple
 
+from clideps.env_vars.dotenv_utils import load_dotenv_paths
+from deepgram import ListenRESTClient, PrerecordedResponse
 from httpx import Timeout
-from openai import OpenAI
 
 from kash.config.logger import CustomLogger, get_logger
-from kash.config.settings import get_system_config_dir
-from kash.media_base.timestamp_citations import html_speaker_id_span, html_timestamp_span
-from kash.shell.clideps.dotenv_utils import load_dotenv_paths
-from kash.utils.errors import ContentError
+from kash.config.settings import global_settings
+from kash.media_base.transcription_format import SpeakerSegment, format_speaker_segments
+from kash.utils.errors import ApiError, ContentError
 
 log: CustomLogger = get_logger(__name__)
 
 
-def openai_whisper_transcribe_audio_small(audio_file_path: str) -> str:
+def deepgram_transcribe_raw(
+    audio_file_path: Path, language: str | None = None
+) -> PrerecordedResponse:
     """
-    Transcribe an audio file. Whisper is very good quality but (as of 2024-05)
-    OpenAI's version does not support diarization and must be under 25MB.
-
-    https://help.openai.com/en/articles/7031512-whisper-api-faq
-    """
-    WHISPER_MAX_SIZE = 25 * 1024 * 1024
-
-    size = getsize(audio_file_path)
-    if size > WHISPER_MAX_SIZE:
-        raise ValueError("Audio file too large for Whisper (%s > %s)" % (size, WHISPER_MAX_SIZE))
-    log.info(
-        "Transcribing via Whisper: %s (size %s)",
-        audio_file_path,
-        size,
-    )
-
-    client = OpenAI()
-    with open(audio_file_path, "rb") as audio_file:
-        transcription = client.audio.transcriptions.create(
-            model="whisper-1",
-            file=audio_file,
-            # For when we want timestamps:
-            # response_format="verbose_json",
-            # timestamp_granularities=["word"]
-        )
-        text = transcription.text
-    return text
-
-
-class SpeakerSegment(NamedTuple):
-    words: list[tuple[float, str]]
-    start: float
-    end: float
-    speaker: int
-    average_confidence: float
-
-
-def deepgram_transcribe_audio(audio_file_path: Path, language: str | None = None) -> str:
-    """
-    Transcribe an audio file using Deepgram.
+    Transcribe an audio file using Deepgram and return the raw response.
     """
     from deepgram import ClientOptionsFromEnv, DeepgramClient, FileSource, PrerecordedOptions
 
@@ -64,7 +26,7 @@ def deepgram_transcribe_audio(audio_file_path: Path, language: str | None = None
         "Transcribing via Deepgram (language %r): %s (size %s)", language, audio_file_path, size
     )
 
-    load_dotenv_paths(True, True, get_system_config_dir())
+    load_dotenv_paths(True, True, global_settings().system_config_dir)
     deepgram = DeepgramClient("", ClientOptionsFromEnv())
 
     with open(audio_file_path, "rb") as audio_file:
@@ -75,7 +37,17 @@ def deepgram_transcribe_audio(audio_file_path: Path, language: str | None = None
     }
 
     options = PrerecordedOptions(model="nova-2", smart_format=True, diarize=True, language=language)
-    response = deepgram.listen.rest.v("1").transcribe_file(payload, options, timeout=Timeout(500))  # pyright: ignore
+    client: ListenRESTClient = deepgram.listen.rest.v("1")  # pyright: ignore
+
+    response = client.transcribe_file(payload, options, timeout=Timeout(500))
+    if not isinstance(response, PrerecordedResponse):
+        raise ApiError("Deepgram returned an unexpected response type")
+
+    return response
+
+
+def deepgram_transcribe_audio(audio_file_path: Path, language: str | None = None) -> str:
+    response = deepgram_transcribe_raw(audio_file_path, language)
 
     log.save_object("Deepgram response", None, response)
 
@@ -87,13 +59,15 @@ def deepgram_transcribe_audio(audio_file_path: Path, language: str | None = None
             f"No speaker segments found in Deepgram response (are voices silent or missing?): {audio_file_path}"
         )
 
-    formatted_segments = format_speaker_segments(diarized_segments)
+    formatted_segments = format_speaker_segments(diarized_segments)  # noqa: F821
 
     return formatted_segments
 
 
 def _deepgram_diarized_segments(data, confidence_threshold=0.3) -> list[SpeakerSegment]:
-    """Process Deepgram diarized results into text segments per speaker."""
+    """
+    Process Deepgram diarized results into text segments per speaker.
+    """
 
     speaker_segments: list[SpeakerSegment] = []
     current_speaker = 0
@@ -164,62 +138,3 @@ def _deepgram_diarized_segments(data, confidence_threshold=0.3) -> list[SpeakerS
         )
 
     return speaker_segments
-
-
-def _is_new_sentence(word: str, next_word: str | None) -> bool:
-    return (
-        (word.endswith(".") or word.endswith("?") or word.endswith("!"))
-        and next_word is not None
-        and next_word[0].isupper()
-    )
-
-
-def _format_words(words: list[tuple[float, str]], include_sentence_timestamps=True) -> str:
-    """Format words with timestamps added in spans."""
-
-    if not words:
-        return ""
-
-    sentences = []
-    current_sentence = []
-    for i, (timestamp, word) in enumerate(words):
-        current_sentence.append(word)
-        next_word = words[i + 1][1] if i + 1 < len(words) else None
-        if _is_new_sentence(word, next_word):
-            sentences.append((timestamp, current_sentence))
-            current_sentence = []
-
-    if current_sentence:
-        sentences.append((words[-1][0], current_sentence))
-
-    formatted_text = []
-    for timestamp, sentence in sentences:
-        formatted_sentence = " ".join(sentence)
-        if include_sentence_timestamps:
-            formatted_text.append(html_timestamp_span(formatted_sentence, timestamp))
-        else:
-            formatted_text.append(formatted_sentence)
-
-    return "\n".join(formatted_text)
-
-
-def format_speaker_segments(speaker_segments: list[SpeakerSegment]) -> str:
-    """
-    Format speaker segments in a simple HTML format with <span> tags including speaker
-    ids and timestamps.
-    """
-
-    # Use \n\n for readability between segments so each speaker is its own
-    # paragraph.
-    SEGMENT_SEP = "\n\n"
-
-    speakers = set(segment.speaker for segment in speaker_segments)
-    if len(speakers) > 1:
-        lines = []
-        for segment in speaker_segments:
-            lines.append(
-                f"{html_speaker_id_span(f'SPEAKER {segment.speaker}:', str(segment.speaker))}\n{_format_words(segment.words)}"
-            )
-        return SEGMENT_SEP.join(lines)
-    else:
-        return SEGMENT_SEP.join(_format_words(segment.words) for segment in speaker_segments)

kash/media_base/transcription_format.py

@@ -0,0 +1,73 @@
+from typing import NamedTuple
+
+from kash.config.logger import CustomLogger, get_logger
+from kash.media_base.timestamp_citations import html_speaker_id_span, html_timestamp_span
+
+log: CustomLogger = get_logger(__name__)
+
+
+def _is_new_sentence(word: str, next_word: str | None) -> bool:
+    return (
+        (word.endswith(".") or word.endswith("?") or word.endswith("!"))
+        and next_word is not None
+        and next_word[0].isupper()
+    )
+
+
+def _format_words(words: list[tuple[float, str]], include_sentence_timestamps=True) -> str:
+    """Format words with timestamps added in spans."""
+
+    if not words:
+        return ""
+
+    sentences = []
+    current_sentence = []
+    for i, (timestamp, word) in enumerate(words):
+        current_sentence.append(word)
+        next_word = words[i + 1][1] if i + 1 < len(words) else None
+        if _is_new_sentence(word, next_word):
+            sentences.append((timestamp, current_sentence))
+            current_sentence = []
+
+    if current_sentence:
+        sentences.append((words[-1][0], current_sentence))
+
+    formatted_text = []
+    for timestamp, sentence in sentences:
+        formatted_sentence = " ".join(sentence)
+        if include_sentence_timestamps:
+            formatted_text.append(html_timestamp_span(formatted_sentence, timestamp))
+        else:
+            formatted_text.append(formatted_sentence)
+
+    return "\n".join(formatted_text)
+
+
+class SpeakerSegment(NamedTuple):
+    words: list[tuple[float, str]]
+    start: float
+    end: float
+    speaker: int
+    average_confidence: float
+
+
+def format_speaker_segments(speaker_segments: list[SpeakerSegment]) -> str:
+    """
+    Format speaker segments in a simple HTML format with <span> tags including speaker
+    ids and timestamps.
+    """
+
+    # Use \n\n for readability between segments so each speaker is its own
+    # paragraph.
+    SEGMENT_SEP = "\n\n"
+
+    speakers = set(segment.speaker for segment in speaker_segments)
+    if len(speakers) > 1:
+        lines = []
+        for segment in speaker_segments:
+            lines.append(
+                f"{html_speaker_id_span(f'SPEAKER {segment.speaker}:', str(segment.speaker))}\n{_format_words(segment.words)}"
+            )
+        return SEGMENT_SEP.join(lines)
+    else:
+        return SEGMENT_SEP.join(_format_words(segment.words) for segment in speaker_segments)

kash/media_base/transcription_whisper.py

@@ -0,0 +1,38 @@
+from os.path import getsize
+
+from openai import OpenAI
+
+from kash.config.logger import CustomLogger, get_logger
+
+log: CustomLogger = get_logger(__name__)
+
+
+def openai_whisper_transcribe_audio_small(audio_file_path: str) -> str:
+    """
+    Transcribe an audio file. Whisper is very good quality but (as of 2024-05)
+    OpenAI's version does not support diarization and must be under 25MB.
+
+    https://help.openai.com/en/articles/7031512-whisper-api-faq
+    """
+    WHISPER_MAX_SIZE = 25 * 1024 * 1024
+
+    size = getsize(audio_file_path)
+    if size > WHISPER_MAX_SIZE:
+        raise ValueError("Audio file too large for Whisper (%s > %s)" % (size, WHISPER_MAX_SIZE))
+    log.info(
+        "Transcribing via Whisper: %s (size %s)",
+        audio_file_path,
+        size,
+    )
+
+    client = OpenAI()
+    with open(audio_file_path, "rb") as audio_file:
+        transcription = client.audio.transcriptions.create(
+            model="whisper-1",
+            file=audio_file,
+            # For when we want timestamps:
+            # response_format="verbose_json",
+            # timestamp_granularities=["word"]
+        )
+        text = transcription.text
+    return text

kash/model/__init__.py

@@ -1,11 +1,7 @@
 """
 The core classes for modeling kash's framework.
-
-We include essential logic here but try to keep logic and dependencies minimal.
 """
 
-# flake8: noqa: F401
-
 from kash.exec_model.args_model import (
     ANY_ARGS,
     NO_ARGS,
@@ -31,7 +27,12 @@ from kash.model.actions_model import (
     PerItemAction,
     TitleTemplate,
 )
-from kash.model.compound_actions_model import ComboAction, SequenceAction, look_up_actions
+from kash.model.compound_actions_model import (
+    ComboAction,
+    SequenceAction,
+    look_up_actions,
+)
+from kash.model.concept_model import Concept, canonicalize_concept, normalize_concepts
 from kash.model.graph_model import GraphData, Link, Node
 from kash.model.items_model import (
     SLUG_MAX_LEN,
@@ -69,3 +70,70 @@ from kash.model.paths_model import StorePath
 from kash.model.preconditions_model import Precondition
 from kash.utils.common.format_utils import fmt_loc
 from kash.utils.file_utils.file_formats_model import FileExt, Format, MediaType
+
+__all__ = [
+    "ANY_ARGS",
+    "NO_ARGS",
+    "ONE_ARG",
+    "ONE_OR_MORE_ARGS",
+    "ONE_OR_NO_ARGS",
+    "TWO_ARGS",
+    "TWO_OR_MORE_ARGS",
+    "ArgCount",
+    "CommandArg",
+    "Command",
+    "CommentedCommand",
+    "BareComment",
+    "Script",
+    "ShellResult",
+    "Action",
+    "ActionInput",
+    "ActionResult",
+    "ExecContext",
+    "LLMOptions",
+    "PathOp",
+    "PathOpType",
+    "PerItemAction",
+    "TitleTemplate",
+    "ComboAction",
+    "SequenceAction",
+    "look_up_actions",
+    "Concept",
+    "canonicalize_concept",
+    "normalize_concepts",
+    "GraphData",
+    "Link",
+    "Node",
+    "SLUG_MAX_LEN",
+    "UNTITLED",
+    "IdType",
+    "Item",
+    "ItemId",
+    "ItemRelations",
+    "ItemType",
+    "State",
+    "SERVICE_APPLE_PODCASTS",
+    "SERVICE_VIMEO",
+    "SERVICE_YOUTUBE",
+    "HeatmapValue",
+    "MediaMetadata",
+    "MediaService",
+    "MediaUrlType",
+    "ALL_COMMON_PARAMS",
+    "COMMON_ACTION_PARAMS",
+    "GLOBAL_PARAMS",
+    "RUNTIME_ACTION_PARAMS",
+    "USER_SETTABLE_PARAMS",
+    "Param",
+    "ParamDeclarations",
+    "RawParamValues",
+    "TypedParamValues",
+    "common_param",
+    "common_params",
+    "StorePath",
+    "Precondition",
+    "fmt_loc",
+    "FileExt",
+    "Format",
+    "MediaType",
+]