kash-shell 0.3.18__py3-none-any.whl → 0.3.21__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. kash/actions/core/{markdownify.py → markdownify_html.py} +3 -6
  2. kash/commands/workspace/workspace_commands.py +10 -88
  3. kash/config/colors.py +8 -6
  4. kash/config/text_styles.py +2 -0
  5. kash/docs/markdown/topics/a1_what_is_kash.md +1 -1
  6. kash/docs/markdown/topics/b1_kash_overview.md +34 -45
  7. kash/exec/__init__.py +3 -0
  8. kash/exec/action_decorators.py +20 -5
  9. kash/exec/action_exec.py +2 -2
  10. kash/exec/{fetch_url_metadata.py → fetch_url_items.py} +42 -14
  11. kash/exec/llm_transforms.py +1 -1
  12. kash/exec/shell_callable_action.py +1 -1
  13. kash/file_storage/file_store.py +7 -1
  14. kash/file_storage/store_filenames.py +4 -0
  15. kash/help/function_param_info.py +1 -1
  16. kash/help/help_pages.py +1 -1
  17. kash/help/help_printing.py +1 -1
  18. kash/llm_utils/llm_completion.py +1 -1
  19. kash/model/actions_model.py +6 -0
  20. kash/model/items_model.py +18 -3
  21. kash/shell/output/shell_output.py +15 -0
  22. kash/utils/api_utils/api_retries.py +305 -0
  23. kash/utils/api_utils/cache_requests_limited.py +84 -0
  24. kash/utils/api_utils/gather_limited.py +987 -0
  25. kash/utils/api_utils/progress_protocol.py +299 -0
  26. kash/utils/common/function_inspect.py +66 -1
  27. kash/utils/common/parse_docstring.py +347 -0
  28. kash/utils/common/testing.py +10 -7
  29. kash/utils/rich_custom/multitask_status.py +631 -0
  30. kash/utils/text_handling/escape_html_tags.py +16 -11
  31. kash/utils/text_handling/markdown_render.py +1 -0
  32. kash/web_content/web_extract.py +34 -15
  33. kash/web_content/web_page_model.py +10 -1
  34. kash/web_gen/templates/base_styles.css.jinja +26 -20
  35. kash/web_gen/templates/components/toc_styles.css.jinja +1 -1
  36. kash/web_gen/templates/components/tooltip_scripts.js.jinja +171 -19
  37. kash/web_gen/templates/components/tooltip_styles.css.jinja +23 -8
  38. {kash_shell-0.3.18.dist-info → kash_shell-0.3.21.dist-info}/METADATA +4 -2
  39. {kash_shell-0.3.18.dist-info → kash_shell-0.3.21.dist-info}/RECORD +42 -37
  40. kash/help/docstring_utils.py +0 -111
  41. {kash_shell-0.3.18.dist-info → kash_shell-0.3.21.dist-info}/WHEEL +0 -0
  42. {kash_shell-0.3.18.dist-info → kash_shell-0.3.21.dist-info}/entry_points.txt +0 -0
  43. {kash_shell-0.3.18.dist-info → kash_shell-0.3.21.dist-info}/licenses/LICENSE +0 -0
@@ -11,13 +11,10 @@ from kash.web_content.web_extract_readabilipy import extract_text_readabilipy
11
11
  log = get_logger(__name__)
12
12
 
13
13
 
14
- @kash_action(
15
- precondition=is_url_resource | has_html_body,
16
- mcp_tool=True,
17
- )
18
- def markdownify(item: Item) -> Item:
14
+ @kash_action(precondition=is_url_resource | has_html_body, mcp_tool=True)
15
+ def markdownify_html(item: Item) -> Item:
19
16
  """
20
- Converts a URL or raw HTML item to Markdown, fetching with the content
17
+ Converts raw HTML or the URL of an HTML page to Markdown, fetching with the content
21
18
  cache if needed. Also uses readability to clean up the HTML.
22
19
  """
23
20
 
@@ -23,14 +23,12 @@ from kash.exec import (
23
23
  resolve_locator_arg,
24
24
  )
25
25
  from kash.exec.action_registry import get_all_actions_defaults
26
- from kash.exec.fetch_url_metadata import fetch_url_metadata
26
+ from kash.exec.fetch_url_items import fetch_url_item
27
27
  from kash.exec.precondition_checks import actions_matching_paths
28
28
  from kash.exec.precondition_registry import get_all_preconditions
29
- from kash.exec.preconditions import is_url_resource
30
29
  from kash.exec_model.shell_model import ShellResult
31
30
  from kash.local_server.local_url_formatters import local_url_formatter
32
31
  from kash.media_base import media_tools
33
- from kash.media_base.media_services import is_media_url
34
32
  from kash.model.items_model import Item, ItemType
35
33
  from kash.model.params_model import GLOBAL_PARAMS
36
34
  from kash.model.paths_model import StorePath, fmt_store_path
@@ -54,12 +52,11 @@ from kash.utils.common.format_utils import fmt_loc
54
52
  from kash.utils.common.obj_replace import remove_values
55
53
  from kash.utils.common.parse_key_vals import parse_key_value
56
54
  from kash.utils.common.type_utils import not_none
57
- from kash.utils.common.url import Url, is_url, parse_http_url
55
+ from kash.utils.common.url import Url
58
56
  from kash.utils.errors import InvalidInput
59
57
  from kash.utils.file_formats.chat_format import tail_chat_history
60
58
  from kash.utils.file_utils.dir_info import is_nonempty_dir
61
59
  from kash.utils.file_utils.file_formats_model import Format
62
- from kash.utils.text_handling.doc_normalization import can_normalize
63
60
  from kash.web_content.file_cache_utils import cache_file
64
61
  from kash.workspaces import (
65
62
  current_ws,
@@ -189,85 +186,6 @@ def cache_content(*urls_or_paths: str, refetch: bool = False) -> None:
189
186
  PrintHooks.spacer()
190
187
 
191
188
 
192
- @kash_command
193
- def download(*urls_or_paths: str, refetch: bool = False, no_format: bool = False) -> ShellResult:
194
- """
195
- Download a URL or resource. Uses cached content if available, unless `refetch` is true.
196
- Inputs can be URLs or paths to URL resources.
197
- Creates both resource and document versions for text content.
198
-
199
- :param no_format: If true, do not also normalize Markdown content.
200
- """
201
- ws = current_ws()
202
- saved_paths = []
203
-
204
- for url_or_path in urls_or_paths:
205
- locator = resolve_locator_arg(url_or_path)
206
- url: Url | None = None
207
-
208
- # Get the URL from the locator
209
- if not isinstance(locator, Path) and is_url(locator):
210
- url = Url(locator)
211
- elif isinstance(locator, StorePath):
212
- url_item = ws.load(locator)
213
- if is_url_resource(url_item):
214
- url = url_item.url
215
-
216
- if not url:
217
- raise InvalidInput(f"Not a URL or URL resource: {fmt_loc(locator)}")
218
-
219
- # Handle media URLs differently
220
- if is_media_url(url):
221
- log.message(
222
- "URL is a media URL, so adding as a resource and will cache media: %s", fmt_loc(url)
223
- )
224
- store_path = ws.import_item(url, as_type=ItemType.resource, reimport=refetch)
225
- saved_paths.append(store_path)
226
- media_tools.cache_media(url)
227
- else:
228
- # Cache the content first
229
- expiration_sec = 0 if refetch else None
230
- cache_result = cache_file(url, expiration_sec=expiration_sec)
231
- original_filename = Path(parse_http_url(url).path).name
232
- mime_type = cache_result.content.headers and cache_result.content.headers.mime_type
233
-
234
- # Create a resource item
235
- resource_item = Item.from_external_path(
236
- cache_result.content.path,
237
- ItemType.resource,
238
- url=url,
239
- mime_type=mime_type,
240
- original_filename=original_filename,
241
- )
242
- # For initial content, do not format or add frontmatter.
243
- store_path = ws.save(resource_item, no_frontmatter=True, no_format=True)
244
- saved_paths.append(store_path)
245
- select(store_path)
246
-
247
- # Also create a doc version for text content if we want to normalize formatting.
248
- if resource_item.format and can_normalize(resource_item.format) and not no_format:
249
- doc_item = Item.from_external_path(
250
- cache_result.content.path,
251
- ItemType.doc,
252
- url=url,
253
- mime_type=mime_type,
254
- original_filename=original_filename,
255
- )
256
- # Now use default formatting and frontmatter.
257
- doc_store_path = ws.save(doc_item)
258
- saved_paths.append(doc_store_path)
259
- select(doc_store_path)
260
-
261
- print_status(
262
- "Downloaded %s %s:\n%s",
263
- len(saved_paths),
264
- plural("item", len(saved_paths)),
265
- fmt_lines(saved_paths),
266
- )
267
-
268
- return ShellResult(show_selection=True)
269
-
270
-
271
189
  @kash_command
272
190
  def history(max: int = 30, raw: bool = False) -> None:
273
191
  """
@@ -536,10 +454,14 @@ def save_clipboard(
536
454
 
537
455
 
538
456
  @kash_command
539
- def fetch_metadata(*files_or_urls: str, refetch: bool = False) -> ShellResult:
457
+ def fetch_url(*files_or_urls: str, refetch: bool = False) -> ShellResult:
540
458
  """
541
- Fetch metadata for the given URLs or resources. Imports new URLs and saves back
542
- the fetched metadata for existing resources.
459
+ Fetch content and metadata for the given URLs or resources, saving to the
460
+ current workspace.
461
+
462
+ Imports new URLs and saves back the fetched metadata for existing resources.
463
+ Also saves a resource item with the content of the URL, either HTML, text, or
464
+ of any other type.
543
465
 
544
466
  Skips items that already have a title and description, unless `refetch` is true.
545
467
  Skips (with a warning) items that are not URL resources.
@@ -552,7 +474,7 @@ def fetch_metadata(*files_or_urls: str, refetch: bool = False) -> ShellResult:
552
474
  store_paths = []
553
475
  for locator in locators:
554
476
  try:
555
- fetched_item = fetch_url_metadata(locator, refetch=refetch)
477
+ fetched_item = fetch_url_item(locator, refetch=refetch)
556
478
  store_paths.append(fetched_item.store_path)
557
479
  except InvalidInput as e:
558
480
  log.warning(
kash/config/colors.py CHANGED
@@ -139,14 +139,15 @@ web_light_translucent = SimpleNamespace(
139
139
  bg_header=hsl_to_hex("hsla(188, 42%, 70%, 0.2)"),
140
140
  bg_alt=hsl_to_hex("hsla(39, 24%, 90%, 0.3)"),
141
141
  bg_alt_solid=hsl_to_hex("hsla(39, 24%, 97%, 1)"),
142
- bg_selected=hsl_to_hex("hsla(188, 44%, 94%, 0.95)"),
142
+ bg_meta_solid=hsl_to_hex("hsla(39, 24%, 94%, 1)"),
143
+ bg_selected=hsl_to_hex("hsla(188, 21%, 94%, 0.9)"),
143
144
  text=hsl_to_hex("hsl(188, 39%, 11%)"),
144
145
  code=hsl_to_hex("hsl(44, 38%, 23%)"),
145
146
  border=hsl_to_hex("hsl(188, 8%, 50%)"),
146
147
  border_hint=hsl_to_hex("hsla(188, 8%, 72%, 0.3)"),
147
148
  border_accent=hsl_to_hex("hsla(305, 18%, 65%, 0.85)"),
148
149
  hover=hsl_to_hex("hsl(188, 12%, 84%)"),
149
- hover_bg=hsl_to_hex("hsla(188, 44%, 94%, 1)"),
150
+ hover_bg=hsl_to_hex("hsla(188, 18%, 97%, 1)"),
150
151
  hint=hsl_to_hex("hsl(188, 11%, 65%)"),
151
152
  hint_strong=hsl_to_hex("hsl(188, 11%, 46%)"),
152
153
  hint_gentle=hsl_to_hex("hsla(188, 11%, 65%, 0.2)"),
@@ -165,14 +166,15 @@ web_light_translucent = SimpleNamespace(
165
166
  web_dark_translucent = SimpleNamespace(
166
167
  primary=hsl_to_hex("hsl(188, 40%, 62%)"),
167
168
  primary_light=hsl_to_hex("hsl(188, 50%, 72%)"),
168
- secondary=hsl_to_hex("hsl(188, 12%, 65%)"),
169
- tertiary=hsl_to_hex("hsl(188, 7%, 40%)"),
169
+ secondary=hsl_to_hex("hsl(188, 12%, 70%)"),
170
+ tertiary=hsl_to_hex("hsl(188, 7%, 45%)"),
170
171
  bg=hsl_to_hex("hsla(220, 14%, 7%, 0.95)"),
171
172
  bg_solid=hsl_to_hex("hsl(220, 14%, 7%)"),
172
173
  bg_header=hsl_to_hex("hsla(188, 42%, 20%, 0.3)"),
173
174
  bg_alt=hsl_to_hex("hsla(220, 14%, 12%, 0.5)"),
174
- bg_alt_solid=hsl_to_hex("hsl(220, 14%, 12%)"),
175
- bg_selected=hsl_to_hex("hsla(188, 12%, 50%, 0.95)"),
175
+ bg_alt_solid=hsl_to_hex("hsl(220, 15%, 16%)"),
176
+ bg_meta_solid=hsl_to_hex("hsl(220, 14%, 25%)"),
177
+ bg_selected=hsl_to_hex("hsla(188, 13%, 33%, 0.95)"),
176
178
  text=hsl_to_hex("hsl(188, 10%, 90%)"),
177
179
  code=hsl_to_hex("hsl(44, 38%, 72%)"),
178
180
  border=hsl_to_hex("hsl(188, 8%, 25%)"),
@@ -277,6 +277,8 @@ EMOJI_HELP = "?"
277
277
 
278
278
  EMOJI_ACTION = "⛭"
279
279
 
280
+ EMOJI_TASK = "⚒"
281
+
280
282
  EMOJI_COMMAND = "⧁" # More ideas: ⦿⧁⧀⦿⦾⟐⦊⟡
281
283
 
282
284
  EMOJI_SHELL = "⦊"
@@ -34,7 +34,7 @@ the Python framework, a few core utilities, and the Kash command-line shell.
34
34
  Additional actions for handling more complex tasks like converting documents and
35
35
  transcribing, researching, or annotating videos, are in the
36
36
  [kash-docs](https://github.com/jlevy/kash-docs) and
37
- [kash-media](https://github.com/jlevy/kash-docs) packages, all available on PyPI and
37
+ [kash-media](https://github.com/jlevy/kash-media) packages, all available on PyPI and
38
38
  quick to install via uv.
39
39
 
40
40
  ### Key Concepts
@@ -165,48 +165,37 @@ works on readable text such as Markdown.
165
165
  This catches errors and allows you to find actions that might apply to a given selected
166
166
  set of items using `suggest_actions`.
167
167
 
168
- ### Programmatic Use
169
-
170
- Since commands and actions are really just Python functions.
171
-
172
- ### Useful Features
173
-
174
- Kash makes a few kinds of messy text manipulations easier:
175
-
176
- - Reusable LLM actions: A common kind of action is to invoke an LLM (like GPT-4o or o1)
177
- on a text item, with a given system and user prompt template.
178
- New LLM actions can be added with a few lines of Python by subclassing an action base
179
- class, typically `Action`, `CachedItemAction` (for any action that doesn't need to be
180
- rerun if it has the same single output), `CachedLLMAction` (if it also is performing
181
- an LLM-based transform), or `ChunkedLLMAction` (if it will be processing a document
182
- broken into <div class="chunk"> elements).
183
-
184
- - Sliding window transformations: LLMs can have trouble processing large inputs, not
185
- just because of context window and because they may make more mistakes when making
186
- lots of changes at once.
187
- Kash supports running actions in a sliding window across the document, then stitching
188
- the results back together when done.
189
-
190
- - Checking and enforcing changes: LLMs do not reliably do what they are asked to do.
191
- So a key part of making them useful is to save outputs at each step of the way and
192
- have a way to review their outputs or provide guardrails on what they can do with
193
- content.
194
-
195
- - Fine-grainded diffs with word tokens: Documents can be represented at the word level,
196
- using “word tokens” to represent words and normalized whitespace (word, sentence, and
197
- paragraph breaks, but not line breaks).
198
- This allows diffs of similar documents regardless of formatting.
199
- For example, it is possible to ask an LLM only to add paragraph breaks, then drop any
200
- other changes it makes to other words.
201
- You can use this intelligent matching of words to “backfill” specific content from one
202
- doc into an edited document, such as pulling timestamps from a full transcript back
203
- into an edited transcript or summary.
204
-
205
- - Paragraph and sentence operations: A lot of operations within actions should be done
206
- in chunks at the paragraph or sentence level.
207
- Kash offers simple tools to subdivide documents into paragraphs and sentences and
208
- these can be used together with sliding windows to process large documents.
209
-
210
- In addition, there are built-in kash commands that are part of the kash tool itself.
211
- These allow you to list items in the workspace, see or change the current selection,
212
- archive items, view logs, etc.
168
+ ### Programmatic Usage
169
+
170
+ Kash can be used entirely programmatically, so that actions are called just like
171
+ functions from Python, but the additional functionality of the items model, saving files
172
+ to a workspace, and so on, are all automatic.
173
+
174
+ This means you can use Kash to build your own CLI apps much more quickly.
175
+
176
+ For an example of this, see [textpress](https://github.com/jlevy/textpress), which wraps
177
+ quite a few kash actions to allow clean publishing of docx or PDF files on
178
+ [textpress.md](https://textpress.md/).
179
+
180
+ ### Utilities and Supporting Libraries
181
+
182
+ Kash includes a number of utility libraries to help with common tasks, either in the
183
+ base `kash-shell` package or or smaller dependencies:
184
+
185
+ - See [frontmatter-format](https://github.com/jlevy/frontmatter-format) for the spec and
186
+ implementation we use of frontmatter YAML format.
187
+
188
+ - See
189
+ [utils/file_utils](https://github.com/jlevy/kash/tree/main/src/kash/utils/file_utils)
190
+ for file format detection, conversion, filename handling, etc.
191
+
192
+ - See [chopdiff](https://github.com/jlevy/chopdiff) for a simple text doc data model
193
+ that includes sentences and paragraphs and fairly advanced diffing, filtered diffing,
194
+ and windowed transformations of text via LLM calls.
195
+
196
+ - See [clideps](https://github.com/jlevy/clideps) for utilities for helping with dot-env
197
+ files, API key setup, and dependency checks.
198
+
199
+ - See [utils/common](https://github.com/jlevy/kash/tree/main/src/kash/utils/common) the
200
+ rest of [utils/](https://github.com/jlevy/kash/tree/main/src/kash/utils) for a variety
201
+ of other general utilities.
kash/exec/__init__.py CHANGED
@@ -1,6 +1,7 @@
1
1
  from kash.exec.action_decorators import kash_action, kash_action_class
2
2
  from kash.exec.action_exec import SkipItem, prepare_action_input, run_action_with_shell_context
3
3
  from kash.exec.command_registry import kash_command
4
+ from kash.exec.fetch_url_items import fetch_url_item, fetch_url_item_content
4
5
  from kash.exec.importing import import_and_register
5
6
  from kash.exec.llm_transforms import llm_transform_item, llm_transform_str
6
7
  from kash.exec.precondition_registry import kash_precondition
@@ -21,6 +22,8 @@ __all__ = [
21
22
  "prepare_action_input",
22
23
  "run_action_with_shell_context",
23
24
  "kash_command",
25
+ "fetch_url_item",
26
+ "fetch_url_item_content",
24
27
  "kash_runtime",
25
28
  "current_runtime_settings",
26
29
  "import_and_register",
@@ -105,7 +105,10 @@ def kash_action_class(cls: type[A]) -> type[A]:
105
105
 
106
106
 
107
107
  def _register_dynamic_action(
108
- action_cls: type[A], action_name: str, action_description: str, source_path: Path | None
108
+ action_cls: type[A],
109
+ action_name: str,
110
+ action_description: str,
111
+ source_path: Path | None,
109
112
  ) -> type[A]:
110
113
  # Set class fields for name and description for convenience.
111
114
  action_cls.name = action_name
@@ -206,6 +209,7 @@ def kash_action(
206
209
  run_per_item: bool | None = None,
207
210
  uses_selection: bool = True,
208
211
  interactive_input: bool = False,
212
+ live_output: bool = False,
209
213
  mcp_tool: bool = False,
210
214
  title_template: TitleTemplate = TitleTemplate("{title}"),
211
215
  llm_options: LLMOptions = LLMOptions(),
@@ -235,13 +239,17 @@ def kash_action(
235
239
  def decorator(orig_func: AF) -> AF:
236
240
  if hasattr(orig_func, "__action_class__"):
237
241
  log.warning(
238
- "Function `%s` is already decorated with `@kash_action`", orig_func.__name__
242
+ "Function `%s` is already decorated with `@kash_action`",
243
+ orig_func.__name__,
239
244
  )
240
245
  return orig_func
241
246
 
242
247
  # Inspect and sanity check the formal params.
243
248
  func_params = inspect_function_params(orig_func)
244
- if len(func_params) == 0 or func_params[0].effective_type not in (ActionInput, Item):
249
+ if len(func_params) == 0 or func_params[0].effective_type not in (
250
+ ActionInput,
251
+ Item,
252
+ ):
245
253
  raise InvalidDefinition(
246
254
  f"Decorator `@kash_action` requires exactly one positional parameter, "
247
255
  f"`input` of type `ActionInput` or `Item` on function `{orig_func.__name__}` but "
@@ -311,6 +319,7 @@ def kash_action(
311
319
  self.uses_selection = uses_selection
312
320
  self.output_type = output_type
313
321
  self.interactive_input = interactive_input
322
+ self.live_output = live_output
314
323
  self.mcp_tool = mcp_tool
315
324
  self.title_template = title_template
316
325
  self.llm_options = llm_options
@@ -332,8 +341,14 @@ def kash_action(
332
341
  kw_args[fp.name] = self.get_param(fp.name)
333
342
 
334
343
  if self.params:
335
- log.info("Action function param declarations:\n%s", fmt_lines(self.params))
336
- log.info("Action function param values:\n%s", self.param_value_summary_str())
344
+ log.info(
345
+ "Action function param declarations:\n%s",
346
+ fmt_lines(self.params),
347
+ )
348
+ log.info(
349
+ "Action function param values:\n%s",
350
+ self.param_value_summary_str(),
351
+ )
337
352
  else:
338
353
  log.info("Action function has no declared params")
339
354
 
kash/exec/action_exec.py CHANGED
@@ -43,7 +43,7 @@ def prepare_action_input(*input_args: CommandArg, refetch: bool = False) -> Acti
43
43
  URL or file resources, either finding them in the workspace or importing them.
44
44
  Also fetches metadata for URLs if they don't already have title and description.
45
45
  """
46
- from kash.exec.fetch_url_metadata import fetch_url_item_metadata
46
+ from kash.exec.fetch_url_items import fetch_url_item_content
47
47
 
48
48
  ws = current_ws()
49
49
 
@@ -55,7 +55,7 @@ def prepare_action_input(*input_args: CommandArg, refetch: bool = False) -> Acti
55
55
  if input_items:
56
56
  log.message("Assembling metadata for input items:\n%s", fmt_lines(input_items))
57
57
  input_items = [
58
- fetch_url_item_metadata(item, refetch=refetch) if is_url_resource(item) else item
58
+ fetch_url_item_content(item, refetch=refetch) if is_url_resource(item) else item
59
59
  for item in input_items
60
60
  ]
61
61
 
@@ -11,7 +11,9 @@ from kash.utils.errors import InvalidInput
11
11
  log = get_logger(__name__)
12
12
 
13
13
 
14
- def fetch_url_metadata(locator: Url | StorePath, refetch: bool = False) -> Item:
14
+ def fetch_url_item(
15
+ locator: Url | StorePath, *, save_content: bool = True, refetch: bool = False
16
+ ) -> Item:
15
17
  from kash.workspaces import current_ws
16
18
 
17
19
  ws = current_ws()
@@ -26,16 +28,23 @@ def fetch_url_metadata(locator: Url | StorePath, refetch: bool = False) -> Item:
26
28
  else:
27
29
  raise InvalidInput(f"Not a URL or URL resource: {fmt_loc(locator)}")
28
30
 
29
- return fetch_url_item_metadata(item, refetch=refetch)
31
+ return fetch_url_item_content(item, save_content=save_content, refetch=refetch)
30
32
 
31
33
 
32
- def fetch_url_item_metadata(item: Item, refetch: bool = False) -> Item:
34
+ def fetch_url_item_content(item: Item, *, save_content: bool = True, refetch: bool = False) -> Item:
33
35
  """
34
- Fetch metadata for a URL using a media service if we recognize the URL,
35
- and otherwise fetching and extracting it from the web page HTML.
36
+ Fetch content and metadata for a URL using a media service if we
37
+ recognize the URL as a known media service. Otherwise, fetch and extract the
38
+ metadata and content from the web page and save it to the URL item.
39
+
40
+ If `save_content` is true, a copy of the content is also saved as
41
+ a resource item.
42
+
43
+ The content item is returned if content was saved. Otherwise, the updated
44
+ URL item is returned.
36
45
  """
37
46
  from kash.web_content.canon_url import canonicalize_url
38
- from kash.web_content.web_extract import fetch_extract
47
+ from kash.web_content.web_extract import fetch_page_content
39
48
  from kash.workspaces import current_ws
40
49
 
41
50
  ws = current_ws()
@@ -54,28 +63,47 @@ def fetch_url_item_metadata(item: Item, refetch: bool = False) -> Item:
54
63
  # Prefer fetching metadata from media using the media service if possible.
55
64
  # Data is cleaner and YouTube for example often blocks regular scraping.
56
65
  media_metadata = get_media_metadata(url)
66
+ url_item: Item | None = None
67
+ content_item: Item | None = None
57
68
  if media_metadata:
58
- fetched_item = Item.from_media_metadata(media_metadata)
69
+ url_item = Item.from_media_metadata(media_metadata)
59
70
  # Preserve and canonicalize any slice suffix on the URL.
60
71
  _base_url, slice = parse_url_slice(item.url)
61
72
  if slice:
62
73
  new_url = add_slice_to_url(media_metadata.url, slice)
63
74
  if new_url != item.url:
64
75
  log.message("Updated URL from metadata and added slice: %s", new_url)
65
- fetched_item.url = new_url
76
+ url_item.url = new_url
66
77
 
67
- fetched_item = item.merged_copy(fetched_item)
78
+ url_item = item.merged_copy(url_item)
68
79
  else:
69
- page_data = fetch_extract(url, refetch=refetch)
70
- fetched_item = item.new_copy_with(
80
+ page_data = fetch_page_content(url, refetch=refetch, cache=save_content)
81
+ url_item = item.new_copy_with(
71
82
  title=page_data.title or item.title,
72
83
  description=page_data.description or item.description,
73
84
  thumbnail_url=page_data.thumbnail_url or item.thumbnail_url,
74
85
  )
86
+ if save_content:
87
+ assert page_data.saved_content
88
+ assert page_data.format_info
89
+ content_item = url_item.new_copy_with(
90
+ external_path=str(page_data.saved_content),
91
+ # Use the original filename, not the local cache filename (which has a hash suffix).
92
+ original_filename=item.get_filename(),
93
+ format=page_data.format_info.format,
94
+ )
95
+ ws.save(content_item)
75
96
 
76
- if not fetched_item.title:
97
+ if not url_item.title:
77
98
  log.warning("Failed to fetch page data: title is missing: %s", item.url)
78
99
 
79
- ws.save(fetched_item)
100
+ # Now save the updated URL item and also the content item if we have one.
101
+ ws.save(url_item)
102
+ assert url_item.store_path
103
+ log.debug("Saved URL item: %s", url_item.fmt_loc())
104
+ if content_item:
105
+ ws.save(content_item)
106
+ assert content_item.store_path
107
+ log.debug("Saved content item: %s", content_item.fmt_loc())
80
108
 
81
- return fetched_item
109
+ return content_item or url_item
@@ -68,7 +68,7 @@ def llm_transform_str(options: LLMOptions, input_str: str, check_no_results: boo
68
68
  diff_filter=options.diff_filter,
69
69
  ).reassemble()
70
70
  else:
71
- log.message(
71
+ log.info(
72
72
  "Running simple LLM transform action %s with model %s",
73
73
  options.op_name,
74
74
  options.model.litellm_name,
@@ -56,7 +56,7 @@ class ShellCallableAction:
56
56
 
57
57
  log.info("Action shell args: %s", shell_args)
58
58
  explicit_values = RawParamValues(shell_args.options)
59
- if not action.interactive_input:
59
+ if not action.interactive_input and not action.live_output:
60
60
  with get_console().status(f"Running action {action.name}…", spinner=SPINNER):
61
61
  result = run_action_with_shell_context(
62
62
  action_cls,
@@ -405,6 +405,7 @@ class FileStore(Workspace):
405
405
  # If external path already exists and is within the workspace, the file was
406
406
  # already saved (e.g. by an action that wrote the item directly to the store).
407
407
  external_path = item.external_path and Path(item.external_path).resolve()
408
+ skipped_save = False
408
409
  if external_path and self._is_in_store(external_path):
409
410
  log.info("Item with external_path already saved: %s", fmt_loc(external_path))
410
411
  rel_path = external_path.relative_to(self.base_dir)
@@ -480,12 +481,17 @@ class FileStore(Workspace):
480
481
  )
481
482
  os.unlink(full_path)
482
483
  store_path = old_store_path
484
+ skipped_save = True
483
485
 
484
486
  # Update in-memory store_path only after successful save.
485
487
  item.store_path = str(store_path)
486
488
  self._id_index_item(store_path)
487
489
 
488
- log.message("%s Saved item: %s", EMOJI_SAVED, fmt_loc(store_path))
490
+ if not skipped_save:
491
+ log.message("%s Saved item: %s", EMOJI_SAVED, fmt_loc(store_path))
492
+ else:
493
+ log.info("%s Already saved: %s", EMOJI_SAVED, fmt_loc(store_path))
494
+
489
495
  return store_path
490
496
 
491
497
  @log_calls(level="debug")
@@ -30,6 +30,10 @@ def folder_for_type(item_type: ItemType) -> Path:
30
30
 
31
31
 
32
32
  def join_suffix(base_slug: str, full_suffix: str) -> str:
33
+ """
34
+ Create a store filename by joining a base slug and a full suffix, i.e. a filename
35
+ extension with or without an item type (`.html` or `.resource.html`, for example).
36
+ """
33
37
  return f"{base_slug}.{full_suffix.lstrip('.')}"
34
38
 
35
39
 
@@ -2,9 +2,9 @@ from collections.abc import Callable
2
2
  from dataclasses import replace
3
3
  from typing import Any
4
4
 
5
- from kash.help.docstring_utils import parse_docstring
6
5
  from kash.model.params_model import ALL_COMMON_PARAMS, Param
7
6
  from kash.utils.common.function_inspect import FuncParam, inspect_function_params
7
+ from kash.utils.common.parse_docstring import parse_docstring
8
8
 
9
9
 
10
10
  def _look_up_param_docs(func: Callable[..., Any], kw_params: list[FuncParam]) -> list[Param]:
kash/help/help_pages.py CHANGED
@@ -3,7 +3,6 @@ from rich.text import Text
3
3
  from kash.config.logger import get_logger
4
4
  from kash.config.text_styles import STYLE_HINT
5
5
  from kash.docs.all_docs import DocSelection, all_docs
6
- from kash.help.docstring_utils import parse_docstring
7
6
  from kash.shell.output.shell_formatting import format_name_and_value
8
7
  from kash.shell.output.shell_output import (
9
8
  PrintHooks,
@@ -12,6 +11,7 @@ from kash.shell.output.shell_output import (
12
11
  print_hrule,
13
12
  print_markdown,
14
13
  )
14
+ from kash.utils.common.parse_docstring import parse_docstring
15
15
 
16
16
  log = get_logger(__name__)
17
17
 
@@ -6,7 +6,6 @@ from kash.docs.all_docs import DocSelection
6
6
  from kash.exec.action_registry import look_up_action_class
7
7
  from kash.exec.command_registry import CommandFunction, look_up_command
8
8
  from kash.help.assistant import assist_preamble, assistance_unstructured
9
- from kash.help.docstring_utils import parse_docstring
10
9
  from kash.help.function_param_info import annotate_param_info
11
10
  from kash.help.help_lookups import look_up_faq
12
11
  from kash.help.tldr_help import tldr_help
@@ -22,6 +21,7 @@ from kash.shell.output.shell_output import (
22
21
  print_help,
23
22
  print_markdown,
24
23
  )
24
+ from kash.utils.common.parse_docstring import parse_docstring
25
25
  from kash.utils.errors import InvalidInput, NoMatch
26
26
  from kash.utils.file_formats.chat_format import ChatHistory, ChatMessage, ChatRole
27
27
 
@@ -173,7 +173,7 @@ def llm_template_completion(
173
173
  )
174
174
 
175
175
  if check_no_results and is_no_results(result.content):
176
- log.message("No results for LLM transform, will ignore: %r", result.content)
176
+ log.info("No results for LLM transform, will ignore: %r", result.content)
177
177
  result.content = ""
178
178
 
179
179
  return result
@@ -270,6 +270,12 @@ class Action(ABC):
270
270
  Does this action ask for input interactively?
271
271
  """
272
272
 
273
+ live_output: bool = False
274
+ """
275
+ Does this action have live output (e.g., progress bars, spinners)?
276
+ If True, the shell should not show its own status spinner.
277
+ """
278
+
273
279
  mcp_tool: bool = False
274
280
  """
275
281
  If True, this action is published as an MCP tool.