kash-shell 0.3.18__py3-none-any.whl → 0.3.21__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kash/actions/core/{markdownify.py → markdownify_html.py} +3 -6
- kash/commands/workspace/workspace_commands.py +10 -88
- kash/config/colors.py +8 -6
- kash/config/text_styles.py +2 -0
- kash/docs/markdown/topics/a1_what_is_kash.md +1 -1
- kash/docs/markdown/topics/b1_kash_overview.md +34 -45
- kash/exec/__init__.py +3 -0
- kash/exec/action_decorators.py +20 -5
- kash/exec/action_exec.py +2 -2
- kash/exec/{fetch_url_metadata.py → fetch_url_items.py} +42 -14
- kash/exec/llm_transforms.py +1 -1
- kash/exec/shell_callable_action.py +1 -1
- kash/file_storage/file_store.py +7 -1
- kash/file_storage/store_filenames.py +4 -0
- kash/help/function_param_info.py +1 -1
- kash/help/help_pages.py +1 -1
- kash/help/help_printing.py +1 -1
- kash/llm_utils/llm_completion.py +1 -1
- kash/model/actions_model.py +6 -0
- kash/model/items_model.py +18 -3
- kash/shell/output/shell_output.py +15 -0
- kash/utils/api_utils/api_retries.py +305 -0
- kash/utils/api_utils/cache_requests_limited.py +84 -0
- kash/utils/api_utils/gather_limited.py +987 -0
- kash/utils/api_utils/progress_protocol.py +299 -0
- kash/utils/common/function_inspect.py +66 -1
- kash/utils/common/parse_docstring.py +347 -0
- kash/utils/common/testing.py +10 -7
- kash/utils/rich_custom/multitask_status.py +631 -0
- kash/utils/text_handling/escape_html_tags.py +16 -11
- kash/utils/text_handling/markdown_render.py +1 -0
- kash/web_content/web_extract.py +34 -15
- kash/web_content/web_page_model.py +10 -1
- kash/web_gen/templates/base_styles.css.jinja +26 -20
- kash/web_gen/templates/components/toc_styles.css.jinja +1 -1
- kash/web_gen/templates/components/tooltip_scripts.js.jinja +171 -19
- kash/web_gen/templates/components/tooltip_styles.css.jinja +23 -8
- {kash_shell-0.3.18.dist-info → kash_shell-0.3.21.dist-info}/METADATA +4 -2
- {kash_shell-0.3.18.dist-info → kash_shell-0.3.21.dist-info}/RECORD +42 -37
- kash/help/docstring_utils.py +0 -111
- {kash_shell-0.3.18.dist-info → kash_shell-0.3.21.dist-info}/WHEEL +0 -0
- {kash_shell-0.3.18.dist-info → kash_shell-0.3.21.dist-info}/entry_points.txt +0 -0
- {kash_shell-0.3.18.dist-info → kash_shell-0.3.21.dist-info}/licenses/LICENSE +0 -0
|
@@ -11,13 +11,10 @@ from kash.web_content.web_extract_readabilipy import extract_text_readabilipy
|
|
|
11
11
|
log = get_logger(__name__)
|
|
12
12
|
|
|
13
13
|
|
|
14
|
-
@kash_action(
|
|
15
|
-
|
|
16
|
-
mcp_tool=True,
|
|
17
|
-
)
|
|
18
|
-
def markdownify(item: Item) -> Item:
|
|
14
|
+
@kash_action(precondition=is_url_resource | has_html_body, mcp_tool=True)
|
|
15
|
+
def markdownify_html(item: Item) -> Item:
|
|
19
16
|
"""
|
|
20
|
-
Converts
|
|
17
|
+
Converts raw HTML or the URL of an HTML page to Markdown, fetching with the content
|
|
21
18
|
cache if needed. Also uses readability to clean up the HTML.
|
|
22
19
|
"""
|
|
23
20
|
|
|
@@ -23,14 +23,12 @@ from kash.exec import (
|
|
|
23
23
|
resolve_locator_arg,
|
|
24
24
|
)
|
|
25
25
|
from kash.exec.action_registry import get_all_actions_defaults
|
|
26
|
-
from kash.exec.
|
|
26
|
+
from kash.exec.fetch_url_items import fetch_url_item
|
|
27
27
|
from kash.exec.precondition_checks import actions_matching_paths
|
|
28
28
|
from kash.exec.precondition_registry import get_all_preconditions
|
|
29
|
-
from kash.exec.preconditions import is_url_resource
|
|
30
29
|
from kash.exec_model.shell_model import ShellResult
|
|
31
30
|
from kash.local_server.local_url_formatters import local_url_formatter
|
|
32
31
|
from kash.media_base import media_tools
|
|
33
|
-
from kash.media_base.media_services import is_media_url
|
|
34
32
|
from kash.model.items_model import Item, ItemType
|
|
35
33
|
from kash.model.params_model import GLOBAL_PARAMS
|
|
36
34
|
from kash.model.paths_model import StorePath, fmt_store_path
|
|
@@ -54,12 +52,11 @@ from kash.utils.common.format_utils import fmt_loc
|
|
|
54
52
|
from kash.utils.common.obj_replace import remove_values
|
|
55
53
|
from kash.utils.common.parse_key_vals import parse_key_value
|
|
56
54
|
from kash.utils.common.type_utils import not_none
|
|
57
|
-
from kash.utils.common.url import Url
|
|
55
|
+
from kash.utils.common.url import Url
|
|
58
56
|
from kash.utils.errors import InvalidInput
|
|
59
57
|
from kash.utils.file_formats.chat_format import tail_chat_history
|
|
60
58
|
from kash.utils.file_utils.dir_info import is_nonempty_dir
|
|
61
59
|
from kash.utils.file_utils.file_formats_model import Format
|
|
62
|
-
from kash.utils.text_handling.doc_normalization import can_normalize
|
|
63
60
|
from kash.web_content.file_cache_utils import cache_file
|
|
64
61
|
from kash.workspaces import (
|
|
65
62
|
current_ws,
|
|
@@ -189,85 +186,6 @@ def cache_content(*urls_or_paths: str, refetch: bool = False) -> None:
|
|
|
189
186
|
PrintHooks.spacer()
|
|
190
187
|
|
|
191
188
|
|
|
192
|
-
@kash_command
|
|
193
|
-
def download(*urls_or_paths: str, refetch: bool = False, no_format: bool = False) -> ShellResult:
|
|
194
|
-
"""
|
|
195
|
-
Download a URL or resource. Uses cached content if available, unless `refetch` is true.
|
|
196
|
-
Inputs can be URLs or paths to URL resources.
|
|
197
|
-
Creates both resource and document versions for text content.
|
|
198
|
-
|
|
199
|
-
:param no_format: If true, do not also normalize Markdown content.
|
|
200
|
-
"""
|
|
201
|
-
ws = current_ws()
|
|
202
|
-
saved_paths = []
|
|
203
|
-
|
|
204
|
-
for url_or_path in urls_or_paths:
|
|
205
|
-
locator = resolve_locator_arg(url_or_path)
|
|
206
|
-
url: Url | None = None
|
|
207
|
-
|
|
208
|
-
# Get the URL from the locator
|
|
209
|
-
if not isinstance(locator, Path) and is_url(locator):
|
|
210
|
-
url = Url(locator)
|
|
211
|
-
elif isinstance(locator, StorePath):
|
|
212
|
-
url_item = ws.load(locator)
|
|
213
|
-
if is_url_resource(url_item):
|
|
214
|
-
url = url_item.url
|
|
215
|
-
|
|
216
|
-
if not url:
|
|
217
|
-
raise InvalidInput(f"Not a URL or URL resource: {fmt_loc(locator)}")
|
|
218
|
-
|
|
219
|
-
# Handle media URLs differently
|
|
220
|
-
if is_media_url(url):
|
|
221
|
-
log.message(
|
|
222
|
-
"URL is a media URL, so adding as a resource and will cache media: %s", fmt_loc(url)
|
|
223
|
-
)
|
|
224
|
-
store_path = ws.import_item(url, as_type=ItemType.resource, reimport=refetch)
|
|
225
|
-
saved_paths.append(store_path)
|
|
226
|
-
media_tools.cache_media(url)
|
|
227
|
-
else:
|
|
228
|
-
# Cache the content first
|
|
229
|
-
expiration_sec = 0 if refetch else None
|
|
230
|
-
cache_result = cache_file(url, expiration_sec=expiration_sec)
|
|
231
|
-
original_filename = Path(parse_http_url(url).path).name
|
|
232
|
-
mime_type = cache_result.content.headers and cache_result.content.headers.mime_type
|
|
233
|
-
|
|
234
|
-
# Create a resource item
|
|
235
|
-
resource_item = Item.from_external_path(
|
|
236
|
-
cache_result.content.path,
|
|
237
|
-
ItemType.resource,
|
|
238
|
-
url=url,
|
|
239
|
-
mime_type=mime_type,
|
|
240
|
-
original_filename=original_filename,
|
|
241
|
-
)
|
|
242
|
-
# For initial content, do not format or add frontmatter.
|
|
243
|
-
store_path = ws.save(resource_item, no_frontmatter=True, no_format=True)
|
|
244
|
-
saved_paths.append(store_path)
|
|
245
|
-
select(store_path)
|
|
246
|
-
|
|
247
|
-
# Also create a doc version for text content if we want to normalize formatting.
|
|
248
|
-
if resource_item.format and can_normalize(resource_item.format) and not no_format:
|
|
249
|
-
doc_item = Item.from_external_path(
|
|
250
|
-
cache_result.content.path,
|
|
251
|
-
ItemType.doc,
|
|
252
|
-
url=url,
|
|
253
|
-
mime_type=mime_type,
|
|
254
|
-
original_filename=original_filename,
|
|
255
|
-
)
|
|
256
|
-
# Now use default formatting and frontmatter.
|
|
257
|
-
doc_store_path = ws.save(doc_item)
|
|
258
|
-
saved_paths.append(doc_store_path)
|
|
259
|
-
select(doc_store_path)
|
|
260
|
-
|
|
261
|
-
print_status(
|
|
262
|
-
"Downloaded %s %s:\n%s",
|
|
263
|
-
len(saved_paths),
|
|
264
|
-
plural("item", len(saved_paths)),
|
|
265
|
-
fmt_lines(saved_paths),
|
|
266
|
-
)
|
|
267
|
-
|
|
268
|
-
return ShellResult(show_selection=True)
|
|
269
|
-
|
|
270
|
-
|
|
271
189
|
@kash_command
|
|
272
190
|
def history(max: int = 30, raw: bool = False) -> None:
|
|
273
191
|
"""
|
|
@@ -536,10 +454,14 @@ def save_clipboard(
|
|
|
536
454
|
|
|
537
455
|
|
|
538
456
|
@kash_command
|
|
539
|
-
def
|
|
457
|
+
def fetch_url(*files_or_urls: str, refetch: bool = False) -> ShellResult:
|
|
540
458
|
"""
|
|
541
|
-
Fetch metadata for the given URLs or resources
|
|
542
|
-
|
|
459
|
+
Fetch content and metadata for the given URLs or resources, saving to the
|
|
460
|
+
current workspace.
|
|
461
|
+
|
|
462
|
+
Imports new URLs and saves back the fetched metadata for existing resources.
|
|
463
|
+
Also saves a resource item with the content of the URL, either HTML, text, or
|
|
464
|
+
of any other type.
|
|
543
465
|
|
|
544
466
|
Skips items that already have a title and description, unless `refetch` is true.
|
|
545
467
|
Skips (with a warning) items that are not URL resources.
|
|
@@ -552,7 +474,7 @@ def fetch_metadata(*files_or_urls: str, refetch: bool = False) -> ShellResult:
|
|
|
552
474
|
store_paths = []
|
|
553
475
|
for locator in locators:
|
|
554
476
|
try:
|
|
555
|
-
fetched_item =
|
|
477
|
+
fetched_item = fetch_url_item(locator, refetch=refetch)
|
|
556
478
|
store_paths.append(fetched_item.store_path)
|
|
557
479
|
except InvalidInput as e:
|
|
558
480
|
log.warning(
|
kash/config/colors.py
CHANGED
|
@@ -139,14 +139,15 @@ web_light_translucent = SimpleNamespace(
|
|
|
139
139
|
bg_header=hsl_to_hex("hsla(188, 42%, 70%, 0.2)"),
|
|
140
140
|
bg_alt=hsl_to_hex("hsla(39, 24%, 90%, 0.3)"),
|
|
141
141
|
bg_alt_solid=hsl_to_hex("hsla(39, 24%, 97%, 1)"),
|
|
142
|
-
|
|
142
|
+
bg_meta_solid=hsl_to_hex("hsla(39, 24%, 94%, 1)"),
|
|
143
|
+
bg_selected=hsl_to_hex("hsla(188, 21%, 94%, 0.9)"),
|
|
143
144
|
text=hsl_to_hex("hsl(188, 39%, 11%)"),
|
|
144
145
|
code=hsl_to_hex("hsl(44, 38%, 23%)"),
|
|
145
146
|
border=hsl_to_hex("hsl(188, 8%, 50%)"),
|
|
146
147
|
border_hint=hsl_to_hex("hsla(188, 8%, 72%, 0.3)"),
|
|
147
148
|
border_accent=hsl_to_hex("hsla(305, 18%, 65%, 0.85)"),
|
|
148
149
|
hover=hsl_to_hex("hsl(188, 12%, 84%)"),
|
|
149
|
-
hover_bg=hsl_to_hex("hsla(188,
|
|
150
|
+
hover_bg=hsl_to_hex("hsla(188, 18%, 97%, 1)"),
|
|
150
151
|
hint=hsl_to_hex("hsl(188, 11%, 65%)"),
|
|
151
152
|
hint_strong=hsl_to_hex("hsl(188, 11%, 46%)"),
|
|
152
153
|
hint_gentle=hsl_to_hex("hsla(188, 11%, 65%, 0.2)"),
|
|
@@ -165,14 +166,15 @@ web_light_translucent = SimpleNamespace(
|
|
|
165
166
|
web_dark_translucent = SimpleNamespace(
|
|
166
167
|
primary=hsl_to_hex("hsl(188, 40%, 62%)"),
|
|
167
168
|
primary_light=hsl_to_hex("hsl(188, 50%, 72%)"),
|
|
168
|
-
secondary=hsl_to_hex("hsl(188, 12%,
|
|
169
|
-
tertiary=hsl_to_hex("hsl(188, 7%,
|
|
169
|
+
secondary=hsl_to_hex("hsl(188, 12%, 70%)"),
|
|
170
|
+
tertiary=hsl_to_hex("hsl(188, 7%, 45%)"),
|
|
170
171
|
bg=hsl_to_hex("hsla(220, 14%, 7%, 0.95)"),
|
|
171
172
|
bg_solid=hsl_to_hex("hsl(220, 14%, 7%)"),
|
|
172
173
|
bg_header=hsl_to_hex("hsla(188, 42%, 20%, 0.3)"),
|
|
173
174
|
bg_alt=hsl_to_hex("hsla(220, 14%, 12%, 0.5)"),
|
|
174
|
-
bg_alt_solid=hsl_to_hex("hsl(220,
|
|
175
|
-
|
|
175
|
+
bg_alt_solid=hsl_to_hex("hsl(220, 15%, 16%)"),
|
|
176
|
+
bg_meta_solid=hsl_to_hex("hsl(220, 14%, 25%)"),
|
|
177
|
+
bg_selected=hsl_to_hex("hsla(188, 13%, 33%, 0.95)"),
|
|
176
178
|
text=hsl_to_hex("hsl(188, 10%, 90%)"),
|
|
177
179
|
code=hsl_to_hex("hsl(44, 38%, 72%)"),
|
|
178
180
|
border=hsl_to_hex("hsl(188, 8%, 25%)"),
|
kash/config/text_styles.py
CHANGED
|
@@ -34,7 +34,7 @@ the Python framework, a few core utilities, and the Kash command-line shell.
|
|
|
34
34
|
Additional actions for handling more complex tasks like converting documents and
|
|
35
35
|
transcribing, researching, or annotating videos, are in the
|
|
36
36
|
[kash-docs](https://github.com/jlevy/kash-docs) and
|
|
37
|
-
[kash-media](https://github.com/jlevy/kash-
|
|
37
|
+
[kash-media](https://github.com/jlevy/kash-media) packages, all available on PyPI and
|
|
38
38
|
quick to install via uv.
|
|
39
39
|
|
|
40
40
|
### Key Concepts
|
|
@@ -165,48 +165,37 @@ works on readable text such as Markdown.
|
|
|
165
165
|
This catches errors and allows you to find actions that might apply to a given selected
|
|
166
166
|
set of items using `suggest_actions`.
|
|
167
167
|
|
|
168
|
-
### Programmatic
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
Kash
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
doc into an edited document, such as pulling timestamps from a full transcript back
|
|
203
|
-
into an edited transcript or summary.
|
|
204
|
-
|
|
205
|
-
- Paragraph and sentence operations: A lot of operations within actions should be done
|
|
206
|
-
in chunks at the paragraph or sentence level.
|
|
207
|
-
Kash offers simple tools to subdivide documents into paragraphs and sentences and
|
|
208
|
-
these can be used together with sliding windows to process large documents.
|
|
209
|
-
|
|
210
|
-
In addition, there are built-in kash commands that are part of the kash tool itself.
|
|
211
|
-
These allow you to list items in the workspace, see or change the current selection,
|
|
212
|
-
archive items, view logs, etc.
|
|
168
|
+
### Programmatic Usage
|
|
169
|
+
|
|
170
|
+
Kash can be used entirely programmatically, so that actions are called just like
|
|
171
|
+
functions from Python, but the additional functionality of the items model, saving files
|
|
172
|
+
to a workspace, and so on, are all automatic.
|
|
173
|
+
|
|
174
|
+
This means you can use Kash to build your own CLI apps much more quickly.
|
|
175
|
+
|
|
176
|
+
For an example of this, see [textpress](https://github.com/jlevy/textpress), which wraps
|
|
177
|
+
quite a few kash actions to allow clean publishing of docx or PDF files on
|
|
178
|
+
[textpress.md](https://textpress.md/).
|
|
179
|
+
|
|
180
|
+
### Utilities and Supporting Libraries
|
|
181
|
+
|
|
182
|
+
Kash includes a number of utility libraries to help with common tasks, either in the
|
|
183
|
+
base `kash-shell` package or or smaller dependencies:
|
|
184
|
+
|
|
185
|
+
- See [frontmatter-format](https://github.com/jlevy/frontmatter-format) for the spec and
|
|
186
|
+
implementation we use of frontmatter YAML format.
|
|
187
|
+
|
|
188
|
+
- See
|
|
189
|
+
[utils/file_utils](https://github.com/jlevy/kash/tree/main/src/kash/utils/file_utils)
|
|
190
|
+
for file format detection, conversion, filename handling, etc.
|
|
191
|
+
|
|
192
|
+
- See [chopdiff](https://github.com/jlevy/chopdiff) for a simple text doc data model
|
|
193
|
+
that includes sentences and paragraphs and fairly advanced diffing, filtered diffing,
|
|
194
|
+
and windowed transformations of text via LLM calls.
|
|
195
|
+
|
|
196
|
+
- See [clideps](https://github.com/jlevy/clideps) for utilities for helping with dot-env
|
|
197
|
+
files, API key setup, and dependency checks.
|
|
198
|
+
|
|
199
|
+
- See [utils/common](https://github.com/jlevy/kash/tree/main/src/kash/utils/common) the
|
|
200
|
+
rest of [utils/](https://github.com/jlevy/kash/tree/main/src/kash/utils) for a variety
|
|
201
|
+
of other general utilities.
|
kash/exec/__init__.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from kash.exec.action_decorators import kash_action, kash_action_class
|
|
2
2
|
from kash.exec.action_exec import SkipItem, prepare_action_input, run_action_with_shell_context
|
|
3
3
|
from kash.exec.command_registry import kash_command
|
|
4
|
+
from kash.exec.fetch_url_items import fetch_url_item, fetch_url_item_content
|
|
4
5
|
from kash.exec.importing import import_and_register
|
|
5
6
|
from kash.exec.llm_transforms import llm_transform_item, llm_transform_str
|
|
6
7
|
from kash.exec.precondition_registry import kash_precondition
|
|
@@ -21,6 +22,8 @@ __all__ = [
|
|
|
21
22
|
"prepare_action_input",
|
|
22
23
|
"run_action_with_shell_context",
|
|
23
24
|
"kash_command",
|
|
25
|
+
"fetch_url_item",
|
|
26
|
+
"fetch_url_item_content",
|
|
24
27
|
"kash_runtime",
|
|
25
28
|
"current_runtime_settings",
|
|
26
29
|
"import_and_register",
|
kash/exec/action_decorators.py
CHANGED
|
@@ -105,7 +105,10 @@ def kash_action_class(cls: type[A]) -> type[A]:
|
|
|
105
105
|
|
|
106
106
|
|
|
107
107
|
def _register_dynamic_action(
|
|
108
|
-
action_cls: type[A],
|
|
108
|
+
action_cls: type[A],
|
|
109
|
+
action_name: str,
|
|
110
|
+
action_description: str,
|
|
111
|
+
source_path: Path | None,
|
|
109
112
|
) -> type[A]:
|
|
110
113
|
# Set class fields for name and description for convenience.
|
|
111
114
|
action_cls.name = action_name
|
|
@@ -206,6 +209,7 @@ def kash_action(
|
|
|
206
209
|
run_per_item: bool | None = None,
|
|
207
210
|
uses_selection: bool = True,
|
|
208
211
|
interactive_input: bool = False,
|
|
212
|
+
live_output: bool = False,
|
|
209
213
|
mcp_tool: bool = False,
|
|
210
214
|
title_template: TitleTemplate = TitleTemplate("{title}"),
|
|
211
215
|
llm_options: LLMOptions = LLMOptions(),
|
|
@@ -235,13 +239,17 @@ def kash_action(
|
|
|
235
239
|
def decorator(orig_func: AF) -> AF:
|
|
236
240
|
if hasattr(orig_func, "__action_class__"):
|
|
237
241
|
log.warning(
|
|
238
|
-
"Function `%s` is already decorated with `@kash_action`",
|
|
242
|
+
"Function `%s` is already decorated with `@kash_action`",
|
|
243
|
+
orig_func.__name__,
|
|
239
244
|
)
|
|
240
245
|
return orig_func
|
|
241
246
|
|
|
242
247
|
# Inspect and sanity check the formal params.
|
|
243
248
|
func_params = inspect_function_params(orig_func)
|
|
244
|
-
if len(func_params) == 0 or func_params[0].effective_type not in (
|
|
249
|
+
if len(func_params) == 0 or func_params[0].effective_type not in (
|
|
250
|
+
ActionInput,
|
|
251
|
+
Item,
|
|
252
|
+
):
|
|
245
253
|
raise InvalidDefinition(
|
|
246
254
|
f"Decorator `@kash_action` requires exactly one positional parameter, "
|
|
247
255
|
f"`input` of type `ActionInput` or `Item` on function `{orig_func.__name__}` but "
|
|
@@ -311,6 +319,7 @@ def kash_action(
|
|
|
311
319
|
self.uses_selection = uses_selection
|
|
312
320
|
self.output_type = output_type
|
|
313
321
|
self.interactive_input = interactive_input
|
|
322
|
+
self.live_output = live_output
|
|
314
323
|
self.mcp_tool = mcp_tool
|
|
315
324
|
self.title_template = title_template
|
|
316
325
|
self.llm_options = llm_options
|
|
@@ -332,8 +341,14 @@ def kash_action(
|
|
|
332
341
|
kw_args[fp.name] = self.get_param(fp.name)
|
|
333
342
|
|
|
334
343
|
if self.params:
|
|
335
|
-
log.info(
|
|
336
|
-
|
|
344
|
+
log.info(
|
|
345
|
+
"Action function param declarations:\n%s",
|
|
346
|
+
fmt_lines(self.params),
|
|
347
|
+
)
|
|
348
|
+
log.info(
|
|
349
|
+
"Action function param values:\n%s",
|
|
350
|
+
self.param_value_summary_str(),
|
|
351
|
+
)
|
|
337
352
|
else:
|
|
338
353
|
log.info("Action function has no declared params")
|
|
339
354
|
|
kash/exec/action_exec.py
CHANGED
|
@@ -43,7 +43,7 @@ def prepare_action_input(*input_args: CommandArg, refetch: bool = False) -> Acti
|
|
|
43
43
|
URL or file resources, either finding them in the workspace or importing them.
|
|
44
44
|
Also fetches metadata for URLs if they don't already have title and description.
|
|
45
45
|
"""
|
|
46
|
-
from kash.exec.
|
|
46
|
+
from kash.exec.fetch_url_items import fetch_url_item_content
|
|
47
47
|
|
|
48
48
|
ws = current_ws()
|
|
49
49
|
|
|
@@ -55,7 +55,7 @@ def prepare_action_input(*input_args: CommandArg, refetch: bool = False) -> Acti
|
|
|
55
55
|
if input_items:
|
|
56
56
|
log.message("Assembling metadata for input items:\n%s", fmt_lines(input_items))
|
|
57
57
|
input_items = [
|
|
58
|
-
|
|
58
|
+
fetch_url_item_content(item, refetch=refetch) if is_url_resource(item) else item
|
|
59
59
|
for item in input_items
|
|
60
60
|
]
|
|
61
61
|
|
|
@@ -11,7 +11,9 @@ from kash.utils.errors import InvalidInput
|
|
|
11
11
|
log = get_logger(__name__)
|
|
12
12
|
|
|
13
13
|
|
|
14
|
-
def
|
|
14
|
+
def fetch_url_item(
|
|
15
|
+
locator: Url | StorePath, *, save_content: bool = True, refetch: bool = False
|
|
16
|
+
) -> Item:
|
|
15
17
|
from kash.workspaces import current_ws
|
|
16
18
|
|
|
17
19
|
ws = current_ws()
|
|
@@ -26,16 +28,23 @@ def fetch_url_metadata(locator: Url | StorePath, refetch: bool = False) -> Item:
|
|
|
26
28
|
else:
|
|
27
29
|
raise InvalidInput(f"Not a URL or URL resource: {fmt_loc(locator)}")
|
|
28
30
|
|
|
29
|
-
return
|
|
31
|
+
return fetch_url_item_content(item, save_content=save_content, refetch=refetch)
|
|
30
32
|
|
|
31
33
|
|
|
32
|
-
def
|
|
34
|
+
def fetch_url_item_content(item: Item, *, save_content: bool = True, refetch: bool = False) -> Item:
|
|
33
35
|
"""
|
|
34
|
-
Fetch metadata for a URL using a media service if we
|
|
35
|
-
|
|
36
|
+
Fetch content and metadata for a URL using a media service if we
|
|
37
|
+
recognize the URL as a known media service. Otherwise, fetch and extract the
|
|
38
|
+
metadata and content from the web page and save it to the URL item.
|
|
39
|
+
|
|
40
|
+
If `save_content` is true, a copy of the content is also saved as
|
|
41
|
+
a resource item.
|
|
42
|
+
|
|
43
|
+
The content item is returned if content was saved. Otherwise, the updated
|
|
44
|
+
URL item is returned.
|
|
36
45
|
"""
|
|
37
46
|
from kash.web_content.canon_url import canonicalize_url
|
|
38
|
-
from kash.web_content.web_extract import
|
|
47
|
+
from kash.web_content.web_extract import fetch_page_content
|
|
39
48
|
from kash.workspaces import current_ws
|
|
40
49
|
|
|
41
50
|
ws = current_ws()
|
|
@@ -54,28 +63,47 @@ def fetch_url_item_metadata(item: Item, refetch: bool = False) -> Item:
|
|
|
54
63
|
# Prefer fetching metadata from media using the media service if possible.
|
|
55
64
|
# Data is cleaner and YouTube for example often blocks regular scraping.
|
|
56
65
|
media_metadata = get_media_metadata(url)
|
|
66
|
+
url_item: Item | None = None
|
|
67
|
+
content_item: Item | None = None
|
|
57
68
|
if media_metadata:
|
|
58
|
-
|
|
69
|
+
url_item = Item.from_media_metadata(media_metadata)
|
|
59
70
|
# Preserve and canonicalize any slice suffix on the URL.
|
|
60
71
|
_base_url, slice = parse_url_slice(item.url)
|
|
61
72
|
if slice:
|
|
62
73
|
new_url = add_slice_to_url(media_metadata.url, slice)
|
|
63
74
|
if new_url != item.url:
|
|
64
75
|
log.message("Updated URL from metadata and added slice: %s", new_url)
|
|
65
|
-
|
|
76
|
+
url_item.url = new_url
|
|
66
77
|
|
|
67
|
-
|
|
78
|
+
url_item = item.merged_copy(url_item)
|
|
68
79
|
else:
|
|
69
|
-
page_data =
|
|
70
|
-
|
|
80
|
+
page_data = fetch_page_content(url, refetch=refetch, cache=save_content)
|
|
81
|
+
url_item = item.new_copy_with(
|
|
71
82
|
title=page_data.title or item.title,
|
|
72
83
|
description=page_data.description or item.description,
|
|
73
84
|
thumbnail_url=page_data.thumbnail_url or item.thumbnail_url,
|
|
74
85
|
)
|
|
86
|
+
if save_content:
|
|
87
|
+
assert page_data.saved_content
|
|
88
|
+
assert page_data.format_info
|
|
89
|
+
content_item = url_item.new_copy_with(
|
|
90
|
+
external_path=str(page_data.saved_content),
|
|
91
|
+
# Use the original filename, not the local cache filename (which has a hash suffix).
|
|
92
|
+
original_filename=item.get_filename(),
|
|
93
|
+
format=page_data.format_info.format,
|
|
94
|
+
)
|
|
95
|
+
ws.save(content_item)
|
|
75
96
|
|
|
76
|
-
if not
|
|
97
|
+
if not url_item.title:
|
|
77
98
|
log.warning("Failed to fetch page data: title is missing: %s", item.url)
|
|
78
99
|
|
|
79
|
-
|
|
100
|
+
# Now save the updated URL item and also the content item if we have one.
|
|
101
|
+
ws.save(url_item)
|
|
102
|
+
assert url_item.store_path
|
|
103
|
+
log.debug("Saved URL item: %s", url_item.fmt_loc())
|
|
104
|
+
if content_item:
|
|
105
|
+
ws.save(content_item)
|
|
106
|
+
assert content_item.store_path
|
|
107
|
+
log.debug("Saved content item: %s", content_item.fmt_loc())
|
|
80
108
|
|
|
81
|
-
return
|
|
109
|
+
return content_item or url_item
|
kash/exec/llm_transforms.py
CHANGED
|
@@ -68,7 +68,7 @@ def llm_transform_str(options: LLMOptions, input_str: str, check_no_results: boo
|
|
|
68
68
|
diff_filter=options.diff_filter,
|
|
69
69
|
).reassemble()
|
|
70
70
|
else:
|
|
71
|
-
log.
|
|
71
|
+
log.info(
|
|
72
72
|
"Running simple LLM transform action %s with model %s",
|
|
73
73
|
options.op_name,
|
|
74
74
|
options.model.litellm_name,
|
|
@@ -56,7 +56,7 @@ class ShellCallableAction:
|
|
|
56
56
|
|
|
57
57
|
log.info("Action shell args: %s", shell_args)
|
|
58
58
|
explicit_values = RawParamValues(shell_args.options)
|
|
59
|
-
if not action.interactive_input:
|
|
59
|
+
if not action.interactive_input and not action.live_output:
|
|
60
60
|
with get_console().status(f"Running action {action.name}…", spinner=SPINNER):
|
|
61
61
|
result = run_action_with_shell_context(
|
|
62
62
|
action_cls,
|
kash/file_storage/file_store.py
CHANGED
|
@@ -405,6 +405,7 @@ class FileStore(Workspace):
|
|
|
405
405
|
# If external path already exists and is within the workspace, the file was
|
|
406
406
|
# already saved (e.g. by an action that wrote the item directly to the store).
|
|
407
407
|
external_path = item.external_path and Path(item.external_path).resolve()
|
|
408
|
+
skipped_save = False
|
|
408
409
|
if external_path and self._is_in_store(external_path):
|
|
409
410
|
log.info("Item with external_path already saved: %s", fmt_loc(external_path))
|
|
410
411
|
rel_path = external_path.relative_to(self.base_dir)
|
|
@@ -480,12 +481,17 @@ class FileStore(Workspace):
|
|
|
480
481
|
)
|
|
481
482
|
os.unlink(full_path)
|
|
482
483
|
store_path = old_store_path
|
|
484
|
+
skipped_save = True
|
|
483
485
|
|
|
484
486
|
# Update in-memory store_path only after successful save.
|
|
485
487
|
item.store_path = str(store_path)
|
|
486
488
|
self._id_index_item(store_path)
|
|
487
489
|
|
|
488
|
-
|
|
490
|
+
if not skipped_save:
|
|
491
|
+
log.message("%s Saved item: %s", EMOJI_SAVED, fmt_loc(store_path))
|
|
492
|
+
else:
|
|
493
|
+
log.info("%s Already saved: %s", EMOJI_SAVED, fmt_loc(store_path))
|
|
494
|
+
|
|
489
495
|
return store_path
|
|
490
496
|
|
|
491
497
|
@log_calls(level="debug")
|
|
@@ -30,6 +30,10 @@ def folder_for_type(item_type: ItemType) -> Path:
|
|
|
30
30
|
|
|
31
31
|
|
|
32
32
|
def join_suffix(base_slug: str, full_suffix: str) -> str:
|
|
33
|
+
"""
|
|
34
|
+
Create a store filename by joining a base slug and a full suffix, i.e. a filename
|
|
35
|
+
extension with or without an item type (`.html` or `.resource.html`, for example).
|
|
36
|
+
"""
|
|
33
37
|
return f"{base_slug}.{full_suffix.lstrip('.')}"
|
|
34
38
|
|
|
35
39
|
|
kash/help/function_param_info.py
CHANGED
|
@@ -2,9 +2,9 @@ from collections.abc import Callable
|
|
|
2
2
|
from dataclasses import replace
|
|
3
3
|
from typing import Any
|
|
4
4
|
|
|
5
|
-
from kash.help.docstring_utils import parse_docstring
|
|
6
5
|
from kash.model.params_model import ALL_COMMON_PARAMS, Param
|
|
7
6
|
from kash.utils.common.function_inspect import FuncParam, inspect_function_params
|
|
7
|
+
from kash.utils.common.parse_docstring import parse_docstring
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
def _look_up_param_docs(func: Callable[..., Any], kw_params: list[FuncParam]) -> list[Param]:
|
kash/help/help_pages.py
CHANGED
|
@@ -3,7 +3,6 @@ from rich.text import Text
|
|
|
3
3
|
from kash.config.logger import get_logger
|
|
4
4
|
from kash.config.text_styles import STYLE_HINT
|
|
5
5
|
from kash.docs.all_docs import DocSelection, all_docs
|
|
6
|
-
from kash.help.docstring_utils import parse_docstring
|
|
7
6
|
from kash.shell.output.shell_formatting import format_name_and_value
|
|
8
7
|
from kash.shell.output.shell_output import (
|
|
9
8
|
PrintHooks,
|
|
@@ -12,6 +11,7 @@ from kash.shell.output.shell_output import (
|
|
|
12
11
|
print_hrule,
|
|
13
12
|
print_markdown,
|
|
14
13
|
)
|
|
14
|
+
from kash.utils.common.parse_docstring import parse_docstring
|
|
15
15
|
|
|
16
16
|
log = get_logger(__name__)
|
|
17
17
|
|
kash/help/help_printing.py
CHANGED
|
@@ -6,7 +6,6 @@ from kash.docs.all_docs import DocSelection
|
|
|
6
6
|
from kash.exec.action_registry import look_up_action_class
|
|
7
7
|
from kash.exec.command_registry import CommandFunction, look_up_command
|
|
8
8
|
from kash.help.assistant import assist_preamble, assistance_unstructured
|
|
9
|
-
from kash.help.docstring_utils import parse_docstring
|
|
10
9
|
from kash.help.function_param_info import annotate_param_info
|
|
11
10
|
from kash.help.help_lookups import look_up_faq
|
|
12
11
|
from kash.help.tldr_help import tldr_help
|
|
@@ -22,6 +21,7 @@ from kash.shell.output.shell_output import (
|
|
|
22
21
|
print_help,
|
|
23
22
|
print_markdown,
|
|
24
23
|
)
|
|
24
|
+
from kash.utils.common.parse_docstring import parse_docstring
|
|
25
25
|
from kash.utils.errors import InvalidInput, NoMatch
|
|
26
26
|
from kash.utils.file_formats.chat_format import ChatHistory, ChatMessage, ChatRole
|
|
27
27
|
|
kash/llm_utils/llm_completion.py
CHANGED
|
@@ -173,7 +173,7 @@ def llm_template_completion(
|
|
|
173
173
|
)
|
|
174
174
|
|
|
175
175
|
if check_no_results and is_no_results(result.content):
|
|
176
|
-
log.
|
|
176
|
+
log.info("No results for LLM transform, will ignore: %r", result.content)
|
|
177
177
|
result.content = ""
|
|
178
178
|
|
|
179
179
|
return result
|
kash/model/actions_model.py
CHANGED
|
@@ -270,6 +270,12 @@ class Action(ABC):
|
|
|
270
270
|
Does this action ask for input interactively?
|
|
271
271
|
"""
|
|
272
272
|
|
|
273
|
+
live_output: bool = False
|
|
274
|
+
"""
|
|
275
|
+
Does this action have live output (e.g., progress bars, spinners)?
|
|
276
|
+
If True, the shell should not show its own status spinner.
|
|
277
|
+
"""
|
|
278
|
+
|
|
273
279
|
mcp_tool: bool = False
|
|
274
280
|
"""
|
|
275
281
|
If True, this action is published as an MCP tool.
|