kash-shell 0.3.12__py3-none-any.whl → 0.3.14__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kash/actions/core/markdownify.py +12 -8
- kash/actions/core/readability.py +8 -7
- kash/actions/core/render_as_html.py +8 -6
- kash/actions/core/show_webpage.py +2 -2
- kash/commands/base/basic_file_commands.py +3 -0
- kash/commands/base/diff_commands.py +38 -3
- kash/commands/base/reformat_command.py +1 -1
- kash/commands/base/show_command.py +1 -1
- kash/commands/workspace/selection_commands.py +1 -1
- kash/commands/workspace/workspace_commands.py +92 -29
- kash/docs/load_source_code.py +1 -1
- kash/exec/action_exec.py +6 -8
- kash/exec/fetch_url_metadata.py +8 -5
- kash/exec/importing.py +4 -4
- kash/exec/llm_transforms.py +1 -1
- kash/exec/preconditions.py +30 -10
- kash/file_storage/file_store.py +105 -43
- kash/file_storage/item_file_format.py +1 -1
- kash/file_storage/store_filenames.py +2 -1
- kash/help/help_embeddings.py +2 -2
- kash/llm_utils/clean_headings.py +1 -1
- kash/{text_handling → llm_utils}/custom_sliding_transforms.py +0 -3
- kash/llm_utils/llm_completion.py +1 -1
- kash/local_server/__init__.py +1 -1
- kash/local_server/local_server_commands.py +2 -1
- kash/mcp/__init__.py +1 -1
- kash/mcp/mcp_server_commands.py +8 -2
- kash/media_base/media_cache.py +10 -3
- kash/model/actions_model.py +3 -0
- kash/model/items_model.py +78 -44
- kash/model/operations_model.py +14 -0
- kash/shell/ui/shell_results.py +2 -1
- kash/shell/utils/native_utils.py +2 -2
- kash/utils/common/format_utils.py +0 -8
- kash/utils/common/import_utils.py +46 -18
- kash/utils/common/url.py +80 -3
- kash/utils/file_utils/file_formats.py +3 -2
- kash/utils/file_utils/file_formats_model.py +47 -45
- kash/utils/file_utils/filename_parsing.py +41 -16
- kash/{text_handling → utils/text_handling}/doc_normalization.py +10 -8
- kash/utils/text_handling/escape_html_tags.py +156 -0
- kash/{text_handling → utils/text_handling}/markdown_utils.py +0 -3
- kash/utils/text_handling/markdownify_utils.py +87 -0
- kash/{text_handling → utils/text_handling}/unified_diffs.py +1 -44
- kash/web_content/file_cache_utils.py +42 -34
- kash/web_content/local_file_cache.py +53 -13
- kash/web_content/web_extract.py +1 -1
- kash/web_content/web_extract_readabilipy.py +4 -2
- kash/web_content/web_fetch.py +42 -7
- kash/web_content/web_page_model.py +2 -1
- kash/web_gen/simple_webpage.py +1 -1
- kash/web_gen/templates/base_styles.css.jinja +134 -16
- kash/web_gen/templates/simple_webpage.html.jinja +1 -1
- kash/workspaces/selections.py +2 -2
- kash/workspaces/workspace_output.py +2 -2
- kash/xonsh_custom/load_into_xonsh.py +4 -2
- {kash_shell-0.3.12.dist-info → kash_shell-0.3.14.dist-info}/METADATA +1 -1
- {kash_shell-0.3.12.dist-info → kash_shell-0.3.14.dist-info}/RECORD +62 -62
- kash/utils/common/inflection.py +0 -22
- kash/workspaces/workspace_importing.py +0 -56
- /kash/{text_handling → utils/text_handling}/markdown_render.py +0 -0
- {kash_shell-0.3.12.dist-info → kash_shell-0.3.14.dist-info}/WHEEL +0 -0
- {kash_shell-0.3.12.dist-info → kash_shell-0.3.14.dist-info}/entry_points.txt +0 -0
- {kash_shell-0.3.12.dist-info → kash_shell-0.3.14.dist-info}/licenses/LICENSE +0 -0
kash/actions/core/markdownify.py
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
from kash.config.logger import get_logger
|
|
2
2
|
from kash.exec import kash_action
|
|
3
|
-
from kash.exec.preconditions import has_html_body,
|
|
3
|
+
from kash.exec.preconditions import has_html_body, is_url_resource
|
|
4
|
+
from kash.exec.runtime_settings import current_runtime_settings
|
|
4
5
|
from kash.model import Format, Item
|
|
5
|
-
from kash.model.
|
|
6
|
+
from kash.model.items_model import ItemType
|
|
7
|
+
from kash.utils.text_handling.markdownify_utils import markdownify_custom
|
|
6
8
|
from kash.web_content.file_cache_utils import get_url_html
|
|
7
9
|
from kash.web_content.web_extract_readabilipy import extract_text_readabilipy
|
|
8
10
|
|
|
@@ -10,21 +12,23 @@ log = get_logger(__name__)
|
|
|
10
12
|
|
|
11
13
|
|
|
12
14
|
@kash_action(
|
|
13
|
-
precondition=
|
|
14
|
-
params=common_params("refetch"),
|
|
15
|
+
precondition=is_url_resource | has_html_body,
|
|
15
16
|
mcp_tool=True,
|
|
16
17
|
)
|
|
17
|
-
def markdownify(item: Item
|
|
18
|
+
def markdownify(item: Item) -> Item:
|
|
18
19
|
"""
|
|
19
20
|
Converts a URL or raw HTML item to Markdown, fetching with the content
|
|
20
21
|
cache if needed. Also uses readability to clean up the HTML.
|
|
21
22
|
"""
|
|
22
|
-
from markdownify import markdownify as markdownify_convert
|
|
23
23
|
|
|
24
|
+
refetch = current_runtime_settings().refetch
|
|
24
25
|
expiration_sec = 0 if refetch else None
|
|
25
26
|
url, html_content = get_url_html(item, expiration_sec=expiration_sec)
|
|
26
27
|
page_data = extract_text_readabilipy(url, html_content)
|
|
27
|
-
|
|
28
|
+
assert page_data.clean_html
|
|
29
|
+
markdown_content = markdownify_custom(page_data.clean_html)
|
|
28
30
|
|
|
29
|
-
output_item = item.derived_copy(
|
|
31
|
+
output_item = item.derived_copy(
|
|
32
|
+
type=ItemType.doc, format=Format.markdown, body=markdown_content
|
|
33
|
+
)
|
|
30
34
|
return output_item
|
kash/actions/core/readability.py
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
from kash.config.logger import get_logger
|
|
2
2
|
from kash.exec import kash_action
|
|
3
|
-
from kash.exec.preconditions import has_html_body,
|
|
3
|
+
from kash.exec.preconditions import has_html_body, is_url_resource
|
|
4
|
+
from kash.exec.runtime_settings import current_runtime_settings
|
|
4
5
|
from kash.model import Format, Item
|
|
5
|
-
from kash.model.params_model import common_params
|
|
6
6
|
from kash.web_content.file_cache_utils import get_url_html
|
|
7
7
|
from kash.web_content.web_extract_readabilipy import extract_text_readabilipy
|
|
8
8
|
|
|
@@ -10,18 +10,19 @@ log = get_logger(__name__)
|
|
|
10
10
|
|
|
11
11
|
|
|
12
12
|
@kash_action(
|
|
13
|
-
precondition=
|
|
14
|
-
params=common_params("refetch"),
|
|
13
|
+
precondition=is_url_resource | has_html_body,
|
|
15
14
|
mcp_tool=True,
|
|
16
15
|
)
|
|
17
|
-
def readability(item: Item
|
|
16
|
+
def readability(item: Item) -> Item:
|
|
18
17
|
"""
|
|
19
18
|
Extracts clean HTML from a raw HTML item.
|
|
20
19
|
See `markdownify` to also convert to Markdown.
|
|
21
20
|
"""
|
|
21
|
+
|
|
22
|
+
refetch = current_runtime_settings().refetch
|
|
22
23
|
expiration_sec = 0 if refetch else None
|
|
23
|
-
|
|
24
|
-
page_data = extract_text_readabilipy(
|
|
24
|
+
locator, html_content = get_url_html(item, expiration_sec=expiration_sec)
|
|
25
|
+
page_data = extract_text_readabilipy(locator, html_content)
|
|
25
26
|
|
|
26
27
|
output_item = item.derived_copy(format=Format.html, body=page_data.clean_html)
|
|
27
28
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from kash.actions.core.tabbed_webpage_config import tabbed_webpage_config
|
|
2
2
|
from kash.actions.core.tabbed_webpage_generate import tabbed_webpage_generate
|
|
3
3
|
from kash.exec import kash_action
|
|
4
|
-
from kash.exec.preconditions import
|
|
4
|
+
from kash.exec.preconditions import has_fullpage_html_body, has_html_body, has_simple_text_body
|
|
5
5
|
from kash.exec_model.args_model import ONE_OR_MORE_ARGS
|
|
6
6
|
from kash.model import ActionInput, ActionResult, Param
|
|
7
7
|
from kash.model.items_model import ItemType
|
|
@@ -11,10 +11,10 @@ from kash.web_gen.simple_webpage import simple_webpage_render
|
|
|
11
11
|
|
|
12
12
|
@kash_action(
|
|
13
13
|
expected_args=ONE_OR_MORE_ARGS,
|
|
14
|
-
precondition=(has_html_body | has_simple_text_body) & ~
|
|
15
|
-
params=(Param("
|
|
14
|
+
precondition=(has_html_body | has_simple_text_body) & ~has_fullpage_html_body,
|
|
15
|
+
params=(Param("no_title", "Don't add a title to the page body.", type=bool),),
|
|
16
16
|
)
|
|
17
|
-
def render_as_html(input: ActionInput,
|
|
17
|
+
def render_as_html(input: ActionInput, no_title: bool = False) -> ActionResult:
|
|
18
18
|
"""
|
|
19
19
|
Convert text, Markdown, or HTML to pretty, formatted HTML using a clean
|
|
20
20
|
and simple page template. Supports GFM-flavored Markdown tables and footnotes.
|
|
@@ -27,11 +27,13 @@ def render_as_html(input: ActionInput, add_title: bool = False) -> ActionResult:
|
|
|
27
27
|
"""
|
|
28
28
|
if len(input.items) == 1:
|
|
29
29
|
input_item = input.items[0]
|
|
30
|
-
html_body = simple_webpage_render(input_item, add_title_h1=
|
|
30
|
+
html_body = simple_webpage_render(input_item, add_title_h1=not no_title)
|
|
31
31
|
result_item = input_item.derived_copy(
|
|
32
32
|
type=ItemType.export, format=Format.html, body=html_body
|
|
33
33
|
)
|
|
34
34
|
return ActionResult([result_item])
|
|
35
35
|
else:
|
|
36
36
|
config_result = tabbed_webpage_config(input)
|
|
37
|
-
return tabbed_webpage_generate(
|
|
37
|
+
return tabbed_webpage_generate(
|
|
38
|
+
ActionInput(items=config_result.items), add_title=not no_title
|
|
39
|
+
)
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from kash.actions.core.render_as_html import render_as_html
|
|
2
2
|
from kash.commands.base.show_command import show
|
|
3
3
|
from kash.exec import kash_action
|
|
4
|
-
from kash.exec.preconditions import
|
|
4
|
+
from kash.exec.preconditions import has_fullpage_html_body, has_html_body, has_simple_text_body
|
|
5
5
|
from kash.exec_model.args_model import ONE_OR_MORE_ARGS
|
|
6
6
|
from kash.exec_model.commands_model import Command
|
|
7
7
|
from kash.exec_model.shell_model import ShellResult
|
|
@@ -10,7 +10,7 @@ from kash.model import ActionInput, ActionResult
|
|
|
10
10
|
|
|
11
11
|
@kash_action(
|
|
12
12
|
expected_args=ONE_OR_MORE_ARGS,
|
|
13
|
-
precondition=(has_html_body | has_simple_text_body) & ~
|
|
13
|
+
precondition=(has_html_body | has_simple_text_body) & ~has_fullpage_html_body,
|
|
14
14
|
)
|
|
15
15
|
def show_webpage(input: ActionInput) -> ActionResult:
|
|
16
16
|
"""
|
|
@@ -81,6 +81,9 @@ def clipboard_paste(path: str = "untitled_paste.txt") -> None:
|
|
|
81
81
|
import pyperclip
|
|
82
82
|
|
|
83
83
|
contents = pyperclip.paste()
|
|
84
|
+
if not contents.strip():
|
|
85
|
+
raise InvalidInput("Clipboard is empty")
|
|
86
|
+
|
|
84
87
|
with atomic_output_file(path, backup_suffix=".{timestamp}.bak") as f:
|
|
85
88
|
f.write_text(contents)
|
|
86
89
|
|
|
@@ -5,16 +5,51 @@ from kash.commands.workspace.selection_commands import select
|
|
|
5
5
|
from kash.config.logger import get_logger
|
|
6
6
|
from kash.exec import import_locator_args, kash_command
|
|
7
7
|
from kash.exec_model.shell_model import ShellResult
|
|
8
|
-
from kash.model.items_model import Item, ItemType
|
|
8
|
+
from kash.model.items_model import Item, ItemRelations, ItemType
|
|
9
|
+
from kash.model.paths_model import StorePath
|
|
9
10
|
from kash.shell.output.shell_output import Wrap, cprint
|
|
10
|
-
from kash.
|
|
11
|
-
from kash.utils.errors import InvalidInput, InvalidOperation
|
|
11
|
+
from kash.utils.errors import ContentError, InvalidInput, InvalidOperation
|
|
12
12
|
from kash.utils.file_utils.file_formats_model import Format
|
|
13
|
+
from kash.utils.text_handling.unified_diffs import unified_diff, unified_diff_files
|
|
13
14
|
from kash.workspaces import current_ws
|
|
14
15
|
|
|
15
16
|
log = get_logger(__name__)
|
|
16
17
|
|
|
17
18
|
|
|
19
|
+
def unified_diff_items(from_item: Item, to_item: Item, strict: bool = True) -> Item:
|
|
20
|
+
"""
|
|
21
|
+
Generate a unified diff between two items. If `strict` is true, will raise
|
|
22
|
+
an error if the items are of different formats.
|
|
23
|
+
"""
|
|
24
|
+
if not from_item.body and not to_item.body:
|
|
25
|
+
raise ContentError(f"No body to diff for {from_item} and {to_item}")
|
|
26
|
+
if not from_item.store_path or not to_item.store_path:
|
|
27
|
+
raise ContentError("No store path on items; save before diffing")
|
|
28
|
+
diff_items = [item for item in [from_item, to_item] if item.format == Format.diff]
|
|
29
|
+
if len(diff_items) == 1:
|
|
30
|
+
raise ContentError(
|
|
31
|
+
f"Cannot compare diffs to non-diffs: {from_item.format}, {to_item.format}"
|
|
32
|
+
)
|
|
33
|
+
if len(diff_items) > 0 or from_item.format != to_item.format:
|
|
34
|
+
msg = f"Diffing items of incompatible format: {from_item.format}, {to_item.format}"
|
|
35
|
+
if strict:
|
|
36
|
+
raise ContentError(msg)
|
|
37
|
+
else:
|
|
38
|
+
log.warning("%s", msg)
|
|
39
|
+
|
|
40
|
+
from_path, to_path = StorePath(from_item.store_path), StorePath(to_item.store_path)
|
|
41
|
+
|
|
42
|
+
diff = unified_diff(from_item.body, to_item.body, str(from_path), str(to_path))
|
|
43
|
+
|
|
44
|
+
return Item(
|
|
45
|
+
type=ItemType.doc,
|
|
46
|
+
title=f"Diff of {from_path} and {to_path}",
|
|
47
|
+
format=Format.diff,
|
|
48
|
+
relations=ItemRelations(diff_of=[from_path, to_path]),
|
|
49
|
+
body=diff.patch_text,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
18
53
|
@kash_command
|
|
19
54
|
def diff_items(*paths: str, force: bool = False) -> ShellResult:
|
|
20
55
|
"""
|
|
@@ -8,9 +8,9 @@ from kash.config.logger import get_logger
|
|
|
8
8
|
from kash.exec import assemble_path_args, kash_command, resolvable_paths
|
|
9
9
|
from kash.exec_model.shell_model import ShellResult
|
|
10
10
|
from kash.shell.output.shell_output import print_status
|
|
11
|
-
from kash.text_handling.doc_normalization import normalize_text_file
|
|
12
11
|
from kash.utils.common.format_utils import fmt_loc
|
|
13
12
|
from kash.utils.file_utils.filename_parsing import join_filename, split_filename
|
|
13
|
+
from kash.utils.text_handling.doc_normalization import normalize_text_file
|
|
14
14
|
|
|
15
15
|
log = get_logger(__name__)
|
|
16
16
|
|
|
@@ -54,7 +54,7 @@ def show(
|
|
|
54
54
|
item = ws.load(input_path)
|
|
55
55
|
if thumbnail and item.thumbnail_url:
|
|
56
56
|
try:
|
|
57
|
-
local_path
|
|
57
|
+
local_path = cache_file(item.thumbnail_url).content.path
|
|
58
58
|
terminal_show_image(local_path)
|
|
59
59
|
except Exception as e:
|
|
60
60
|
log.info("Had trouble showing thumbnail image (will skip): %s", e)
|
|
@@ -2,6 +2,7 @@ from os.path import basename
|
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
|
|
4
4
|
from frontmatter_format import fmf_strip_frontmatter
|
|
5
|
+
from prettyfmt import plural
|
|
5
6
|
from strif import copyfile_atomic
|
|
6
7
|
|
|
7
8
|
from kash.config.logger import get_logger
|
|
@@ -10,7 +11,6 @@ from kash.exec_model.shell_model import ShellResult
|
|
|
10
11
|
from kash.model.paths_model import StorePath
|
|
11
12
|
from kash.shell.ui.shell_results import shell_print_selection_history
|
|
12
13
|
from kash.utils.common.format_utils import fmt_loc
|
|
13
|
-
from kash.utils.common.inflection import plural
|
|
14
14
|
from kash.utils.errors import InvalidInput
|
|
15
15
|
from kash.workspaces import Selection, current_ws
|
|
16
16
|
|
|
@@ -2,7 +2,7 @@ import os
|
|
|
2
2
|
from pathlib import Path
|
|
3
3
|
|
|
4
4
|
from frontmatter_format import to_yaml_string
|
|
5
|
-
from prettyfmt import fmt_lines
|
|
5
|
+
from prettyfmt import fmt_lines, plural
|
|
6
6
|
from rich.text import Text
|
|
7
7
|
|
|
8
8
|
from kash.commands.base.basic_file_commands import trash
|
|
@@ -26,7 +26,7 @@ from kash.exec.action_registry import get_all_actions_defaults
|
|
|
26
26
|
from kash.exec.fetch_url_metadata import fetch_url_metadata
|
|
27
27
|
from kash.exec.precondition_checks import actions_matching_paths
|
|
28
28
|
from kash.exec.precondition_registry import get_all_preconditions
|
|
29
|
-
from kash.exec.preconditions import
|
|
29
|
+
from kash.exec.preconditions import is_url_resource
|
|
30
30
|
from kash.exec_model.shell_model import ShellResult
|
|
31
31
|
from kash.local_server.local_url_formatters import local_url_formatter
|
|
32
32
|
from kash.media_base import media_tools
|
|
@@ -51,14 +51,14 @@ from kash.shell.output.shell_output import (
|
|
|
51
51
|
)
|
|
52
52
|
from kash.shell.utils.native_utils import tail_file
|
|
53
53
|
from kash.utils.common.format_utils import fmt_loc
|
|
54
|
-
from kash.utils.common.inflection import plural
|
|
55
54
|
from kash.utils.common.obj_replace import remove_values
|
|
56
55
|
from kash.utils.common.parse_key_vals import parse_key_value
|
|
57
56
|
from kash.utils.common.type_utils import not_none
|
|
58
|
-
from kash.utils.common.url import Url, is_url
|
|
57
|
+
from kash.utils.common.url import Url, is_url, parse_http_url
|
|
59
58
|
from kash.utils.errors import InvalidInput
|
|
60
59
|
from kash.utils.file_formats.chat_format import tail_chat_history
|
|
61
60
|
from kash.utils.file_utils.dir_info import is_nonempty_dir
|
|
61
|
+
from kash.utils.file_utils.file_formats_model import Format
|
|
62
62
|
from kash.web_content.file_cache_utils import cache_file
|
|
63
63
|
from kash.workspaces import (
|
|
64
64
|
current_ws,
|
|
@@ -181,48 +181,85 @@ def cache_content(*urls_or_paths: str, refetch: bool = False) -> None:
|
|
|
181
181
|
PrintHooks.spacer()
|
|
182
182
|
for url_or_path in urls_or_paths:
|
|
183
183
|
locator = resolve_locator_arg(url_or_path)
|
|
184
|
-
|
|
185
|
-
cache_str = " (already cached)" if was_cached else ""
|
|
184
|
+
cache_result = cache_file(locator, expiration_sec=expiration_sec)
|
|
185
|
+
cache_str = " (already cached)" if cache_result.was_cached else ""
|
|
186
186
|
cprint(f"{fmt_loc(url_or_path)}{cache_str}:", style=STYLE_EMPH, text_wrap=Wrap.NONE)
|
|
187
|
-
cprint(f"{
|
|
187
|
+
cprint(f"{cache_result.content.path}", text_wrap=Wrap.INDENT_ONLY)
|
|
188
188
|
PrintHooks.spacer()
|
|
189
189
|
|
|
190
190
|
|
|
191
191
|
@kash_command
|
|
192
|
-
def download(*urls_or_paths: str, refetch: bool = False) ->
|
|
192
|
+
def download(*urls_or_paths: str, refetch: bool = False) -> ShellResult:
|
|
193
193
|
"""
|
|
194
194
|
Download a URL or resource. Uses cached content if available, unless `refetch` is true.
|
|
195
195
|
Inputs can be URLs or paths to URL resources.
|
|
196
|
+
Creates both resource and document versions for text content.
|
|
196
197
|
"""
|
|
197
|
-
expiration_sec = 0 if refetch else None
|
|
198
|
-
|
|
199
|
-
# TODO: Add option to include frontmatter metadata for text files.
|
|
200
198
|
ws = current_ws()
|
|
199
|
+
saved_paths = []
|
|
200
|
+
|
|
201
201
|
for url_or_path in urls_or_paths:
|
|
202
202
|
locator = resolve_locator_arg(url_or_path)
|
|
203
|
-
url = None
|
|
203
|
+
url: Url | None = None
|
|
204
|
+
|
|
205
|
+
# Get the URL from the locator
|
|
204
206
|
if not isinstance(locator, Path) and is_url(locator):
|
|
205
207
|
url = Url(locator)
|
|
206
|
-
|
|
208
|
+
elif isinstance(locator, StorePath):
|
|
207
209
|
url_item = ws.load(locator)
|
|
208
|
-
if
|
|
210
|
+
if is_url_resource(url_item):
|
|
209
211
|
url = url_item.url
|
|
212
|
+
|
|
210
213
|
if not url:
|
|
211
214
|
raise InvalidInput(f"Not a URL or URL resource: {fmt_loc(locator)}")
|
|
212
215
|
|
|
216
|
+
# Handle media URLs differently
|
|
213
217
|
if is_media_url(url):
|
|
214
|
-
store_path = ws.import_item(locator, as_type=ItemType.resource)
|
|
215
218
|
log.message(
|
|
216
|
-
"URL is a media URL, so
|
|
219
|
+
"URL is a media URL, so adding as a resource and will cache media: %s", fmt_loc(url)
|
|
217
220
|
)
|
|
221
|
+
store_path = ws.import_item(url, as_type=ItemType.resource, reimport=refetch)
|
|
222
|
+
saved_paths.append(store_path)
|
|
218
223
|
media_tools.cache_media(url)
|
|
219
224
|
else:
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
225
|
+
# Cache the content first
|
|
226
|
+
expiration_sec = 0 if refetch else None
|
|
227
|
+
cache_result = cache_file(url, expiration_sec=expiration_sec)
|
|
228
|
+
original_filename = Path(parse_http_url(url).path).name
|
|
229
|
+
mime_type = cache_result.content.headers and cache_result.content.headers.mime_type
|
|
230
|
+
|
|
231
|
+
# Create a resource item
|
|
232
|
+
resource_item = Item.from_external_path(
|
|
233
|
+
cache_result.content.path,
|
|
234
|
+
ItemType.resource,
|
|
235
|
+
url=url,
|
|
236
|
+
mime_type=mime_type,
|
|
237
|
+
original_filename=original_filename,
|
|
238
|
+
)
|
|
239
|
+
store_path = ws.save(resource_item, no_frontmatter=True, no_format=True)
|
|
240
|
+
saved_paths.append(store_path)
|
|
241
|
+
|
|
242
|
+
# Also create a doc version for text content
|
|
243
|
+
if resource_item.format and resource_item.format.supports_frontmatter:
|
|
244
|
+
doc_item = Item.from_external_path(
|
|
245
|
+
cache_result.content.path,
|
|
246
|
+
ItemType.doc,
|
|
247
|
+
url=url,
|
|
248
|
+
mime_type=mime_type,
|
|
249
|
+
original_filename=original_filename,
|
|
250
|
+
)
|
|
251
|
+
doc_store_path = ws.save(doc_item, no_frontmatter=False, no_format=False)
|
|
252
|
+
saved_paths.append(doc_store_path)
|
|
224
253
|
|
|
225
|
-
|
|
254
|
+
print_status(
|
|
255
|
+
"Downloaded %s %s:\n%s",
|
|
256
|
+
len(saved_paths),
|
|
257
|
+
plural("item", len(saved_paths)),
|
|
258
|
+
fmt_lines(saved_paths),
|
|
259
|
+
)
|
|
260
|
+
select(*saved_paths)
|
|
261
|
+
|
|
262
|
+
return ShellResult(show_selection=True)
|
|
226
263
|
|
|
227
264
|
|
|
228
265
|
@kash_command
|
|
@@ -437,7 +474,7 @@ def import_item(
|
|
|
437
474
|
*files_or_urls: str, type: ItemType | None = None, inplace: bool = False
|
|
438
475
|
) -> ShellResult:
|
|
439
476
|
"""
|
|
440
|
-
Add a file or URL resource to the workspace as an item
|
|
477
|
+
Add a file or URL resource to the workspace as an item.
|
|
441
478
|
|
|
442
479
|
:param inplace: If set and the item is already in the store, reimport the item,
|
|
443
480
|
adding or rewriting metadata frontmatter.
|
|
@@ -464,6 +501,34 @@ def import_item(
|
|
|
464
501
|
return ShellResult(show_selection=True)
|
|
465
502
|
|
|
466
503
|
|
|
504
|
+
@kash_command
|
|
505
|
+
def save_clipboard(
|
|
506
|
+
title: str | None = "pasted_text",
|
|
507
|
+
type: ItemType = ItemType.resource,
|
|
508
|
+
format: Format = Format.plaintext,
|
|
509
|
+
) -> ShellResult:
|
|
510
|
+
"""
|
|
511
|
+
Import the contents of the OS-native clipboard as a new item in the workspace.
|
|
512
|
+
|
|
513
|
+
:param title: The title of the new item (default: "pasted_text").
|
|
514
|
+
:param type: The type of the new item (default: resource).
|
|
515
|
+
:param format: The format of the new item (default: plaintext).
|
|
516
|
+
"""
|
|
517
|
+
import pyperclip
|
|
518
|
+
|
|
519
|
+
contents = pyperclip.paste()
|
|
520
|
+
if not contents.strip():
|
|
521
|
+
raise InvalidInput("Clipboard is empty")
|
|
522
|
+
|
|
523
|
+
ws = current_ws()
|
|
524
|
+
store_path = ws.save(Item(type=type, format=format, title=title, body=contents))
|
|
525
|
+
|
|
526
|
+
print_status("Imported clipboard contents to:\n%s", fmt_lines([fmt_loc(store_path)]))
|
|
527
|
+
select(store_path)
|
|
528
|
+
|
|
529
|
+
return ShellResult(show_selection=True)
|
|
530
|
+
|
|
531
|
+
|
|
467
532
|
@kash_command
|
|
468
533
|
def fetch_metadata(*files_or_urls: str, refetch: bool = False) -> ShellResult:
|
|
469
534
|
"""
|
|
@@ -472,8 +537,6 @@ def fetch_metadata(*files_or_urls: str, refetch: bool = False) -> ShellResult:
|
|
|
472
537
|
|
|
473
538
|
Skips items that already have a title and description, unless `refetch` is true.
|
|
474
539
|
Skips (with a warning) items that are not URL resources.
|
|
475
|
-
|
|
476
|
-
:param use_cache: If true, also save page in content cache.
|
|
477
540
|
"""
|
|
478
541
|
if not files_or_urls:
|
|
479
542
|
locators = assemble_store_path_args()
|
|
@@ -483,12 +546,12 @@ def fetch_metadata(*files_or_urls: str, refetch: bool = False) -> ShellResult:
|
|
|
483
546
|
store_paths = []
|
|
484
547
|
for locator in locators:
|
|
485
548
|
try:
|
|
486
|
-
if isinstance(locator, Path):
|
|
487
|
-
raise InvalidInput()
|
|
488
549
|
fetched_item = fetch_url_metadata(locator, refetch=refetch)
|
|
489
550
|
store_paths.append(fetched_item.store_path)
|
|
490
|
-
except InvalidInput:
|
|
491
|
-
log.warning(
|
|
551
|
+
except InvalidInput as e:
|
|
552
|
+
log.warning(
|
|
553
|
+
"Not a URL or URL resource, will not fetch metadata: %s: %s", fmt_loc(locator), e
|
|
554
|
+
)
|
|
492
555
|
|
|
493
556
|
if store_paths:
|
|
494
557
|
select(*store_paths)
|
|
@@ -670,7 +733,7 @@ def reset_ignore_file(append: bool = False) -> None:
|
|
|
670
733
|
ignore_path = ws.base_dir / ws.dirs.ignore_file
|
|
671
734
|
write_ignore(ignore_path, append=append)
|
|
672
735
|
|
|
673
|
-
log.message("
|
|
736
|
+
log.message("Rewritten kash ignore file: %s", fmt_loc(ignore_path))
|
|
674
737
|
|
|
675
738
|
|
|
676
739
|
@kash_command
|
kash/docs/load_source_code.py
CHANGED
|
@@ -104,7 +104,7 @@ def load_source_code() -> SourceCode:
|
|
|
104
104
|
kash_base_path / "model" / "assistant_response_model.py",
|
|
105
105
|
),
|
|
106
106
|
text_tool_src=read_source_code(
|
|
107
|
-
kash_base_path / "text_handling",
|
|
107
|
+
kash_base_path / "utils" / "text_handling",
|
|
108
108
|
kash_base_path / "utils" / "lang_utils",
|
|
109
109
|
# TODO: Include additional dep libs like chopdiff TextDoc too?
|
|
110
110
|
),
|
kash/exec/action_exec.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import time
|
|
2
2
|
from dataclasses import replace
|
|
3
3
|
|
|
4
|
-
from prettyfmt import fmt_lines
|
|
4
|
+
from prettyfmt import fmt_lines, plural
|
|
5
5
|
|
|
6
6
|
from kash.config.logger import get_logger
|
|
7
7
|
from kash.config.text_styles import (
|
|
@@ -10,7 +10,7 @@ from kash.config.text_styles import (
|
|
|
10
10
|
EMOJI_SUCCESS,
|
|
11
11
|
EMOJI_TIMING,
|
|
12
12
|
)
|
|
13
|
-
from kash.exec.preconditions import
|
|
13
|
+
from kash.exec.preconditions import is_url_resource
|
|
14
14
|
from kash.exec.resolve_args import assemble_action_args
|
|
15
15
|
from kash.exec_model.args_model import CommandArg
|
|
16
16
|
from kash.file_storage.file_store import FileStore
|
|
@@ -28,12 +28,10 @@ from kash.model.operations_model import Input, Operation, Source
|
|
|
28
28
|
from kash.model.params_model import ALL_COMMON_PARAMS, GLOBAL_PARAMS, RawParamValues
|
|
29
29
|
from kash.model.paths_model import StorePath
|
|
30
30
|
from kash.shell.output.shell_output import PrintHooks
|
|
31
|
-
from kash.utils.common.inflection import plural
|
|
32
31
|
from kash.utils.common.task_stack import task_stack
|
|
33
32
|
from kash.utils.common.type_utils import not_none
|
|
34
33
|
from kash.utils.errors import ContentError, InvalidOutput, get_nonfatal_exceptions
|
|
35
34
|
from kash.workspaces import Selection, current_ws
|
|
36
|
-
from kash.workspaces.workspace_importing import import_and_load
|
|
37
35
|
|
|
38
36
|
log = get_logger(__name__)
|
|
39
37
|
|
|
@@ -50,13 +48,13 @@ def prepare_action_input(*input_args: CommandArg, refetch: bool = False) -> Acti
|
|
|
50
48
|
|
|
51
49
|
# Ensure input items are already saved in the workspace and load the corresponding items.
|
|
52
50
|
# This also imports any URLs.
|
|
53
|
-
input_items = [import_and_load(
|
|
51
|
+
input_items = [ws.import_and_load(arg) for arg in input_args]
|
|
54
52
|
|
|
55
53
|
# URLs should have metadata like a title and be valid, so we fetch them.
|
|
56
54
|
if input_items:
|
|
57
55
|
log.message("Assembling metadata for input items:\n%s", fmt_lines(input_items))
|
|
58
56
|
input_items = [
|
|
59
|
-
fetch_url_item_metadata(item, refetch=refetch) if
|
|
57
|
+
fetch_url_item_metadata(item, refetch=refetch) if is_url_resource(item) else item
|
|
60
58
|
for item in input_items
|
|
61
59
|
]
|
|
62
60
|
|
|
@@ -299,7 +297,7 @@ def save_action_result(
|
|
|
299
297
|
skipped_paths.append(store_path)
|
|
300
298
|
continue
|
|
301
299
|
|
|
302
|
-
ws.save(item, as_tmp=as_tmp, no_format=no_format)
|
|
300
|
+
ws.save(item, overwrite=result.overwrite, as_tmp=as_tmp, no_format=no_format)
|
|
303
301
|
|
|
304
302
|
if skipped_paths:
|
|
305
303
|
log.message(
|
|
@@ -384,7 +382,7 @@ def run_action_with_caching(
|
|
|
384
382
|
|
|
385
383
|
PrintHooks.before_done_message()
|
|
386
384
|
log.message(
|
|
387
|
-
"%s
|
|
385
|
+
"%s Action: `%s` completed with %s %s",
|
|
388
386
|
EMOJI_SUCCESS,
|
|
389
387
|
action.name,
|
|
390
388
|
len(result.items),
|
kash/exec/fetch_url_metadata.py
CHANGED
|
@@ -1,19 +1,18 @@
|
|
|
1
1
|
from kash.config.logger import get_logger
|
|
2
|
-
from kash.exec.preconditions import
|
|
2
|
+
from kash.exec.preconditions import is_url_resource
|
|
3
3
|
from kash.media_base.media_services import get_media_metadata
|
|
4
4
|
from kash.model.items_model import Item, ItemType
|
|
5
5
|
from kash.model.paths_model import StorePath
|
|
6
6
|
from kash.utils.common.format_utils import fmt_loc
|
|
7
7
|
from kash.utils.common.url import Url, is_url
|
|
8
8
|
from kash.utils.errors import InvalidInput
|
|
9
|
-
from kash.web_content.canon_url import canonicalize_url
|
|
10
|
-
from kash.web_content.web_extract import fetch_extract
|
|
11
|
-
from kash.workspaces import current_ws
|
|
12
9
|
|
|
13
10
|
log = get_logger(__name__)
|
|
14
11
|
|
|
15
12
|
|
|
16
13
|
def fetch_url_metadata(locator: Url | StorePath, refetch: bool = False) -> Item:
|
|
14
|
+
from kash.workspaces import current_ws
|
|
15
|
+
|
|
17
16
|
ws = current_ws()
|
|
18
17
|
if is_url(locator):
|
|
19
18
|
# Import or find URL as a resource in the current workspace.
|
|
@@ -21,7 +20,7 @@ def fetch_url_metadata(locator: Url | StorePath, refetch: bool = False) -> Item:
|
|
|
21
20
|
item = ws.load(store_path)
|
|
22
21
|
elif isinstance(locator, StorePath):
|
|
23
22
|
item = ws.load(locator)
|
|
24
|
-
if not
|
|
23
|
+
if not is_url_resource(item):
|
|
25
24
|
raise InvalidInput(f"Not a URL resource: {fmt_loc(locator)}")
|
|
26
25
|
else:
|
|
27
26
|
raise InvalidInput(f"Not a URL or URL resource: {fmt_loc(locator)}")
|
|
@@ -34,6 +33,10 @@ def fetch_url_item_metadata(item: Item, refetch: bool = False) -> Item:
|
|
|
34
33
|
Fetch metadata for a URL using a media service if we recognize the URL,
|
|
35
34
|
and otherwise fetching and extracting it from the web page HTML.
|
|
36
35
|
"""
|
|
36
|
+
from kash.web_content.canon_url import canonicalize_url
|
|
37
|
+
from kash.web_content.web_extract import fetch_extract
|
|
38
|
+
from kash.workspaces import current_ws
|
|
39
|
+
|
|
37
40
|
ws = current_ws()
|
|
38
41
|
if not refetch and item.title and item.description:
|
|
39
42
|
log.message(
|
kash/exec/importing.py
CHANGED
|
@@ -5,7 +5,7 @@ from prettyfmt import fmt_lines, fmt_path
|
|
|
5
5
|
from kash.config.logger import get_logger
|
|
6
6
|
from kash.exec.action_registry import action_classes, refresh_action_classes
|
|
7
7
|
from kash.exec.command_registry import get_all_commands
|
|
8
|
-
from kash.utils.common.import_utils import Tallies,
|
|
8
|
+
from kash.utils.common.import_utils import Tallies, import_recursive
|
|
9
9
|
|
|
10
10
|
log = get_logger(__name__)
|
|
11
11
|
|
|
@@ -13,12 +13,12 @@ log = get_logger(__name__)
|
|
|
13
13
|
def import_and_register(
|
|
14
14
|
package_name: str | None,
|
|
15
15
|
parent_dir: Path,
|
|
16
|
-
|
|
16
|
+
resource_names: list[str] | None = None,
|
|
17
17
|
tallies: Tallies | None = None,
|
|
18
18
|
):
|
|
19
19
|
"""
|
|
20
20
|
This hook can be used for auto-registering commands and actions from any
|
|
21
|
-
subdirectory of a given package.
|
|
21
|
+
module or subdirectory of a given package.
|
|
22
22
|
|
|
23
23
|
Useful to call from `__init__.py` files to import a directory of code,
|
|
24
24
|
auto-registering annotated commands and actions and also handles refreshing the
|
|
@@ -38,7 +38,7 @@ def import_and_register(
|
|
|
38
38
|
prev_command_count = len(get_all_commands())
|
|
39
39
|
prev_action_count = len(ac)
|
|
40
40
|
|
|
41
|
-
|
|
41
|
+
import_recursive(package_name, parent_dir, resource_names, tallies)
|
|
42
42
|
|
|
43
43
|
new_command_count = len(get_all_commands()) - prev_command_count
|
|
44
44
|
new_action_count = len(ac) - prev_action_count
|
kash/exec/llm_transforms.py
CHANGED
|
@@ -13,9 +13,9 @@ from kash.llm_utils.llm_completion import llm_template_completion
|
|
|
13
13
|
from kash.llm_utils.llm_messages import Message, MessageTemplate
|
|
14
14
|
from kash.model.actions_model import LLMOptions
|
|
15
15
|
from kash.model.items_model import Item
|
|
16
|
-
from kash.text_handling.doc_normalization import normalize_formatting
|
|
17
16
|
from kash.utils.errors import InvalidInput
|
|
18
17
|
from kash.utils.file_utils.file_formats_model import Format
|
|
18
|
+
from kash.utils.text_handling.doc_normalization import normalize_formatting
|
|
19
19
|
|
|
20
20
|
log = get_logger(__name__)
|
|
21
21
|
|