kash-shell 0.3.8__py3-none-any.whl → 0.3.10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kash/actions/__init__.py +4 -4
- kash/actions/core/markdownify.py +5 -2
- kash/actions/core/readability.py +5 -2
- kash/actions/core/render_as_html.py +18 -0
- kash/actions/core/webpage_config.py +12 -4
- kash/commands/__init__.py +8 -20
- kash/commands/base/basic_file_commands.py +15 -0
- kash/commands/base/debug_commands.py +15 -2
- kash/commands/base/general_commands.py +27 -18
- kash/commands/base/logs_commands.py +1 -4
- kash/commands/base/model_commands.py +8 -8
- kash/commands/base/search_command.py +3 -2
- kash/commands/base/show_command.py +5 -3
- kash/commands/extras/parse_uv_lock.py +186 -0
- kash/commands/help/doc_commands.py +2 -31
- kash/commands/help/welcome.py +33 -0
- kash/commands/workspace/selection_commands.py +11 -6
- kash/commands/workspace/workspace_commands.py +19 -16
- kash/config/colors.py +2 -0
- kash/config/env_settings.py +72 -0
- kash/config/init.py +2 -2
- kash/config/logger.py +61 -59
- kash/config/logger_basic.py +12 -5
- kash/config/server_config.py +6 -6
- kash/config/settings.py +117 -67
- kash/config/setup.py +35 -9
- kash/config/suppress_warnings.py +30 -12
- kash/config/text_styles.py +3 -13
- kash/docs/load_api_docs.py +2 -1
- kash/docs/markdown/topics/a2_installation.md +7 -3
- kash/docs/markdown/topics/a3_getting_started.md +3 -2
- kash/docs/markdown/warning.md +3 -8
- kash/docs/markdown/welcome.md +4 -0
- kash/docs_base/load_recipe_snippets.py +1 -1
- kash/docs_base/recipes/{general_system_commands.ksh → general_system_commands.sh} +1 -1
- kash/{concepts → embeddings}/cosine.py +2 -1
- kash/embeddings/text_similarity.py +57 -0
- kash/exec/__init__.py +20 -3
- kash/exec/action_decorators.py +18 -4
- kash/exec/action_exec.py +41 -23
- kash/exec/action_registry.py +13 -48
- kash/exec/command_registry.py +2 -1
- kash/exec/fetch_url_metadata.py +4 -6
- kash/exec/importing.py +56 -0
- kash/exec/llm_transforms.py +6 -6
- kash/exec/precondition_registry.py +2 -1
- kash/exec/preconditions.py +16 -1
- kash/exec/shell_callable_action.py +33 -19
- kash/file_storage/file_store.py +23 -14
- kash/file_storage/item_file_format.py +13 -3
- kash/file_storage/metadata_dirs.py +11 -2
- kash/help/assistant.py +2 -2
- kash/help/assistant_instructions.py +2 -1
- kash/help/help_embeddings.py +2 -2
- kash/help/help_printing.py +14 -10
- kash/help/tldr_help.py +5 -3
- kash/llm_utils/clean_headings.py +1 -1
- kash/llm_utils/llm_api_keys.py +4 -4
- kash/llm_utils/llm_completion.py +2 -2
- kash/llm_utils/llm_features.py +68 -0
- kash/llm_utils/llm_messages.py +1 -2
- kash/llm_utils/llm_names.py +1 -1
- kash/llm_utils/llms.py +17 -12
- kash/local_server/__init__.py +5 -2
- kash/local_server/local_server.py +56 -46
- kash/local_server/local_server_commands.py +15 -15
- kash/local_server/local_server_routes.py +2 -2
- kash/local_server/local_url_formatters.py +1 -1
- kash/mcp/__init__.py +5 -2
- kash/mcp/mcp_cli.py +54 -17
- kash/mcp/mcp_server_commands.py +5 -6
- kash/mcp/mcp_server_routes.py +14 -11
- kash/mcp/mcp_server_sse.py +61 -34
- kash/mcp/mcp_server_stdio.py +0 -8
- kash/media_base/audio_processing.py +81 -7
- kash/media_base/media_cache.py +18 -18
- kash/media_base/media_services.py +1 -1
- kash/media_base/media_tools.py +6 -6
- kash/media_base/services/local_file_media.py +2 -2
- kash/media_base/{speech_transcription.py → transcription_deepgram.py} +25 -109
- kash/media_base/transcription_format.py +73 -0
- kash/media_base/transcription_whisper.py +38 -0
- kash/model/__init__.py +73 -5
- kash/model/actions_model.py +38 -4
- kash/model/concept_model.py +30 -0
- kash/model/items_model.py +56 -13
- kash/model/params_model.py +24 -0
- kash/shell/completions/completion_scoring.py +37 -5
- kash/shell/output/kerm_codes.py +1 -2
- kash/shell/output/shell_formatting.py +14 -4
- kash/shell/shell_main.py +2 -2
- kash/shell/utils/exception_printing.py +6 -0
- kash/shell/utils/native_utils.py +26 -20
- kash/text_handling/custom_sliding_transforms.py +12 -4
- kash/text_handling/doc_normalization.py +6 -2
- kash/text_handling/markdown_render.py +117 -0
- kash/text_handling/markdown_utils.py +204 -0
- kash/utils/common/import_utils.py +12 -3
- kash/utils/common/type_utils.py +0 -29
- kash/utils/common/url.py +80 -28
- kash/utils/errors.py +6 -0
- kash/utils/file_utils/{dir_size.py → dir_info.py} +25 -4
- kash/utils/file_utils/file_ext.py +2 -3
- kash/utils/file_utils/file_formats.py +28 -2
- kash/utils/file_utils/file_formats_model.py +50 -19
- kash/utils/file_utils/filename_parsing.py +10 -4
- kash/web_content/dir_store.py +1 -2
- kash/web_content/file_cache_utils.py +37 -10
- kash/web_content/file_processing.py +68 -0
- kash/web_content/local_file_cache.py +12 -9
- kash/web_content/web_extract.py +8 -3
- kash/web_content/web_fetch.py +12 -4
- kash/web_gen/tabbed_webpage.py +5 -2
- kash/web_gen/templates/base_styles.css.jinja +120 -14
- kash/web_gen/templates/base_webpage.html.jinja +60 -13
- kash/web_gen/templates/content_styles.css.jinja +4 -2
- kash/web_gen/templates/item_view.html.jinja +2 -2
- kash/web_gen/templates/tabbed_webpage.html.jinja +1 -2
- kash/workspaces/__init__.py +15 -2
- kash/workspaces/selections.py +18 -3
- kash/workspaces/source_items.py +4 -2
- kash/workspaces/workspace_output.py +11 -4
- kash/workspaces/workspaces.py +5 -11
- kash/xonsh_custom/command_nl_utils.py +40 -19
- kash/xonsh_custom/custom_shell.py +44 -12
- kash/xonsh_custom/customize_prompt.py +39 -21
- kash/xonsh_custom/load_into_xonsh.py +26 -27
- kash/xonsh_custom/shell_load_commands.py +2 -2
- kash/xonsh_custom/xonsh_completers.py +2 -249
- kash/xonsh_custom/xonsh_keybindings.py +282 -0
- kash/xonsh_custom/xonsh_modern_tools.py +3 -3
- kash/xontrib/kash_extension.py +5 -6
- {kash_shell-0.3.8.dist-info → kash_shell-0.3.10.dist-info}/METADATA +26 -12
- {kash_shell-0.3.8.dist-info → kash_shell-0.3.10.dist-info}/RECORD +140 -140
- {kash_shell-0.3.8.dist-info → kash_shell-0.3.10.dist-info}/entry_points.txt +1 -1
- kash/concepts/concept_formats.py +0 -23
- kash/concepts/text_similarity.py +0 -112
- kash/shell/clideps/api_keys.py +0 -99
- kash/shell/clideps/dotenv_setup.py +0 -114
- kash/shell/clideps/dotenv_utils.py +0 -89
- kash/shell/clideps/pkg_deps.py +0 -232
- kash/shell/clideps/platforms.py +0 -11
- kash/shell/clideps/terminal_features.py +0 -56
- kash/shell/utils/osc_utils.py +0 -95
- kash/shell/utils/terminal_images.py +0 -133
- kash/text_handling/markdown_util.py +0 -167
- kash/utils/common/atomic_var.py +0 -158
- kash/utils/common/string_replace.py +0 -93
- kash/utils/common/string_template.py +0 -101
- /kash/docs_base/recipes/{python_dev_commands.ksh → python_dev_commands.sh} +0 -0
- /kash/docs_base/recipes/{tldr_standard_commands.ksh → tldr_standard_commands.sh} +0 -0
- /kash/{concepts → embeddings}/embeddings.py +0 -0
- {kash_shell-0.3.8.dist-info → kash_shell-0.3.10.dist-info}/WHEEL +0 -0
- {kash_shell-0.3.8.dist-info → kash_shell-0.3.10.dist-info}/licenses/LICENSE +0 -0
kash/exec/action_exec.py
CHANGED
|
@@ -32,7 +32,7 @@ from kash.workspaces.workspace_importing import import_and_load
|
|
|
32
32
|
log = get_logger(__name__)
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
def prepare_action_input(*input_args: CommandArg) -> ActionInput:
|
|
35
|
+
def prepare_action_input(*input_args: CommandArg, refetch: bool = False) -> ActionInput:
|
|
36
36
|
"""
|
|
37
37
|
Prepare input args, which may be URLs or paths, into items that correspond to
|
|
38
38
|
URL or file resources, either finding them in the workspace or importing them.
|
|
@@ -50,13 +50,16 @@ def prepare_action_input(*input_args: CommandArg) -> ActionInput:
|
|
|
50
50
|
if input_items:
|
|
51
51
|
log.message("Assembling metadata for input items:\n%s", fmt_lines(input_items))
|
|
52
52
|
input_items = [
|
|
53
|
-
fetch_url_item_metadata(item) if is_url_item(item) else item
|
|
53
|
+
fetch_url_item_metadata(item, refetch=refetch) if is_url_item(item) else item
|
|
54
|
+
for item in input_items
|
|
54
55
|
]
|
|
55
56
|
|
|
56
57
|
return ActionInput(input_items)
|
|
57
58
|
|
|
58
59
|
|
|
59
|
-
def validate_action_input(
|
|
60
|
+
def validate_action_input(
|
|
61
|
+
context: ExecContext, ws: FileStore, action: Action, action_input: ActionInput
|
|
62
|
+
) -> Operation:
|
|
60
63
|
"""
|
|
61
64
|
Validate an action input, ensuring the right number of args, all explicit params are filled,
|
|
62
65
|
and the precondition holds and return an `Operation` that describes what will happen.
|
|
@@ -74,7 +77,9 @@ def validate_action_input(ws: FileStore, action: Action, action_input: ActionInp
|
|
|
74
77
|
# If the inputs are paths, record the input paths, including hashes.
|
|
75
78
|
store_paths = [StorePath(not_none(item.store_path)) for item in input_items if item.store_path]
|
|
76
79
|
inputs = [Input(store_path, ws.hash(store_path)) for store_path in store_paths]
|
|
77
|
-
|
|
80
|
+
# Add any non-default runtime options into the options summary.
|
|
81
|
+
options = {**action.param_value_summary(), **context.runtime_options}
|
|
82
|
+
operation = Operation(action.name, inputs, options)
|
|
78
83
|
|
|
79
84
|
return operation
|
|
80
85
|
|
|
@@ -93,10 +98,7 @@ def log_action(action: Action, action_input: ActionInput, operation: Operation):
|
|
|
93
98
|
|
|
94
99
|
|
|
95
100
|
def check_for_existing_result(
|
|
96
|
-
context: ExecContext,
|
|
97
|
-
action_input: ActionInput,
|
|
98
|
-
operation: Operation,
|
|
99
|
-
rerun: bool = False,
|
|
101
|
+
context: ExecContext, action_input: ActionInput, operation: Operation
|
|
100
102
|
) -> ActionResult | None:
|
|
101
103
|
"""
|
|
102
104
|
Check if we already have the results for this operation (same action and inputs)
|
|
@@ -105,7 +107,7 @@ def check_for_existing_result(
|
|
|
105
107
|
"""
|
|
106
108
|
action = context.action
|
|
107
109
|
ws = context.workspace
|
|
108
|
-
rerun = context.rerun
|
|
110
|
+
rerun = context.rerun
|
|
109
111
|
|
|
110
112
|
existing_result = None
|
|
111
113
|
|
|
@@ -261,7 +263,12 @@ def _run_for_each_item(context: ExecContext, input: ActionInput) -> ActionResult
|
|
|
261
263
|
|
|
262
264
|
|
|
263
265
|
def save_action_result(
|
|
264
|
-
ws: FileStore,
|
|
266
|
+
ws: FileStore,
|
|
267
|
+
result: ActionResult,
|
|
268
|
+
action_input: ActionInput,
|
|
269
|
+
*,
|
|
270
|
+
as_tmp: bool = False,
|
|
271
|
+
no_format: bool = False,
|
|
265
272
|
) -> tuple[list[StorePath], list[StorePath]]:
|
|
266
273
|
"""
|
|
267
274
|
Save the result of an action to the workspace. Handles skipping duplicates and
|
|
@@ -277,7 +284,7 @@ def save_action_result(
|
|
|
277
284
|
skipped_paths.append(store_path)
|
|
278
285
|
continue
|
|
279
286
|
|
|
280
|
-
ws.save(item)
|
|
287
|
+
ws.save(item, as_tmp=as_tmp, no_format=no_format)
|
|
281
288
|
|
|
282
289
|
if skipped_paths:
|
|
283
290
|
log.message(
|
|
@@ -308,9 +315,7 @@ def save_action_result(
|
|
|
308
315
|
|
|
309
316
|
|
|
310
317
|
def run_action_with_caching(
|
|
311
|
-
context: ExecContext,
|
|
312
|
-
action_input: ActionInput,
|
|
313
|
-
rerun: bool = False,
|
|
318
|
+
context: ExecContext, action_input: ActionInput
|
|
314
319
|
) -> tuple[ActionResult, list[StorePath], list[StorePath]]:
|
|
315
320
|
"""
|
|
316
321
|
Run an action, including validation, only rerunning if `rerun` requested or
|
|
@@ -328,15 +333,15 @@ def run_action_with_caching(
|
|
|
328
333
|
item.context = context
|
|
329
334
|
|
|
330
335
|
# Assemble the operation and validate the action input.
|
|
331
|
-
operation = validate_action_input(ws, action, action_input)
|
|
336
|
+
operation = validate_action_input(context, ws, action, action_input)
|
|
332
337
|
|
|
333
338
|
# Log what we're about to run.
|
|
334
339
|
log_action(action, action_input, operation)
|
|
335
340
|
|
|
336
341
|
# Check if a previous run already produced the result.
|
|
337
|
-
existing_result = check_for_existing_result(context, action_input, operation
|
|
342
|
+
existing_result = check_for_existing_result(context, action_input, operation)
|
|
338
343
|
|
|
339
|
-
if existing_result and not rerun:
|
|
344
|
+
if existing_result and not context.rerun:
|
|
340
345
|
# Use the cached result.
|
|
341
346
|
result = existing_result
|
|
342
347
|
result_store_paths = [StorePath(not_none(item.store_path)) for item in result.items]
|
|
@@ -353,7 +358,9 @@ def run_action_with_caching(
|
|
|
353
358
|
else:
|
|
354
359
|
# Run it!
|
|
355
360
|
result = run_action_operation(context, action_input, operation)
|
|
356
|
-
result_store_paths, archived_store_paths = save_action_result(
|
|
361
|
+
result_store_paths, archived_store_paths = save_action_result(
|
|
362
|
+
ws, result, action_input, as_tmp=context.tmp_output, no_format=context.no_format
|
|
363
|
+
)
|
|
357
364
|
|
|
358
365
|
PrintHooks.before_done_message()
|
|
359
366
|
log.message(
|
|
@@ -371,9 +378,12 @@ def run_action_with_shell_context(
|
|
|
371
378
|
action_spec: str | type[Action],
|
|
372
379
|
explicit_param_values: RawParamValues,
|
|
373
380
|
*provided_args: str,
|
|
374
|
-
rerun=False,
|
|
381
|
+
rerun: bool = False,
|
|
382
|
+
refetch: bool = False,
|
|
375
383
|
override_state: State | None = None,
|
|
376
|
-
|
|
384
|
+
tmp_output: bool = False,
|
|
385
|
+
no_format: bool = False,
|
|
386
|
+
internal_call: bool = False,
|
|
377
387
|
) -> ActionResult:
|
|
378
388
|
"""
|
|
379
389
|
Main function to run an action from the shell. Wraps `run_action_if_needed` to
|
|
@@ -404,8 +414,16 @@ def run_action_with_shell_context(
|
|
|
404
414
|
action = action_cls.create(explicit_parsed, ws_parsed)
|
|
405
415
|
action_name = action.name
|
|
406
416
|
|
|
407
|
-
# Execution context.
|
|
408
|
-
context = ExecContext(
|
|
417
|
+
# Execution context. This is fixed for the duration of the action.
|
|
418
|
+
context = ExecContext(
|
|
419
|
+
action=action,
|
|
420
|
+
workspace_dir=ws.base_dir,
|
|
421
|
+
rerun=rerun,
|
|
422
|
+
refetch=refetch,
|
|
423
|
+
override_state=override_state,
|
|
424
|
+
tmp_output=tmp_output,
|
|
425
|
+
no_format=no_format,
|
|
426
|
+
)
|
|
409
427
|
|
|
410
428
|
# Collect args from the provided args or otherwise the current selection.
|
|
411
429
|
args, from_selection = assemble_action_args(*provided_args, use_selection=action.uses_selection)
|
|
@@ -422,7 +440,7 @@ def run_action_with_shell_context(
|
|
|
422
440
|
)
|
|
423
441
|
|
|
424
442
|
# Get items for each input arg.
|
|
425
|
-
input = prepare_action_input(*args)
|
|
443
|
+
input = prepare_action_input(*args, refetch=refetch)
|
|
426
444
|
|
|
427
445
|
# Finally, run the action.
|
|
428
446
|
result, result_store_paths, archived_store_paths = run_action_with_caching(context, input)
|
kash/exec/action_registry.py
CHANGED
|
@@ -1,18 +1,14 @@
|
|
|
1
|
-
from pathlib import Path
|
|
2
|
-
|
|
3
1
|
from cachetools import Cache, cached
|
|
4
|
-
from
|
|
2
|
+
from strif import AtomicVar
|
|
5
3
|
|
|
6
4
|
from kash.config.logger import get_logger
|
|
7
5
|
from kash.model.actions_model import Action
|
|
8
|
-
from kash.utils.common.atomic_var import AtomicVar
|
|
9
|
-
from kash.utils.common.import_utils import Tallies, import_subdirs
|
|
10
6
|
from kash.utils.errors import InvalidInput
|
|
11
7
|
|
|
12
8
|
log = get_logger(__name__)
|
|
13
9
|
|
|
14
10
|
# Global registry of action classes.
|
|
15
|
-
|
|
11
|
+
action_classes: AtomicVar[dict[str, type[Action]]] = AtomicVar({})
|
|
16
12
|
|
|
17
13
|
|
|
18
14
|
# Want it fast to get the full list of actions (important for tab completions
|
|
@@ -30,64 +26,29 @@ def register_action_class(cls: type[Action]):
|
|
|
30
26
|
"""
|
|
31
27
|
Register an action class.
|
|
32
28
|
"""
|
|
33
|
-
with
|
|
34
|
-
if cls.name in
|
|
29
|
+
with action_classes.updates() as ac:
|
|
30
|
+
if cls.name in ac:
|
|
35
31
|
log.warning(
|
|
36
32
|
"Duplicate action name (defined twice by accident?): %s (%s)",
|
|
37
33
|
cls.name,
|
|
38
34
|
cls,
|
|
39
35
|
)
|
|
40
|
-
|
|
36
|
+
ac[cls.name] = cls
|
|
41
37
|
|
|
42
38
|
clear_action_cache()
|
|
43
39
|
|
|
44
40
|
|
|
45
|
-
def import_action_subdirs(
|
|
46
|
-
subdirs: list[str],
|
|
47
|
-
package_name: str | None,
|
|
48
|
-
parent_dir: Path,
|
|
49
|
-
tallies: Tallies | None = None,
|
|
50
|
-
):
|
|
51
|
-
"""
|
|
52
|
-
Hook to call from `__init__.py` in a directory containing actions,
|
|
53
|
-
so that they are auto-registered on import.
|
|
54
|
-
|
|
55
|
-
Usage:
|
|
56
|
-
```
|
|
57
|
-
import_action_subdirs(["subdir_name"], __package__, Path(__file__).parent)
|
|
58
|
-
```
|
|
59
|
-
"""
|
|
60
|
-
if tallies is None:
|
|
61
|
-
tallies = {}
|
|
62
|
-
with _action_classes.updates() as action_classes:
|
|
63
|
-
prev_count = len(action_classes)
|
|
64
|
-
|
|
65
|
-
if not package_name:
|
|
66
|
-
raise ValueError(f"Package name missing importing actions: {fmt_path(parent_dir)}")
|
|
67
|
-
|
|
68
|
-
import_subdirs(package_name, parent_dir, subdirs, tallies)
|
|
69
|
-
reload_all_action_classes()
|
|
70
|
-
|
|
71
|
-
log.info(
|
|
72
|
-
"Loaded actions: %s new actions in %s directories below %s:\n%s",
|
|
73
|
-
len(action_classes) - prev_count,
|
|
74
|
-
len(tallies),
|
|
75
|
-
fmt_path(parent_dir),
|
|
76
|
-
fmt_lines(f"{k}: {v} files" for k, v in tallies.items()),
|
|
77
|
-
)
|
|
78
|
-
|
|
79
|
-
|
|
80
41
|
@cached(_action_classes_cache)
|
|
81
42
|
def get_all_action_classes() -> dict[str, type[Action]]:
|
|
82
43
|
# Be sure actions are imported.
|
|
83
44
|
import kash.actions # noqa: F401
|
|
84
45
|
|
|
85
46
|
# Returns a copy for safety.
|
|
86
|
-
|
|
87
|
-
if len(
|
|
47
|
+
ac = action_classes.copy()
|
|
48
|
+
if len(ac) == 0:
|
|
88
49
|
log.error("No actions found! Was there an import error?")
|
|
89
50
|
|
|
90
|
-
return dict(
|
|
51
|
+
return dict(ac)
|
|
91
52
|
|
|
92
53
|
|
|
93
54
|
def look_up_action_class(action_name: str) -> type[Action]:
|
|
@@ -97,7 +58,11 @@ def look_up_action_class(action_name: str) -> type[Action]:
|
|
|
97
58
|
return actions[action_name]
|
|
98
59
|
|
|
99
60
|
|
|
100
|
-
def
|
|
61
|
+
def refresh_action_classes() -> dict[str, type[Action]]:
|
|
62
|
+
"""
|
|
63
|
+
Reload all action classes, refreshing the cache. Call after registering
|
|
64
|
+
new action classes.
|
|
65
|
+
"""
|
|
101
66
|
clear_action_cache()
|
|
102
67
|
return get_all_action_classes()
|
|
103
68
|
|
kash/exec/command_registry.py
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
from collections.abc import Callable
|
|
2
2
|
from typing import overload
|
|
3
3
|
|
|
4
|
+
from strif import AtomicVar
|
|
5
|
+
|
|
4
6
|
from kash.config.logger import get_logger
|
|
5
7
|
from kash.exec_model.shell_model import ShellResult
|
|
6
|
-
from kash.utils.common.atomic_var import AtomicVar
|
|
7
8
|
from kash.utils.errors import InvalidInput
|
|
8
9
|
|
|
9
10
|
log = get_logger(__name__)
|
kash/exec/fetch_url_metadata.py
CHANGED
|
@@ -13,9 +13,7 @@ from kash.workspaces import current_ws
|
|
|
13
13
|
log = get_logger(__name__)
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
def fetch_url_metadata(
|
|
17
|
-
locator: Url | StorePath, use_cache: bool = True, refetch: bool = False
|
|
18
|
-
) -> Item:
|
|
16
|
+
def fetch_url_metadata(locator: Url | StorePath, refetch: bool = False) -> Item:
|
|
19
17
|
ws = current_ws()
|
|
20
18
|
if is_url(locator):
|
|
21
19
|
# Import or find URL as a resource in the current workspace.
|
|
@@ -28,10 +26,10 @@ def fetch_url_metadata(
|
|
|
28
26
|
else:
|
|
29
27
|
raise InvalidInput(f"Not a URL or URL resource: {fmt_loc(locator)}")
|
|
30
28
|
|
|
31
|
-
return fetch_url_item_metadata(item,
|
|
29
|
+
return fetch_url_item_metadata(item, refetch=refetch)
|
|
32
30
|
|
|
33
31
|
|
|
34
|
-
def fetch_url_item_metadata(item: Item,
|
|
32
|
+
def fetch_url_item_metadata(item: Item, refetch: bool = False) -> Item:
|
|
35
33
|
"""
|
|
36
34
|
Fetch metadata for a URL using a media service if we recognize the URL,
|
|
37
35
|
and otherwise fetching and extracting it from the web page HTML.
|
|
@@ -56,7 +54,7 @@ def fetch_url_item_metadata(item: Item, use_cache: bool = True, refetch: bool =
|
|
|
56
54
|
fetched_item = Item.from_media_metadata(media_metadata)
|
|
57
55
|
fetched_item = item.merged_copy(fetched_item)
|
|
58
56
|
else:
|
|
59
|
-
page_data = fetch_extract(url,
|
|
57
|
+
page_data = fetch_extract(url, refetch=refetch)
|
|
60
58
|
fetched_item = item.new_copy_with(
|
|
61
59
|
title=page_data.title or item.title,
|
|
62
60
|
description=page_data.description or item.description,
|
kash/exec/importing.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
3
|
+
from prettyfmt import fmt_lines, fmt_path
|
|
4
|
+
|
|
5
|
+
from kash.config.logger import get_logger
|
|
6
|
+
from kash.exec.action_registry import action_classes, refresh_action_classes
|
|
7
|
+
from kash.exec.command_registry import get_all_commands
|
|
8
|
+
from kash.utils.common.import_utils import Tallies, import_subdirs
|
|
9
|
+
|
|
10
|
+
log = get_logger(__name__)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def import_and_register(
|
|
14
|
+
package_name: str | None,
|
|
15
|
+
parent_dir: Path,
|
|
16
|
+
subdir_names: list[str] | None = None,
|
|
17
|
+
tallies: Tallies | None = None,
|
|
18
|
+
):
|
|
19
|
+
"""
|
|
20
|
+
This hook can be used for auto-registering commands and actions from any
|
|
21
|
+
subdirectory of a given package.
|
|
22
|
+
|
|
23
|
+
Useful to call from `__init__.py` files to import a directory of code,
|
|
24
|
+
auto-registering annotated commands and actions and also handles refreshing the
|
|
25
|
+
action cache if new actions are registered.
|
|
26
|
+
|
|
27
|
+
Usage:
|
|
28
|
+
```
|
|
29
|
+
import_and_register(["subdir1", "subdir2"], __package__, Path(__file__).parent)
|
|
30
|
+
```
|
|
31
|
+
"""
|
|
32
|
+
if not package_name:
|
|
33
|
+
raise ValueError(f"Package name missing importing actions: {fmt_path(parent_dir)}")
|
|
34
|
+
if tallies is None:
|
|
35
|
+
tallies = {}
|
|
36
|
+
|
|
37
|
+
with action_classes.updates() as ac:
|
|
38
|
+
prev_command_count = len(get_all_commands())
|
|
39
|
+
prev_action_count = len(ac)
|
|
40
|
+
|
|
41
|
+
import_subdirs(package_name, parent_dir, subdir_names, tallies)
|
|
42
|
+
|
|
43
|
+
new_command_count = len(get_all_commands()) - prev_command_count
|
|
44
|
+
new_action_count = len(ac) - prev_action_count
|
|
45
|
+
|
|
46
|
+
if new_action_count > 0:
|
|
47
|
+
refresh_action_classes()
|
|
48
|
+
|
|
49
|
+
log.info(
|
|
50
|
+
"Loaded %s new commands and %s new actions in %s directories below %s:\n%s",
|
|
51
|
+
new_command_count,
|
|
52
|
+
new_action_count,
|
|
53
|
+
len(tallies),
|
|
54
|
+
fmt_path(parent_dir),
|
|
55
|
+
fmt_lines(f"{k}: {v} files" for k, v in tallies.items()),
|
|
56
|
+
)
|
kash/exec/llm_transforms.py
CHANGED
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
from dataclasses import replace
|
|
2
2
|
|
|
3
3
|
from chopdiff.docs import DiffFilter, TextDoc
|
|
4
|
-
from chopdiff.transforms import WindowSettings,
|
|
4
|
+
from chopdiff.transforms import WindowSettings, filtered_transform
|
|
5
|
+
from clideps.env_vars.dotenv_utils import load_dotenv_paths
|
|
5
6
|
from flowmark import fill_markdown
|
|
6
7
|
|
|
7
8
|
from kash.config.logger import get_logger
|
|
9
|
+
from kash.config.settings import global_settings
|
|
8
10
|
from kash.llm_utils import LLMName
|
|
9
11
|
from kash.llm_utils.fuzzy_parsing import strip_markdown_fence
|
|
10
12
|
from kash.llm_utils.llm_completion import llm_template_completion
|
|
11
13
|
from kash.llm_utils.llm_messages import Message, MessageTemplate
|
|
12
14
|
from kash.model.actions_model import LLMOptions
|
|
13
15
|
from kash.model.items_model import Item, ItemType
|
|
14
|
-
from kash.shell.clideps.dotenv_utils import load_dotenv_paths
|
|
15
16
|
from kash.utils.errors import InvalidInput
|
|
16
17
|
from kash.utils.file_utils.file_formats_model import Format
|
|
17
18
|
|
|
@@ -24,7 +25,7 @@ def windowed_llm_transform(
|
|
|
24
25
|
template: MessageTemplate,
|
|
25
26
|
input: str,
|
|
26
27
|
windowing: WindowSettings | None,
|
|
27
|
-
diff_filter: DiffFilter,
|
|
28
|
+
diff_filter: DiffFilter | None = None,
|
|
28
29
|
check_no_results: bool = True,
|
|
29
30
|
) -> TextDoc:
|
|
30
31
|
def doc_transform(input_doc: TextDoc) -> TextDoc:
|
|
@@ -47,7 +48,7 @@ def windowed_llm_transform(
|
|
|
47
48
|
|
|
48
49
|
|
|
49
50
|
def llm_transform_str(options: LLMOptions, input_str: str, check_no_results: bool = True) -> str:
|
|
50
|
-
load_dotenv_paths()
|
|
51
|
+
load_dotenv_paths(True, True, global_settings().system_config_dir)
|
|
51
52
|
|
|
52
53
|
if options.windowing and options.windowing.size:
|
|
53
54
|
log.message(
|
|
@@ -56,7 +57,6 @@ def llm_transform_str(options: LLMOptions, input_str: str, check_no_results: boo
|
|
|
56
57
|
options.op_name,
|
|
57
58
|
options.windowing,
|
|
58
59
|
)
|
|
59
|
-
diff_filter = options.diff_filter or accept_all
|
|
60
60
|
|
|
61
61
|
result_str = windowed_llm_transform(
|
|
62
62
|
options.model,
|
|
@@ -64,7 +64,7 @@ def llm_transform_str(options: LLMOptions, input_str: str, check_no_results: boo
|
|
|
64
64
|
options.body_template,
|
|
65
65
|
input_str,
|
|
66
66
|
options.windowing,
|
|
67
|
-
diff_filter,
|
|
67
|
+
diff_filter=options.diff_filter,
|
|
68
68
|
).reassemble()
|
|
69
69
|
else:
|
|
70
70
|
log.message(
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
from collections.abc import Callable
|
|
2
2
|
|
|
3
|
+
from strif import AtomicVar
|
|
4
|
+
|
|
3
5
|
from kash.config.logger import get_logger
|
|
4
6
|
from kash.model.items_model import Item
|
|
5
7
|
from kash.model.preconditions_model import Precondition
|
|
6
|
-
from kash.utils.common.atomic_var import AtomicVar
|
|
7
8
|
|
|
8
9
|
log = get_logger(__name__)
|
|
9
10
|
|
kash/exec/preconditions.py
CHANGED
|
@@ -7,7 +7,7 @@ from chopdiff.html import has_timestamp
|
|
|
7
7
|
|
|
8
8
|
from kash.exec.precondition_registry import kash_precondition
|
|
9
9
|
from kash.model.items_model import Item, ItemType
|
|
10
|
-
from kash.text_handling.
|
|
10
|
+
from kash.text_handling.markdown_utils import extract_bullet_points
|
|
11
11
|
from kash.utils.file_utils.file_formats_model import Format
|
|
12
12
|
|
|
13
13
|
|
|
@@ -16,6 +16,21 @@ def is_resource(item: Item) -> bool:
|
|
|
16
16
|
return item.type == ItemType.resource
|
|
17
17
|
|
|
18
18
|
|
|
19
|
+
@kash_precondition
|
|
20
|
+
def is_doc_resource(item: Item) -> bool:
|
|
21
|
+
return bool(is_resource(item) and item.format and item.format.is_doc)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
@kash_precondition
|
|
25
|
+
def is_html_resource(item: Item) -> bool:
|
|
26
|
+
return bool(is_resource(item) and item.format and item.format == Format.html)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@kash_precondition
|
|
30
|
+
def is_docx_resource(item: Item) -> bool:
|
|
31
|
+
return bool(is_resource(item) and item.format and item.format == Format.docx)
|
|
32
|
+
|
|
33
|
+
|
|
19
34
|
@kash_precondition
|
|
20
35
|
def is_concept(item: Item) -> bool:
|
|
21
36
|
return item.type == ItemType.concept
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from funlog import log_tallies
|
|
2
2
|
|
|
3
|
+
from kash.config.env_settings import KashEnv
|
|
3
4
|
from kash.config.logger import get_console, get_logger
|
|
4
5
|
from kash.config.text_styles import COLOR_ERROR, SPINNER
|
|
5
6
|
from kash.exec.action_exec import run_action_with_shell_context
|
|
@@ -31,34 +32,43 @@ class ShellCallableAction:
|
|
|
31
32
|
def __call__(self, args: list[str]) -> ShellResult | None:
|
|
32
33
|
from kash.commands.help import help_commands
|
|
33
34
|
|
|
34
|
-
|
|
35
|
-
PrintHooks.before_shell_action_run()
|
|
35
|
+
log.debug("ShellCallableAction: %s: %s", self.action_cls.name, args)
|
|
36
36
|
|
|
37
|
-
shell_args = parse_shell_args(args)
|
|
38
|
-
|
|
39
|
-
# We will instantiate the action later but we create an unconfigured
|
|
40
|
-
# instance for help/info.
|
|
41
|
-
action = action_cls.create(None)
|
|
42
|
-
if shell_args.show_help:
|
|
43
|
-
print_action_help(action, verbose=True)
|
|
44
|
-
return ShellResult()
|
|
45
|
-
elif shell_args.options.get("show_source", False):
|
|
46
|
-
return help_commands.source_code(action_cls.name)
|
|
47
|
-
|
|
48
|
-
# Handle --rerun option at action invocation time.
|
|
49
|
-
rerun = bool(shell_args.options.get("rerun", False))
|
|
50
|
-
|
|
51
|
-
log.info("Action shell args: %s", shell_args)
|
|
52
37
|
try:
|
|
38
|
+
action_cls = self.action_cls
|
|
39
|
+
PrintHooks.before_shell_action_run()
|
|
40
|
+
|
|
41
|
+
shell_args = parse_shell_args(args)
|
|
42
|
+
|
|
43
|
+
# We will instantiate the action later but we create an unconfigured
|
|
44
|
+
# instance for help/info.
|
|
45
|
+
action = action_cls.create(None)
|
|
46
|
+
if shell_args.show_help:
|
|
47
|
+
print_action_help(action, verbose=True)
|
|
48
|
+
return ShellResult()
|
|
49
|
+
elif shell_args.options.get("show_source", False):
|
|
50
|
+
return help_commands.source_code(action_cls.name)
|
|
51
|
+
|
|
52
|
+
# Handle --rerun and --refetch options at action invocation time.
|
|
53
|
+
rerun = bool(shell_args.options.get("rerun", False))
|
|
54
|
+
refetch = bool(shell_args.options.get("refetch", False))
|
|
55
|
+
no_format = bool(shell_args.options.get("no_format", False))
|
|
56
|
+
|
|
57
|
+
log.info("Action shell args: %s", shell_args)
|
|
53
58
|
explicit_values = RawParamValues(shell_args.options)
|
|
54
59
|
if not action.interactive_input:
|
|
55
60
|
with get_console().status(f"Running action {action.name}…", spinner=SPINNER):
|
|
56
61
|
result = run_action_with_shell_context(
|
|
57
|
-
action_cls,
|
|
62
|
+
action_cls,
|
|
63
|
+
explicit_values,
|
|
64
|
+
*shell_args.args,
|
|
65
|
+
rerun=rerun,
|
|
66
|
+
refetch=refetch,
|
|
67
|
+
no_format=no_format,
|
|
58
68
|
)
|
|
59
69
|
else:
|
|
60
70
|
result = run_action_with_shell_context(
|
|
61
|
-
action_cls, explicit_values, *shell_args.args, rerun=rerun
|
|
71
|
+
action_cls, explicit_values, *shell_args.args, rerun=rerun, refetch=refetch
|
|
62
72
|
)
|
|
63
73
|
# We don't return the result to keep the xonsh shell output clean.
|
|
64
74
|
except NONFATAL_EXCEPTIONS as e:
|
|
@@ -66,6 +76,10 @@ class ShellCallableAction:
|
|
|
66
76
|
log.error(f"[{COLOR_ERROR}]Action error:[/{COLOR_ERROR}] %s", summarize_traceback(e))
|
|
67
77
|
log.info("Action error details: %s", e, exc_info=True)
|
|
68
78
|
return ShellResult(exception=e)
|
|
79
|
+
except Exception as e:
|
|
80
|
+
# Log here while we are in the true call stack (not inside the xonsh call stack).
|
|
81
|
+
log.error("Action error: %s", e, exc_info=KashEnv.KASH_SHOW_TRACEBACK.read_bool(True))
|
|
82
|
+
raise
|
|
69
83
|
finally:
|
|
70
84
|
log_tallies(level="warning", if_slower_than=10.0)
|
|
71
85
|
# output_separator()
|
kash/file_storage/file_store.py
CHANGED
|
@@ -97,10 +97,6 @@ class FileStore(Workspace):
|
|
|
97
97
|
|
|
98
98
|
add_to_ignore(self.base_dir / ".gitignore", [".kash/"])
|
|
99
99
|
|
|
100
|
-
self.vector_index = None
|
|
101
|
-
# FIXME: Add vector index support dynamically if available.
|
|
102
|
-
# self.vector_index = WsVectorIndex(self.base_dir / self.dirs.index_dir)
|
|
103
|
-
|
|
104
100
|
# Initialize selection with history support.
|
|
105
101
|
self.selections = SelectionHistory.init(self.base_dir / self.dirs.selection_yml)
|
|
106
102
|
|
|
@@ -274,10 +270,10 @@ class FileStore(Workspace):
|
|
|
274
270
|
return self._tmp_path_for(item), False, None
|
|
275
271
|
elif item.store_path:
|
|
276
272
|
return StorePath(item.store_path), True, None
|
|
277
|
-
elif item_id in self.id_map:
|
|
273
|
+
elif item_id in self.id_map and self.exists(self.id_map[item_id]):
|
|
278
274
|
# If this item has an identity and we've saved under that id before, use the same store path.
|
|
279
275
|
store_path = self.id_map[item_id]
|
|
280
|
-
log.
|
|
276
|
+
log.warning(
|
|
281
277
|
"Found existing item with same id:\n%s",
|
|
282
278
|
fmt_lines([fmt_loc(store_path), item_id]),
|
|
283
279
|
)
|
|
@@ -313,10 +309,16 @@ class FileStore(Workspace):
|
|
|
313
309
|
)
|
|
314
310
|
|
|
315
311
|
@log_calls()
|
|
316
|
-
def save(
|
|
312
|
+
def save(
|
|
313
|
+
self, item: Item, *, overwrite: bool = True, as_tmp: bool = False, no_format: bool = False
|
|
314
|
+
) -> StorePath:
|
|
317
315
|
"""
|
|
318
|
-
Save the item. Uses the store_path if it's already set or generates a new one.
|
|
319
|
-
Updates item.store_path
|
|
316
|
+
Save the item. Uses the `store_path` if it's already set or generates a new one.
|
|
317
|
+
Updates `item.store_path`.
|
|
318
|
+
|
|
319
|
+
If `as_tmp` is true, will save the item to a temporary file.
|
|
320
|
+
If `overwrite` is false, will skip saving if the item already exists.
|
|
321
|
+
If `no_format` is true, will not normalize body text formatting (for Markdown).
|
|
320
322
|
"""
|
|
321
323
|
# If external file already exists within the workspace, the file is already saved (without metadata).
|
|
322
324
|
external_path = item.external_path and Path(item.external_path).resolve()
|
|
@@ -352,7 +354,7 @@ class FileStore(Workspace):
|
|
|
352
354
|
if item.external_path:
|
|
353
355
|
copyfile_atomic(item.external_path, full_path)
|
|
354
356
|
else:
|
|
355
|
-
write_item(item, full_path)
|
|
357
|
+
write_item(item, full_path, normalize=not no_format)
|
|
356
358
|
except OSError as e:
|
|
357
359
|
log.error("Error saving item: %s", e)
|
|
358
360
|
try:
|
|
@@ -402,6 +404,7 @@ class FileStore(Workspace):
|
|
|
402
404
|
def import_item(
|
|
403
405
|
self,
|
|
404
406
|
locator: Locator,
|
|
407
|
+
*,
|
|
405
408
|
as_type: ItemType | None = None,
|
|
406
409
|
reimport: bool = False,
|
|
407
410
|
) -> StorePath:
|
|
@@ -500,7 +503,9 @@ class FileStore(Workspace):
|
|
|
500
503
|
as_type: ItemType | None = None,
|
|
501
504
|
reimport: bool = False,
|
|
502
505
|
) -> list[StorePath]:
|
|
503
|
-
return [
|
|
506
|
+
return [
|
|
507
|
+
self.import_item(locator, as_type=as_type, reimport=reimport) for locator in locators
|
|
508
|
+
]
|
|
504
509
|
|
|
505
510
|
def _filter_selection_paths(self):
|
|
506
511
|
"""
|
|
@@ -541,7 +546,7 @@ class FileStore(Workspace):
|
|
|
541
546
|
# TODO: Update metadata of all relations that point to this path too.
|
|
542
547
|
|
|
543
548
|
def archive(
|
|
544
|
-
self, store_path: StorePath, missing_ok: bool = False, quiet: bool = False
|
|
549
|
+
self, store_path: StorePath, *, missing_ok: bool = False, quiet: bool = False
|
|
545
550
|
) -> StorePath:
|
|
546
551
|
"""
|
|
547
552
|
Archive the item by moving it into the archive directory.
|
|
@@ -557,6 +562,9 @@ class FileStore(Workspace):
|
|
|
557
562
|
if missing_ok and not orig_path.exists():
|
|
558
563
|
log.message("Item to archive not found so moving on: %s", fmt_loc(orig_path))
|
|
559
564
|
return store_path
|
|
565
|
+
if not orig_path.exists():
|
|
566
|
+
log.warning("Item to archive not found: %s", fmt_loc(orig_path))
|
|
567
|
+
return store_path
|
|
560
568
|
move_file(orig_path, archive_path)
|
|
561
569
|
self._remove_references([store_path])
|
|
562
570
|
|
|
@@ -576,7 +584,7 @@ class FileStore(Workspace):
|
|
|
576
584
|
move_file(full_input_path, original_path)
|
|
577
585
|
return StorePath(store_path)
|
|
578
586
|
|
|
579
|
-
def log_workspace_info(self, once: bool = False):
|
|
587
|
+
def log_workspace_info(self, *, once: bool = False):
|
|
580
588
|
"""
|
|
581
589
|
Log helpful information about the workspace.
|
|
582
590
|
"""
|
|
@@ -607,7 +615,7 @@ class FileStore(Workspace):
|
|
|
607
615
|
|
|
608
616
|
if self.is_global_ws:
|
|
609
617
|
PrintHooks.spacer()
|
|
610
|
-
log.warning("Note you are currently using the default
|
|
618
|
+
log.warning("Note you are currently using the default global workspace.")
|
|
611
619
|
cprint(
|
|
612
620
|
"Create or switch to another workspace with the `workspace` command.",
|
|
613
621
|
style=STYLE_HINT,
|
|
@@ -619,6 +627,7 @@ class FileStore(Workspace):
|
|
|
619
627
|
def walk_items(
|
|
620
628
|
self,
|
|
621
629
|
store_path: StorePath | None = None,
|
|
630
|
+
*,
|
|
622
631
|
use_ignore: bool = True,
|
|
623
632
|
) -> Generator[StorePath, None, None]:
|
|
624
633
|
"""
|