kash-shell 0.3.10__py3-none-any.whl → 0.3.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kash/actions/core/format_markdown_template.py +2 -5
- kash/actions/core/markdownify.py +2 -4
- kash/actions/core/readability.py +2 -4
- kash/actions/core/render_as_html.py +30 -11
- kash/actions/core/show_webpage.py +6 -11
- kash/actions/core/strip_html.py +4 -8
- kash/actions/core/{webpage_config.py → tabbed_webpage_config.py} +5 -3
- kash/actions/core/{webpage_generate.py → tabbed_webpage_generate.py} +5 -4
- kash/commands/base/basic_file_commands.py +21 -3
- kash/commands/base/files_command.py +29 -10
- kash/commands/extras/parse_uv_lock.py +12 -3
- kash/commands/workspace/selection_commands.py +1 -1
- kash/commands/workspace/workspace_commands.py +2 -3
- kash/config/colors.py +2 -2
- kash/config/env_settings.py +2 -42
- kash/config/logger.py +30 -25
- kash/config/logger_basic.py +6 -6
- kash/config/settings.py +23 -7
- kash/config/setup.py +33 -5
- kash/config/text_styles.py +25 -22
- kash/embeddings/cosine.py +12 -4
- kash/embeddings/embeddings.py +16 -6
- kash/embeddings/text_similarity.py +10 -4
- kash/exec/__init__.py +3 -0
- kash/exec/action_decorators.py +10 -25
- kash/exec/action_exec.py +43 -23
- kash/exec/llm_transforms.py +6 -3
- kash/exec/preconditions.py +10 -12
- kash/exec/resolve_args.py +4 -0
- kash/exec/runtime_settings.py +134 -0
- kash/exec/shell_callable_action.py +5 -3
- kash/file_storage/file_store.py +37 -38
- kash/file_storage/item_file_format.py +6 -3
- kash/file_storage/store_filenames.py +6 -3
- kash/help/function_param_info.py +1 -1
- kash/llm_utils/init_litellm.py +16 -0
- kash/llm_utils/llm_api_keys.py +6 -2
- kash/llm_utils/llm_completion.py +11 -4
- kash/local_server/local_server_routes.py +1 -7
- kash/mcp/mcp_cli.py +3 -2
- kash/mcp/mcp_server_routes.py +11 -12
- kash/media_base/transcription_deepgram.py +15 -2
- kash/model/__init__.py +1 -1
- kash/model/actions_model.py +6 -54
- kash/model/exec_model.py +79 -0
- kash/model/items_model.py +102 -35
- kash/model/operations_model.py +38 -15
- kash/model/paths_model.py +2 -0
- kash/shell/output/shell_output.py +10 -8
- kash/shell/shell_main.py +2 -2
- kash/shell/utils/exception_printing.py +2 -2
- kash/shell/utils/shell_function_wrapper.py +15 -15
- kash/text_handling/doc_normalization.py +16 -8
- kash/text_handling/markdown_render.py +1 -0
- kash/text_handling/markdown_utils.py +105 -2
- kash/utils/common/format_utils.py +2 -8
- kash/utils/common/function_inspect.py +360 -110
- kash/utils/common/inflection.py +22 -0
- kash/utils/common/task_stack.py +4 -15
- kash/utils/errors.py +14 -9
- kash/utils/file_utils/file_ext.py +4 -0
- kash/utils/file_utils/file_formats_model.py +32 -1
- kash/utils/file_utils/file_sort_filter.py +10 -3
- kash/web_gen/__init__.py +0 -4
- kash/web_gen/simple_webpage.py +52 -0
- kash/web_gen/tabbed_webpage.py +23 -16
- kash/web_gen/template_render.py +37 -2
- kash/web_gen/templates/base_styles.css.jinja +84 -59
- kash/web_gen/templates/base_webpage.html.jinja +85 -67
- kash/web_gen/templates/item_view.html.jinja +47 -37
- kash/web_gen/templates/simple_webpage.html.jinja +24 -0
- kash/web_gen/templates/tabbed_webpage.html.jinja +42 -32
- kash/workspaces/__init__.py +12 -3
- kash/workspaces/workspace_dirs.py +58 -0
- kash/workspaces/workspace_importing.py +1 -1
- kash/workspaces/workspaces.py +26 -90
- {kash_shell-0.3.10.dist-info → kash_shell-0.3.12.dist-info}/METADATA +7 -7
- {kash_shell-0.3.10.dist-info → kash_shell-0.3.12.dist-info}/RECORD +81 -76
- kash/shell/utils/argparse_utils.py +0 -20
- kash/utils/lang_utils/inflection.py +0 -18
- {kash_shell-0.3.10.dist-info → kash_shell-0.3.12.dist-info}/WHEEL +0 -0
- {kash_shell-0.3.10.dist-info → kash_shell-0.3.12.dist-info}/entry_points.txt +0 -0
- {kash_shell-0.3.10.dist-info → kash_shell-0.3.12.dist-info}/licenses/LICENSE +0 -0
kash/exec/action_exec.py
CHANGED
|
@@ -4,7 +4,12 @@ from dataclasses import replace
|
|
|
4
4
|
from prettyfmt import fmt_lines
|
|
5
5
|
|
|
6
6
|
from kash.config.logger import get_logger
|
|
7
|
-
from kash.config.text_styles import
|
|
7
|
+
from kash.config.text_styles import (
|
|
8
|
+
EMOJI_SKIP,
|
|
9
|
+
EMOJI_START,
|
|
10
|
+
EMOJI_SUCCESS,
|
|
11
|
+
EMOJI_TIMING,
|
|
12
|
+
)
|
|
8
13
|
from kash.exec.preconditions import is_url_item
|
|
9
14
|
from kash.exec.resolve_args import assemble_action_args
|
|
10
15
|
from kash.exec_model.args_model import CommandArg
|
|
@@ -17,15 +22,16 @@ from kash.model.actions_model import (
|
|
|
17
22
|
ExecContext,
|
|
18
23
|
PathOpType,
|
|
19
24
|
)
|
|
25
|
+
from kash.model.exec_model import RuntimeSettings
|
|
20
26
|
from kash.model.items_model import Item, State
|
|
21
27
|
from kash.model.operations_model import Input, Operation, Source
|
|
22
28
|
from kash.model.params_model import ALL_COMMON_PARAMS, GLOBAL_PARAMS, RawParamValues
|
|
23
29
|
from kash.model.paths_model import StorePath
|
|
24
|
-
from kash.shell.output.shell_output import PrintHooks
|
|
30
|
+
from kash.shell.output.shell_output import PrintHooks
|
|
31
|
+
from kash.utils.common.inflection import plural
|
|
25
32
|
from kash.utils.common.task_stack import task_stack
|
|
26
33
|
from kash.utils.common.type_utils import not_none
|
|
27
|
-
from kash.utils.errors import
|
|
28
|
-
from kash.utils.lang_utils.inflection import plural
|
|
34
|
+
from kash.utils.errors import ContentError, InvalidOutput, get_nonfatal_exceptions
|
|
29
35
|
from kash.workspaces import Selection, current_ws
|
|
30
36
|
from kash.workspaces.workspace_importing import import_and_load
|
|
31
37
|
|
|
@@ -63,6 +69,7 @@ def validate_action_input(
|
|
|
63
69
|
"""
|
|
64
70
|
Validate an action input, ensuring the right number of args, all explicit params are filled,
|
|
65
71
|
and the precondition holds and return an `Operation` that describes what will happen.
|
|
72
|
+
For flexibility, we don't require the items to be saved (have a store path).
|
|
66
73
|
"""
|
|
67
74
|
input_items = action_input.items
|
|
68
75
|
# Validations:
|
|
@@ -75,10 +82,16 @@ def validate_action_input(
|
|
|
75
82
|
|
|
76
83
|
# Now make a note of the the operation we will perform.
|
|
77
84
|
# If the inputs are paths, record the input paths, including hashes.
|
|
78
|
-
|
|
79
|
-
|
|
85
|
+
def input_for(item: Item) -> Input:
|
|
86
|
+
if item.store_path:
|
|
87
|
+
return Input(StorePath(item.store_path), ws.hash(StorePath(item.store_path)))
|
|
88
|
+
else:
|
|
89
|
+
return Input(path=None, source_info="unsaved")
|
|
90
|
+
|
|
91
|
+
inputs = [input_for(item) for item in input_items]
|
|
92
|
+
|
|
80
93
|
# Add any non-default runtime options into the options summary.
|
|
81
|
-
options = {**action.param_value_summary(), **context.
|
|
94
|
+
options = {**action.param_value_summary(), **context.settings.non_default_options}
|
|
82
95
|
operation = Operation(action.name, inputs, options)
|
|
83
96
|
|
|
84
97
|
return operation
|
|
@@ -89,7 +102,7 @@ def log_action(action: Action, action_input: ActionInput, operation: Operation):
|
|
|
89
102
|
Log the action and the operation we are about to run.
|
|
90
103
|
"""
|
|
91
104
|
PrintHooks.before_log_action_run()
|
|
92
|
-
|
|
105
|
+
log.message("%s Action: `%s`", EMOJI_START, action.name)
|
|
93
106
|
log.message("Running: `%s`", operation.command_line(with_options=True))
|
|
94
107
|
if len(action.param_value_summary()) > 0:
|
|
95
108
|
log.message("Parameters:\n%s", action.param_value_summary_str())
|
|
@@ -106,8 +119,9 @@ def check_for_existing_result(
|
|
|
106
119
|
already exist.
|
|
107
120
|
"""
|
|
108
121
|
action = context.action
|
|
109
|
-
|
|
110
|
-
|
|
122
|
+
settings = context.settings
|
|
123
|
+
ws = settings.workspace
|
|
124
|
+
rerun = settings.rerun
|
|
111
125
|
|
|
112
126
|
existing_result = None
|
|
113
127
|
|
|
@@ -154,6 +168,7 @@ def run_action_operation(
|
|
|
154
168
|
|
|
155
169
|
# Run the action.
|
|
156
170
|
action = context.action
|
|
171
|
+
settings = context.settings
|
|
157
172
|
if action.run_per_item:
|
|
158
173
|
result = _run_for_each_item(context, action_input)
|
|
159
174
|
else:
|
|
@@ -172,9 +187,9 @@ def run_action_operation(
|
|
|
172
187
|
item.update_history(Source(operation=this_op, output_num=i, cacheable=action.cacheable))
|
|
173
188
|
|
|
174
189
|
# Override the state if appropriate (this handles marking items as transient).
|
|
175
|
-
if
|
|
190
|
+
if settings.override_state:
|
|
176
191
|
for item in result.items:
|
|
177
|
-
item.state =
|
|
192
|
+
item.state = settings.override_state
|
|
178
193
|
|
|
179
194
|
log.info("Action `%s` result: %s", action.name, result)
|
|
180
195
|
|
|
@@ -233,7 +248,7 @@ def _run_for_each_item(context: ExecContext, input: ActionInput) -> ActionResult
|
|
|
233
248
|
log.info("Caught SkipItem exception, skipping run on this item")
|
|
234
249
|
result_items.append(item)
|
|
235
250
|
continue
|
|
236
|
-
except
|
|
251
|
+
except get_nonfatal_exceptions() as e:
|
|
237
252
|
errors.append(e)
|
|
238
253
|
had_error = True
|
|
239
254
|
|
|
@@ -293,14 +308,18 @@ def save_action_result(
|
|
|
293
308
|
fmt_lines(skipped_paths),
|
|
294
309
|
)
|
|
295
310
|
|
|
296
|
-
|
|
311
|
+
unsaved_items = [item for item in input_items if not item.store_path]
|
|
312
|
+
input_store_paths = [StorePath(item.store_path) for item in input_items if item.store_path]
|
|
297
313
|
result_store_paths = [StorePath(item.store_path) for item in result.items if item.store_path]
|
|
298
314
|
old_inputs = sorted(set(input_store_paths) - set(result_store_paths))
|
|
315
|
+
if unsaved_items:
|
|
316
|
+
log.info("unsaved_items:\n%s", fmt_lines(unsaved_items))
|
|
299
317
|
log.info("result_store_paths:\n%s", fmt_lines(result_store_paths))
|
|
300
|
-
|
|
318
|
+
if old_inputs:
|
|
319
|
+
log.info("old_inputs:\n%s", fmt_lines(old_inputs))
|
|
301
320
|
|
|
302
321
|
# If there is a hint that the action replaces the input, archive any inputs that are not in the result.
|
|
303
|
-
archived_store_paths = []
|
|
322
|
+
archived_store_paths: list[StorePath] = []
|
|
304
323
|
if result.replaces_input and input_items:
|
|
305
324
|
for input_store_path in old_inputs:
|
|
306
325
|
# Note some outputs may be missing if replace_input was used.
|
|
@@ -325,7 +344,8 @@ def run_action_with_caching(
|
|
|
325
344
|
Note: Mutates the input but only to add `context` to each item.
|
|
326
345
|
"""
|
|
327
346
|
action = context.action
|
|
328
|
-
|
|
347
|
+
settings = context.settings
|
|
348
|
+
ws = settings.workspace
|
|
329
349
|
|
|
330
350
|
# For convenience, we include the context to each item too (this helps so per-item
|
|
331
351
|
# functions don't have to take context args everywhere).
|
|
@@ -341,7 +361,7 @@ def run_action_with_caching(
|
|
|
341
361
|
# Check if a previous run already produced the result.
|
|
342
362
|
existing_result = check_for_existing_result(context, action_input, operation)
|
|
343
363
|
|
|
344
|
-
if existing_result and not
|
|
364
|
+
if existing_result and not settings.rerun:
|
|
345
365
|
# Use the cached result.
|
|
346
366
|
result = existing_result
|
|
347
367
|
result_store_paths = [StorePath(not_none(item.store_path)) for item in result.items]
|
|
@@ -349,7 +369,7 @@ def run_action_with_caching(
|
|
|
349
369
|
|
|
350
370
|
PrintHooks.before_done_message()
|
|
351
371
|
log.message(
|
|
352
|
-
"%s
|
|
372
|
+
"%s Skipped: `%s` completed with %s %s",
|
|
353
373
|
EMOJI_SKIP,
|
|
354
374
|
action.name,
|
|
355
375
|
len(result.items),
|
|
@@ -359,12 +379,12 @@ def run_action_with_caching(
|
|
|
359
379
|
# Run it!
|
|
360
380
|
result = run_action_operation(context, action_input, operation)
|
|
361
381
|
result_store_paths, archived_store_paths = save_action_result(
|
|
362
|
-
ws, result, action_input, as_tmp=
|
|
382
|
+
ws, result, action_input, as_tmp=settings.tmp_output, no_format=settings.no_format
|
|
363
383
|
)
|
|
364
384
|
|
|
365
385
|
PrintHooks.before_done_message()
|
|
366
386
|
log.message(
|
|
367
|
-
"%s
|
|
387
|
+
"%s Done: `%s` completed with %s %s",
|
|
368
388
|
EMOJI_SUCCESS,
|
|
369
389
|
action.name,
|
|
370
390
|
len(result.items),
|
|
@@ -415,8 +435,7 @@ def run_action_with_shell_context(
|
|
|
415
435
|
action_name = action.name
|
|
416
436
|
|
|
417
437
|
# Execution context. This is fixed for the duration of the action.
|
|
418
|
-
|
|
419
|
-
action=action,
|
|
438
|
+
settings = RuntimeSettings(
|
|
420
439
|
workspace_dir=ws.base_dir,
|
|
421
440
|
rerun=rerun,
|
|
422
441
|
refetch=refetch,
|
|
@@ -424,6 +443,7 @@ def run_action_with_shell_context(
|
|
|
424
443
|
tmp_output=tmp_output,
|
|
425
444
|
no_format=no_format,
|
|
426
445
|
)
|
|
446
|
+
context = ExecContext(action, settings)
|
|
427
447
|
|
|
428
448
|
# Collect args from the provided args or otherwise the current selection.
|
|
429
449
|
args, from_selection = assemble_action_args(*provided_args, use_selection=action.uses_selection)
|
kash/exec/llm_transforms.py
CHANGED
|
@@ -12,7 +12,8 @@ from kash.llm_utils.fuzzy_parsing import strip_markdown_fence
|
|
|
12
12
|
from kash.llm_utils.llm_completion import llm_template_completion
|
|
13
13
|
from kash.llm_utils.llm_messages import Message, MessageTemplate
|
|
14
14
|
from kash.model.actions_model import LLMOptions
|
|
15
|
-
from kash.model.items_model import Item
|
|
15
|
+
from kash.model.items_model import Item
|
|
16
|
+
from kash.text_handling.doc_normalization import normalize_formatting
|
|
16
17
|
from kash.utils.errors import InvalidInput
|
|
17
18
|
from kash.utils.file_utils.file_formats_model import Format
|
|
18
19
|
|
|
@@ -90,6 +91,7 @@ def llm_transform_item(
|
|
|
90
91
|
normalize: bool = True,
|
|
91
92
|
strip_fence: bool = True,
|
|
92
93
|
check_no_results: bool = True,
|
|
94
|
+
format: Format | None = None,
|
|
93
95
|
) -> Item:
|
|
94
96
|
"""
|
|
95
97
|
Main function for running an LLM action on an item.
|
|
@@ -110,12 +112,13 @@ def llm_transform_item(
|
|
|
110
112
|
log.message("LLM transform from action `%s` on item: %s", action.name, item)
|
|
111
113
|
log.message("LLM options: %s", action.llm_options)
|
|
112
114
|
|
|
113
|
-
|
|
115
|
+
format = format or item.format or Format.markdown
|
|
116
|
+
result_item = item.derived_copy(body=None, format=format)
|
|
114
117
|
result_str = llm_transform_str(llm_options, item.body, check_no_results=check_no_results)
|
|
115
118
|
if strip_fence:
|
|
116
119
|
result_str = strip_markdown_fence(result_str)
|
|
117
120
|
if normalize:
|
|
118
|
-
result_str =
|
|
121
|
+
result_str = normalize_formatting(result_str, format=format)
|
|
119
122
|
|
|
120
123
|
result_item.body = result_str
|
|
121
124
|
return result_item
|
kash/exec/preconditions.py
CHANGED
|
@@ -8,6 +8,7 @@ from chopdiff.html import has_timestamp
|
|
|
8
8
|
from kash.exec.precondition_registry import kash_precondition
|
|
9
9
|
from kash.model.items_model import Item, ItemType
|
|
10
10
|
from kash.text_handling.markdown_utils import extract_bullet_points
|
|
11
|
+
from kash.utils.file_utils.file_formats import is_full_html_page
|
|
11
12
|
from kash.utils.file_utils.file_formats_model import Format
|
|
12
13
|
|
|
13
14
|
|
|
@@ -84,13 +85,18 @@ def contains_curly_vars(item: Item) -> bool:
|
|
|
84
85
|
|
|
85
86
|
|
|
86
87
|
@kash_precondition
|
|
87
|
-
def
|
|
88
|
-
return has_body(item) and item.format
|
|
88
|
+
def has_simple_text_body(item: Item) -> bool:
|
|
89
|
+
return bool(has_body(item) and item.format and item.format.is_simple_text)
|
|
89
90
|
|
|
90
91
|
|
|
91
92
|
@kash_precondition
|
|
92
93
|
def has_html_body(item: Item) -> bool:
|
|
93
|
-
return has_body(item) and item.format
|
|
94
|
+
return bool(has_body(item) and item.format and item.format.is_html)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
@kash_precondition
|
|
98
|
+
def has_full_html_page_body(item: Item) -> bool:
|
|
99
|
+
return bool(has_html_body(item) and item.body and is_full_html_page(item.body))
|
|
94
100
|
|
|
95
101
|
|
|
96
102
|
@kash_precondition
|
|
@@ -100,7 +106,7 @@ def is_plaintext(item: Item) -> bool:
|
|
|
100
106
|
|
|
101
107
|
@kash_precondition
|
|
102
108
|
def is_markdown(item: Item) -> bool:
|
|
103
|
-
return has_body(item) and item.format
|
|
109
|
+
return bool(has_body(item) and item.format and item.format.is_markdown)
|
|
104
110
|
|
|
105
111
|
|
|
106
112
|
@kash_precondition
|
|
@@ -113,14 +119,6 @@ def is_html(item: Item) -> bool:
|
|
|
113
119
|
return has_body(item) and item.format == Format.html
|
|
114
120
|
|
|
115
121
|
|
|
116
|
-
@kash_precondition
|
|
117
|
-
def is_text_doc(item: Item) -> bool:
|
|
118
|
-
"""
|
|
119
|
-
A document that can be processed by LLMs and other plaintext tools.
|
|
120
|
-
"""
|
|
121
|
-
return (is_plaintext(item) or is_markdown(item)) and has_body(item)
|
|
122
|
-
|
|
123
|
-
|
|
124
122
|
@kash_precondition
|
|
125
123
|
def is_markdown_list(item: Item) -> bool:
|
|
126
124
|
try:
|
kash/exec/resolve_args.py
CHANGED
|
@@ -105,6 +105,10 @@ def assemble_action_args(
|
|
|
105
105
|
|
|
106
106
|
|
|
107
107
|
def resolvable_paths(paths: Sequence[StorePath | Path]) -> list[StorePath]:
|
|
108
|
+
"""
|
|
109
|
+
Return which of the given StorePaths are resolvable (exist) in the
|
|
110
|
+
current workspace.
|
|
111
|
+
"""
|
|
108
112
|
ws = current_ws()
|
|
109
113
|
resolvable = list(filter(None, (ws.resolve_path(p) for p in paths)))
|
|
110
114
|
return resolvable
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import contextvars
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
from kash.config.logger import get_logger
|
|
6
|
+
from kash.model.exec_model import RuntimeSettings
|
|
7
|
+
from kash.model.items_model import State
|
|
8
|
+
from kash.workspaces.workspace_dirs import enclosing_ws_dir, global_ws_dir
|
|
9
|
+
|
|
10
|
+
log = get_logger(__name__)
|
|
11
|
+
|
|
12
|
+
_current_settings: contextvars.ContextVar[RuntimeSettings | None] = contextvars.ContextVar(
|
|
13
|
+
"current_runtime_settings", default=None
|
|
14
|
+
)
|
|
15
|
+
"""
|
|
16
|
+
Context variable that tracks the current runtime settings. Only used if it is
|
|
17
|
+
explicitly set with a `with runtime_settings(...):` block.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def current_runtime_settings() -> RuntimeSettings:
|
|
22
|
+
"""
|
|
23
|
+
Get the current runtime settings. Uses the ambient context var settings if
|
|
24
|
+
set and otherwise infers the workspace from the current working directory
|
|
25
|
+
with default runtime settings.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
ambient_settings = _current_settings.get()
|
|
29
|
+
if ambient_settings:
|
|
30
|
+
return ambient_settings
|
|
31
|
+
|
|
32
|
+
default_ws_dir = enclosing_ws_dir() or global_ws_dir()
|
|
33
|
+
return RuntimeSettings(default_ws_dir)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True)
|
|
37
|
+
class WsContext:
|
|
38
|
+
global_ws_dir: Path
|
|
39
|
+
enclosing_ws_dir: Path | None
|
|
40
|
+
override_dir: Path | None
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def current_ws_dir(self) -> Path:
|
|
44
|
+
if self.override_dir:
|
|
45
|
+
return self.override_dir
|
|
46
|
+
elif self.enclosing_ws_dir:
|
|
47
|
+
return self.enclosing_ws_dir
|
|
48
|
+
else:
|
|
49
|
+
return self.global_ws_dir
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def current_ws_context() -> WsContext:
|
|
53
|
+
"""
|
|
54
|
+
Context path info about the current workspace, including the global workspace
|
|
55
|
+
directory, any workspace directory that encloses the current working directory,
|
|
56
|
+
and override set via runtime settings.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
override_dir = None
|
|
60
|
+
ambient_settings = _current_settings.get()
|
|
61
|
+
if ambient_settings:
|
|
62
|
+
override_dir = ambient_settings.workspace_dir
|
|
63
|
+
|
|
64
|
+
return WsContext(global_ws_dir(), enclosing_ws_dir(), override_dir)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass
|
|
68
|
+
class RuntimeSettingsManager:
|
|
69
|
+
"""
|
|
70
|
+
Manage the context for executing actions, including `RuntimeSettings` and
|
|
71
|
+
`Workspace`.
|
|
72
|
+
|
|
73
|
+
This is a minimal base class for use as a context manager. Most functionality
|
|
74
|
+
is still in `FileStore`.
|
|
75
|
+
|
|
76
|
+
Workspaces may be detected based on the current working directory or explicitly
|
|
77
|
+
set using a `with` block:
|
|
78
|
+
```
|
|
79
|
+
ws = get_ws("my_workspace")
|
|
80
|
+
with ws:
|
|
81
|
+
# code that calls current_ws() will use this workspace
|
|
82
|
+
```
|
|
83
|
+
"""
|
|
84
|
+
|
|
85
|
+
settings: RuntimeSettings
|
|
86
|
+
|
|
87
|
+
def __enter__(self):
|
|
88
|
+
self._token = _current_settings.set(self.settings)
|
|
89
|
+
log.info("New runtime context: %s", self.settings)
|
|
90
|
+
return self.settings
|
|
91
|
+
|
|
92
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
93
|
+
_current_settings.reset(self._token)
|
|
94
|
+
log.info("Exiting runtime context: %s", self.settings)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def kash_runtime(
|
|
98
|
+
workspace_dir: Path | None,
|
|
99
|
+
*,
|
|
100
|
+
rerun: bool = False,
|
|
101
|
+
refetch: bool = False,
|
|
102
|
+
override_state: State | None = None,
|
|
103
|
+
tmp_output: bool = False,
|
|
104
|
+
no_format: bool = False,
|
|
105
|
+
) -> RuntimeSettingsManager:
|
|
106
|
+
"""
|
|
107
|
+
Set a specific kash execution context for a with block.
|
|
108
|
+
This allows defining a workspace and other execution settings as the ambient
|
|
109
|
+
context within the block.
|
|
110
|
+
|
|
111
|
+
If `workspace_dir` is not provided, the current workspace will be inferred
|
|
112
|
+
from the working directory or fall back to the global workspace.
|
|
113
|
+
|
|
114
|
+
Example usage:
|
|
115
|
+
```
|
|
116
|
+
with kash_runtime(ws_path, rerun=args.rerun) as runtime:
|
|
117
|
+
runtime.workspace.log_workspace_info()
|
|
118
|
+
# Perform actions.
|
|
119
|
+
```
|
|
120
|
+
"""
|
|
121
|
+
from kash.workspaces.workspaces import current_ws
|
|
122
|
+
|
|
123
|
+
if workspace_dir is None:
|
|
124
|
+
workspace_dir = current_ws().base_dir
|
|
125
|
+
|
|
126
|
+
settings = RuntimeSettings(
|
|
127
|
+
workspace_dir=workspace_dir,
|
|
128
|
+
rerun=rerun,
|
|
129
|
+
refetch=refetch,
|
|
130
|
+
override_state=override_state,
|
|
131
|
+
tmp_output=tmp_output,
|
|
132
|
+
no_format=no_format,
|
|
133
|
+
)
|
|
134
|
+
return RuntimeSettingsManager(settings=settings)
|
|
@@ -13,7 +13,7 @@ from kash.model.params_model import RawParamValues
|
|
|
13
13
|
from kash.shell.output.shell_output import PrintHooks
|
|
14
14
|
from kash.shell.utils.exception_printing import summarize_traceback
|
|
15
15
|
from kash.utils.common.parse_shell_args import parse_shell_args
|
|
16
|
-
from kash.utils.errors import
|
|
16
|
+
from kash.utils.errors import get_nonfatal_exceptions
|
|
17
17
|
|
|
18
18
|
log = get_logger(__name__)
|
|
19
19
|
|
|
@@ -71,14 +71,16 @@ class ShellCallableAction:
|
|
|
71
71
|
action_cls, explicit_values, *shell_args.args, rerun=rerun, refetch=refetch
|
|
72
72
|
)
|
|
73
73
|
# We don't return the result to keep the xonsh shell output clean.
|
|
74
|
-
except
|
|
74
|
+
except get_nonfatal_exceptions() as e:
|
|
75
75
|
PrintHooks.nonfatal_exception()
|
|
76
76
|
log.error(f"[{COLOR_ERROR}]Action error:[/{COLOR_ERROR}] %s", summarize_traceback(e))
|
|
77
77
|
log.info("Action error details: %s", e, exc_info=True)
|
|
78
78
|
return ShellResult(exception=e)
|
|
79
79
|
except Exception as e:
|
|
80
80
|
# Log here while we are in the true call stack (not inside the xonsh call stack).
|
|
81
|
-
log.error(
|
|
81
|
+
log.error(
|
|
82
|
+
"Action error: %s", e, exc_info=KashEnv.KASH_SHOW_TRACEBACK.read_bool(default=True)
|
|
83
|
+
)
|
|
82
84
|
raise
|
|
83
85
|
finally:
|
|
84
86
|
log_tallies(level="warning", if_slower_than=10.0)
|
kash/file_storage/file_store.py
CHANGED
|
@@ -5,7 +5,7 @@ import time
|
|
|
5
5
|
from collections.abc import Callable, Generator
|
|
6
6
|
from os.path import join, relpath
|
|
7
7
|
from pathlib import Path
|
|
8
|
-
from typing import
|
|
8
|
+
from typing import Concatenate, ParamSpec, TypeVar
|
|
9
9
|
|
|
10
10
|
from funlog import format_duration, log_calls
|
|
11
11
|
from prettyfmt import fmt_lines, fmt_path
|
|
@@ -13,13 +13,13 @@ from strif import copyfile_atomic, hash_file, move_file
|
|
|
13
13
|
from typing_extensions import override
|
|
14
14
|
|
|
15
15
|
from kash.config.logger import get_log_settings, get_logger
|
|
16
|
-
from kash.config.text_styles import EMOJI_SAVED
|
|
16
|
+
from kash.config.text_styles import EMOJI_SAVED
|
|
17
17
|
from kash.file_storage.item_file_format import read_item, write_item
|
|
18
18
|
from kash.file_storage.metadata_dirs import MetadataDirs
|
|
19
19
|
from kash.file_storage.store_filenames import folder_for_type, join_suffix, parse_item_filename
|
|
20
20
|
from kash.model.items_model import Item, ItemId, ItemType
|
|
21
21
|
from kash.model.paths_model import StorePath
|
|
22
|
-
from kash.shell.output.shell_output import PrintHooks
|
|
22
|
+
from kash.shell.output.shell_output import PrintHooks
|
|
23
23
|
from kash.utils.common.format_utils import fmt_loc
|
|
24
24
|
from kash.utils.common.uniquifier import Uniquifier
|
|
25
25
|
from kash.utils.common.url import Locator, Url, is_url
|
|
@@ -34,16 +34,18 @@ from kash.workspaces.workspaces import Workspace
|
|
|
34
34
|
log = get_logger(__name__)
|
|
35
35
|
|
|
36
36
|
|
|
37
|
+
SelfT = TypeVar("SelfT")
|
|
37
38
|
T = TypeVar("T")
|
|
39
|
+
P = ParamSpec("P")
|
|
38
40
|
|
|
39
41
|
|
|
40
|
-
def synchronized(method: Callable[
|
|
42
|
+
def synchronized(method: Callable[Concatenate[SelfT, P], T]) -> Callable[Concatenate[SelfT, P], T]:
|
|
41
43
|
"""
|
|
42
44
|
Simple way to synchronize a few methods.
|
|
43
45
|
"""
|
|
44
46
|
|
|
45
47
|
@functools.wraps(method)
|
|
46
|
-
def synchronized_method(self, *args:
|
|
48
|
+
def synchronized_method(self, *args: P.args, **kwargs: P.kwargs) -> T:
|
|
47
49
|
with self._lock:
|
|
48
50
|
return method(self, *args, **kwargs)
|
|
49
51
|
|
|
@@ -316,9 +318,11 @@ class FileStore(Workspace):
|
|
|
316
318
|
Save the item. Uses the `store_path` if it's already set or generates a new one.
|
|
317
319
|
Updates `item.store_path`.
|
|
318
320
|
|
|
321
|
+
Unless `no_format` is true, also normalizes body text formatting (for Markdown)
|
|
322
|
+
and updates the item's body to match.
|
|
323
|
+
|
|
319
324
|
If `as_tmp` is true, will save the item to a temporary file.
|
|
320
325
|
If `overwrite` is false, will skip saving if the item already exists.
|
|
321
|
-
If `no_format` is true, will not normalize body text formatting (for Markdown).
|
|
322
326
|
"""
|
|
323
327
|
# If external file already exists within the workspace, the file is already saved (without metadata).
|
|
324
328
|
external_path = item.external_path and Path(item.external_path).resolve()
|
|
@@ -450,19 +454,11 @@ class FileStore(Workspace):
|
|
|
450
454
|
if not path.exists():
|
|
451
455
|
raise FileNotFound(f"File not found: {fmt_loc(path)}")
|
|
452
456
|
|
|
453
|
-
#
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
# Best guesses on item types if not specified.
|
|
457
|
-
item_type = as_type
|
|
458
|
-
if not item_type and filename_item_type:
|
|
459
|
-
item_type = filename_item_type
|
|
460
|
-
if not item_type and format:
|
|
461
|
-
item_type = ItemType.for_format(format)
|
|
462
|
-
if not item_type:
|
|
463
|
-
item_type = ItemType.resource
|
|
457
|
+
# First treat it as an external file to analyze file type and format.
|
|
458
|
+
item = Item.from_external_path(path)
|
|
464
459
|
|
|
465
|
-
|
|
460
|
+
# If it's a text/frontmatter-friendly, read it fully.
|
|
461
|
+
if item.format and item.format.supports_frontmatter:
|
|
466
462
|
log.message("Importing text file: %s", fmt_loc(path))
|
|
467
463
|
# This will read the file with or without frontmatter.
|
|
468
464
|
# We are importing so we want to drop the external path so we save the body.
|
|
@@ -483,18 +479,27 @@ class FileStore(Workspace):
|
|
|
483
479
|
store_path = self.save(item)
|
|
484
480
|
log.info("Imported text file: %s", item.as_str())
|
|
485
481
|
else:
|
|
486
|
-
log.message("Importing non-text file: %s", fmt_loc(path))
|
|
487
482
|
# Binary or other files we just copy over as-is, preserving the name.
|
|
488
483
|
# We know the extension is recognized.
|
|
489
|
-
|
|
490
|
-
store_path, _found, _prev = self.store_path_for(item)
|
|
484
|
+
store_path, _found, old_store_path = self.store_path_for(item)
|
|
491
485
|
if self.exists(store_path):
|
|
492
486
|
raise FileExists(f"Resource already in store: {fmt_loc(store_path)}")
|
|
493
487
|
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
log.message("Importing resource: %s -> %s", fmt_loc(path), fmt_loc(store_path))
|
|
488
|
+
log.message("Importing resource: %s", fmt_loc(path))
|
|
497
489
|
copyfile_atomic(path, self.base_dir / store_path, make_parents=True)
|
|
490
|
+
|
|
491
|
+
# Optimization: Don't import an identical file twice.
|
|
492
|
+
if old_store_path:
|
|
493
|
+
old_hash = self.hash(old_store_path)
|
|
494
|
+
new_hash = self.hash(store_path)
|
|
495
|
+
if old_hash == new_hash:
|
|
496
|
+
log.message(
|
|
497
|
+
"Imported resource is identical to the previous import: %s",
|
|
498
|
+
fmt_loc(old_store_path),
|
|
499
|
+
)
|
|
500
|
+
os.unlink(self.base_dir / store_path)
|
|
501
|
+
store_path = old_store_path
|
|
502
|
+
log.message("Imported resource: %s", fmt_loc(store_path))
|
|
498
503
|
return store_path
|
|
499
504
|
|
|
500
505
|
def import_items(
|
|
@@ -584,12 +589,13 @@ class FileStore(Workspace):
|
|
|
584
589
|
move_file(full_input_path, original_path)
|
|
585
590
|
return StorePath(store_path)
|
|
586
591
|
|
|
587
|
-
|
|
592
|
+
@synchronized
|
|
593
|
+
def log_workspace_info(self, *, once: bool = False) -> bool:
|
|
588
594
|
"""
|
|
589
595
|
Log helpful information about the workspace.
|
|
590
596
|
"""
|
|
591
597
|
if once and self.info_logged:
|
|
592
|
-
return
|
|
598
|
+
return False
|
|
593
599
|
|
|
594
600
|
self.info_logged = True
|
|
595
601
|
|
|
@@ -604,25 +610,18 @@ class FileStore(Workspace):
|
|
|
604
610
|
fmt_path(get_log_settings().log_file_path.absolute(), rel_to_cwd=False),
|
|
605
611
|
)
|
|
606
612
|
log.message(
|
|
607
|
-
"
|
|
608
|
-
|
|
609
|
-
log.message(
|
|
610
|
-
"Content cache: %s",
|
|
613
|
+
"Caches: %s, %s",
|
|
614
|
+
fmt_path(self.base_dir / self.dirs.media_cache_dir, rel_to_cwd=False),
|
|
611
615
|
fmt_path(self.base_dir / self.dirs.content_cache_dir, rel_to_cwd=False),
|
|
612
616
|
)
|
|
617
|
+
log.message("Current working directory: %s", fmt_path(Path.cwd(), rel_to_cwd=False))
|
|
618
|
+
|
|
613
619
|
for warning in self.warnings:
|
|
614
620
|
log.warning("%s", warning)
|
|
615
621
|
|
|
616
|
-
if self.is_global_ws:
|
|
617
|
-
PrintHooks.spacer()
|
|
618
|
-
log.warning("Note you are currently using the default global workspace.")
|
|
619
|
-
cprint(
|
|
620
|
-
"Create or switch to another workspace with the `workspace` command.",
|
|
621
|
-
style=STYLE_HINT,
|
|
622
|
-
)
|
|
623
|
-
|
|
624
622
|
log.info("File store startup took %s.", format_duration(self.end_time - self.start_time))
|
|
625
623
|
# TODO: Log more info like number of items by type.
|
|
624
|
+
return True
|
|
626
625
|
|
|
627
626
|
def walk_items(
|
|
628
627
|
self,
|
|
@@ -7,7 +7,7 @@ from prettyfmt import custom_key_sort, fmt_size_human
|
|
|
7
7
|
from kash.config.logger import get_logger
|
|
8
8
|
from kash.model.items_model import ITEM_FIELDS, Item
|
|
9
9
|
from kash.model.operations_model import OPERATION_FIELDS
|
|
10
|
-
from kash.text_handling.doc_normalization import
|
|
10
|
+
from kash.text_handling.doc_normalization import normalize_formatting
|
|
11
11
|
from kash.utils.common.format_utils import fmt_loc
|
|
12
12
|
from kash.utils.file_utils.file_formats_model import Format
|
|
13
13
|
from kash.utils.file_utils.mtime_cache import MtimeCache
|
|
@@ -25,7 +25,7 @@ _item_cache = MtimeCache[Item](max_size=2000, name="Item")
|
|
|
25
25
|
def write_item(item: Item, path: Path, normalize: bool = True):
|
|
26
26
|
"""
|
|
27
27
|
Write a text item to a file with standard frontmatter format YAML.
|
|
28
|
-
|
|
28
|
+
By default normalizes formatting of the body text and updates the item's body.
|
|
29
29
|
"""
|
|
30
30
|
item.validate()
|
|
31
31
|
if item.is_binary:
|
|
@@ -37,7 +37,7 @@ def write_item(item: Item, path: Path, normalize: bool = True):
|
|
|
37
37
|
_item_cache.delete(path)
|
|
38
38
|
|
|
39
39
|
if normalize:
|
|
40
|
-
body =
|
|
40
|
+
body = normalize_formatting(item.body_text(), item.format)
|
|
41
41
|
else:
|
|
42
42
|
body = item.body_text()
|
|
43
43
|
|
|
@@ -79,6 +79,9 @@ def write_item(item: Item, path: Path, normalize: bool = True):
|
|
|
79
79
|
# Update cache.
|
|
80
80
|
_item_cache.update(path, item)
|
|
81
81
|
|
|
82
|
+
# Update the item's body to reflect normalization.
|
|
83
|
+
item.body = body
|
|
84
|
+
|
|
82
85
|
|
|
83
86
|
def read_item(path: Path, base_dir: Path | None) -> Item:
|
|
84
87
|
"""
|
|
@@ -1,15 +1,18 @@
|
|
|
1
|
+
from functools import cache
|
|
1
2
|
from pathlib import Path
|
|
2
3
|
|
|
3
4
|
from kash.config.logger import get_logger
|
|
4
5
|
from kash.model.items_model import ItemType
|
|
6
|
+
from kash.utils.common.inflection import plural
|
|
5
7
|
from kash.utils.file_utils.file_formats_model import FileExt, Format
|
|
6
8
|
from kash.utils.file_utils.filename_parsing import split_filename
|
|
7
|
-
from kash.utils.lang_utils.inflection import plural
|
|
8
9
|
|
|
9
10
|
log = get_logger(__name__)
|
|
10
11
|
|
|
11
12
|
|
|
12
|
-
|
|
13
|
+
@cache
|
|
14
|
+
def _get_type_to_folder() -> dict[str, str]:
|
|
15
|
+
return {name: plural(name) for name, _value in ItemType.__members__.items()}
|
|
13
16
|
|
|
14
17
|
|
|
15
18
|
def folder_for_type(item_type: ItemType) -> Path:
|
|
@@ -22,7 +25,7 @@ def folder_for_type(item_type: ItemType) -> Path:
|
|
|
22
25
|
export -> exports
|
|
23
26
|
etc.
|
|
24
27
|
"""
|
|
25
|
-
return Path(
|
|
28
|
+
return Path(_get_type_to_folder()[item_type.name])
|
|
26
29
|
|
|
27
30
|
|
|
28
31
|
def join_suffix(base_slug: str, full_suffix: str) -> str:
|