kash-shell 0.3.27__py3-none-any.whl → 0.3.30__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kash/actions/core/markdownify_html.py +1 -4
- kash/actions/core/minify_html.py +4 -5
- kash/actions/core/render_as_html.py +9 -7
- kash/actions/core/save_sidematter_meta.py +47 -0
- kash/actions/core/zip_sidematter.py +47 -0
- kash/commands/base/basic_file_commands.py +7 -4
- kash/commands/base/diff_commands.py +6 -4
- kash/commands/base/files_command.py +31 -30
- kash/commands/base/general_commands.py +3 -2
- kash/commands/base/logs_commands.py +6 -4
- kash/commands/base/reformat_command.py +3 -2
- kash/commands/base/search_command.py +4 -3
- kash/commands/base/show_command.py +9 -7
- kash/commands/help/assistant_commands.py +6 -4
- kash/commands/help/help_commands.py +7 -4
- kash/commands/workspace/selection_commands.py +18 -16
- kash/commands/workspace/workspace_commands.py +39 -26
- kash/config/setup.py +2 -27
- kash/docs/markdown/topics/a1_what_is_kash.md +26 -18
- kash/exec/action_decorators.py +2 -2
- kash/exec/action_exec.py +56 -50
- kash/exec/fetch_url_items.py +36 -9
- kash/exec/preconditions.py +2 -2
- kash/exec/resolve_args.py +4 -1
- kash/exec/runtime_settings.py +1 -0
- kash/file_storage/file_store.py +59 -23
- kash/file_storage/item_file_format.py +91 -26
- kash/help/help_types.py +1 -1
- kash/llm_utils/llms.py +6 -1
- kash/local_server/local_server_commands.py +2 -1
- kash/mcp/mcp_server_commands.py +3 -2
- kash/mcp/mcp_server_routes.py +1 -1
- kash/model/actions_model.py +31 -30
- kash/model/compound_actions_model.py +4 -3
- kash/model/exec_model.py +30 -3
- kash/model/items_model.py +114 -57
- kash/model/params_model.py +4 -4
- kash/shell/output/shell_output.py +1 -2
- kash/utils/file_formats/chat_format.py +7 -4
- kash/utils/file_utils/file_ext.py +1 -0
- kash/utils/file_utils/file_formats.py +4 -2
- kash/utils/file_utils/file_formats_model.py +12 -0
- kash/utils/text_handling/doc_normalization.py +1 -1
- kash/utils/text_handling/markdown_footnotes.py +224 -0
- kash/utils/text_handling/markdown_utils.py +532 -41
- kash/utils/text_handling/markdownify_utils.py +2 -1
- kash/web_gen/templates/components/tooltip_scripts.js.jinja +186 -1
- kash/web_gen/templates/components/youtube_popover_scripts.js.jinja +223 -0
- kash/web_gen/templates/components/youtube_popover_styles.css.jinja +150 -0
- kash/web_gen/templates/content_styles.css.jinja +53 -1
- kash/web_gen/templates/youtube_webpage.html.jinja +47 -0
- kash/web_gen/webpage_render.py +103 -0
- kash/workspaces/workspaces.py +0 -5
- kash/xonsh_custom/custom_shell.py +4 -3
- kash/xonsh_custom/shell_load_commands.py +3 -9
- {kash_shell-0.3.27.dist-info → kash_shell-0.3.30.dist-info}/METADATA +34 -24
- {kash_shell-0.3.27.dist-info → kash_shell-0.3.30.dist-info}/RECORD +60 -55
- kash/llm_utils/llm_features.py +0 -72
- kash/web_gen/simple_webpage.py +0 -55
- {kash_shell-0.3.27.dist-info → kash_shell-0.3.30.dist-info}/WHEEL +0 -0
- {kash_shell-0.3.27.dist-info → kash_shell-0.3.30.dist-info}/entry_points.txt +0 -0
- {kash_shell-0.3.27.dist-info → kash_shell-0.3.30.dist-info}/licenses/LICENSE +0 -0
kash/config/setup.py
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
from enum import Enum
|
|
2
1
|
from functools import cache
|
|
3
2
|
from pathlib import Path
|
|
4
|
-
from typing import Any
|
|
5
3
|
|
|
6
4
|
from clideps.env_vars.dotenv_utils import load_dotenv_paths
|
|
7
5
|
|
|
@@ -77,29 +75,6 @@ def kash_setup(
|
|
|
77
75
|
|
|
78
76
|
|
|
79
77
|
def _lib_setup():
|
|
80
|
-
from
|
|
81
|
-
from ruamel.yaml import Representer
|
|
78
|
+
from sidematter_format import register_default_yaml_representers
|
|
82
79
|
|
|
83
|
-
|
|
84
|
-
"""
|
|
85
|
-
Represent Enums as their values.
|
|
86
|
-
Helps make it easy to serialize enums to YAML everywhere.
|
|
87
|
-
We use the convention of storing enum values as readable strings.
|
|
88
|
-
"""
|
|
89
|
-
return dumper.represent_str(data.value)
|
|
90
|
-
|
|
91
|
-
add_default_yaml_customizer(
|
|
92
|
-
lambda yaml: yaml.representer.add_multi_representer(Enum, represent_enum)
|
|
93
|
-
)
|
|
94
|
-
|
|
95
|
-
# Maybe useful?
|
|
96
|
-
|
|
97
|
-
# from pydantic import BaseModel
|
|
98
|
-
|
|
99
|
-
# def represent_pydantic(dumper: Representer, data: BaseModel) -> Any:
|
|
100
|
-
# """Represent Pydantic models as YAML dictionaries."""
|
|
101
|
-
# return dumper.represent_dict(data.model_dump())
|
|
102
|
-
|
|
103
|
-
# add_default_yaml_customizer(
|
|
104
|
-
# lambda yaml: yaml.representer.add_multi_representer(BaseModel, represent_pydantic)
|
|
105
|
-
# )
|
|
80
|
+
register_default_yaml_representers()
|
|
@@ -1,3 +1,16 @@
|
|
|
1
|
+
## Hello!
|
|
2
|
+
|
|
3
|
+
If you’re seeing this, you there’s a good chance I shared it with you for feedback.
|
|
4
|
+
Thank you for checking out Kash.
|
|
5
|
+
|
|
6
|
+
It’s new, the result of some experimentation over the past few months.
|
|
7
|
+
I like a lot of things about it but it isn’t mature and I’d love your help to make it
|
|
8
|
+
more usable. If you try it please **let me know** what works and what doesn’t work.
|
|
9
|
+
Or if you just don’t get it, where you lost interest or got stuck.
|
|
10
|
+
My contact info is at [github.com/jlevy](https://github.com/jlevy) or [follow or DM
|
|
11
|
+
me](https://x.com/ojoshe) (I’m fastest on Twitter DMs).
|
|
12
|
+
Thank you. :)
|
|
13
|
+
|
|
1
14
|
## What is Kash?
|
|
2
15
|
|
|
3
16
|
> “*Simple should be simple.
|
|
@@ -12,9 +25,15 @@ It operates on “items” such as URLs, files, or Markdown notes within a works
|
|
|
12
25
|
directory.
|
|
13
26
|
|
|
14
27
|
You can use Kash as an **interactive, AI-native command-line** shell for practical
|
|
15
|
-
knowledge tasks.
|
|
16
|
-
|
|
17
|
-
|
|
28
|
+
knowledge tasks.
|
|
29
|
+
|
|
30
|
+
But it’s actually not just a shell, and you can skip the shell entirely.
|
|
31
|
+
It’s really simply **a Python library** that lets you convert a simple Python function
|
|
32
|
+
into “actions” that work in a clean way on plain files in a workspace.
|
|
33
|
+
An action is also an MCP tool, so it integrates with other tools like Anthropic Desktop
|
|
34
|
+
or Cursor.
|
|
35
|
+
|
|
36
|
+
So basically, it gives a unified way to use the shell, Python functions, and MCP tools.
|
|
18
37
|
|
|
19
38
|
It’s new and still has some rough edges, but it’s now working well enough it is feeling
|
|
20
39
|
quite powerful. It now serves as a replacement for my usual shell (previously bash or
|
|
@@ -73,10 +92,10 @@ quick to install via uv.
|
|
|
73
92
|
- **Support for any API:** Kash is tool agnostic and runs locally, on file inputs in
|
|
74
93
|
simple formats, so you own and manage your data and workspaces however you like.
|
|
75
94
|
You can use it with any models or APIs you like, and is already set up to use the APIs
|
|
76
|
-
of **OpenAI GPT-
|
|
77
|
-
Grok**, **Mistral**, **Groq (Llama, Qwen, Deepseek)** (via
|
|
78
|
-
**Perplexity**, **Firecrawl**, **Exa**, and any Python
|
|
79
|
-
There is also some experimental support for **LlamaIndex** and **ChromaDB**.
|
|
95
|
+
of **OpenAI** (GPT-5 is now the default model), **Anthropic Claude**, **Google
|
|
96
|
+
Gemini**, **xAI Grok**, **Mistral**, **Groq (Llama, Qwen, Deepseek)** (via
|
|
97
|
+
**LiteLLM**), **Deepgram**, **Perplexity**, **Firecrawl**, **Exa**, and any Python
|
|
98
|
+
libraries. There is also some experimental support for **LlamaIndex** and **ChromaDB**.
|
|
80
99
|
|
|
81
100
|
- **MCP support:** Finally, an action is also an **MCP tool server** so you can use it
|
|
82
101
|
in any MCP client, like Anthropic Desktop or Cursor.
|
|
@@ -110,14 +129,3 @@ I’ve separately built a new desktop terminal app, Kerm, which adds support for
|
|
|
110
129
|
“Kerm codes” protocol for such visual components, encoded as OSC codes then rendered in
|
|
111
130
|
the terminal. Because Kash supports these codes, as this develops you will get the
|
|
112
131
|
visuals of a web app layered on the flexibility of a text-based terminal.
|
|
113
|
-
|
|
114
|
-
### Is Kash Mature?
|
|
115
|
-
|
|
116
|
-
It’s the result of a couple months of coding and experimentation, and it’s still in
|
|
117
|
-
progress and has rough edges.
|
|
118
|
-
Please help me make it better by sharing your ideas and feedback!
|
|
119
|
-
It’s easiest to DM me at [twitter.com/ojoshe](https://x.com/ojoshe).
|
|
120
|
-
My contact info is at [github.com/jlevy](https://github.com/jlevy).
|
|
121
|
-
|
|
122
|
-
[**Please follow or DM me**](https://x.com/ojoshe) for future updates or if you have
|
|
123
|
-
ideas, feedback, or use cases for Kash!
|
kash/exec/action_decorators.py
CHANGED
|
@@ -31,7 +31,7 @@ from kash.model.actions_model import (
|
|
|
31
31
|
ParamSource,
|
|
32
32
|
TitleTemplate,
|
|
33
33
|
)
|
|
34
|
-
from kash.model.exec_model import ExecContext
|
|
34
|
+
from kash.model.exec_model import ActionContext, ExecContext
|
|
35
35
|
from kash.model.items_model import Item, ItemType
|
|
36
36
|
from kash.model.params_model import Param, ParamDeclarations, TypedParamValues
|
|
37
37
|
from kash.model.preconditions_model import Precondition
|
|
@@ -328,7 +328,7 @@ def kash_action(
|
|
|
328
328
|
super().__post_init__()
|
|
329
329
|
|
|
330
330
|
@override
|
|
331
|
-
def run(self, input: ActionInput, context:
|
|
331
|
+
def run(self, input: ActionInput, context: ActionContext) -> ActionResult:
|
|
332
332
|
# Map the final, current actions param values back to the function parameters.
|
|
333
333
|
pos_args: list[Any] = []
|
|
334
334
|
kw_args: dict[str, Any] = {}
|
kash/exec/action_exec.py
CHANGED
|
@@ -23,7 +23,7 @@ from kash.model.actions_model import (
|
|
|
23
23
|
ExecContext,
|
|
24
24
|
PathOpType,
|
|
25
25
|
)
|
|
26
|
-
from kash.model.exec_model import RuntimeSettings
|
|
26
|
+
from kash.model.exec_model import ActionContext, RuntimeSettings
|
|
27
27
|
from kash.model.items_model import Item, State
|
|
28
28
|
from kash.model.operations_model import Input, Operation, Source
|
|
29
29
|
from kash.model.params_model import ALL_COMMON_PARAMS, GLOBAL_PARAMS, RawParamValues
|
|
@@ -42,6 +42,7 @@ def prepare_action_input(*input_args: CommandArg, refetch: bool = False) -> Acti
|
|
|
42
42
|
Prepare input args, which may be URLs or paths, into items that correspond to
|
|
43
43
|
URL or file resources, either finding them in the workspace or importing them.
|
|
44
44
|
Also fetches metadata for URLs if they don't already have title and description.
|
|
45
|
+
Automatically imports any URLs and copies any sidematter-format files (metadata/assets).
|
|
45
46
|
"""
|
|
46
47
|
from kash.exec.fetch_url_items import fetch_url_item_content
|
|
47
48
|
|
|
@@ -49,7 +50,7 @@ def prepare_action_input(*input_args: CommandArg, refetch: bool = False) -> Acti
|
|
|
49
50
|
|
|
50
51
|
# Ensure input items are already saved in the workspace and load the corresponding items.
|
|
51
52
|
# This also imports any URLs.
|
|
52
|
-
input_items = [ws.import_and_load(arg) for arg in input_args]
|
|
53
|
+
input_items = [ws.import_and_load(arg, with_sidematter=True) for arg in input_args]
|
|
53
54
|
|
|
54
55
|
# URLs should have metadata like a title and be valid, so we fetch them.
|
|
55
56
|
if input_items:
|
|
@@ -109,9 +110,7 @@ def log_action(action: Action, action_input: ActionInput, operation: Operation):
|
|
|
109
110
|
log.info("Input items are:\n%s", fmt_lines(action_input.items))
|
|
110
111
|
|
|
111
112
|
|
|
112
|
-
def check_for_existing_result(
|
|
113
|
-
context: ExecContext, action_input: ActionInput, operation: Operation
|
|
114
|
-
) -> ActionResult | None:
|
|
113
|
+
def check_for_existing_result(context: ActionContext) -> ActionResult | None:
|
|
115
114
|
"""
|
|
116
115
|
Check if we already have the results for this operation (same action and inputs)
|
|
117
116
|
If so return it, unless rerun is requested, in which case we just log that the results
|
|
@@ -126,7 +125,7 @@ def check_for_existing_result(
|
|
|
126
125
|
|
|
127
126
|
# Check if a previous run already produced the result.
|
|
128
127
|
# To do this we preassemble outputs.
|
|
129
|
-
preassembled_result = action.
|
|
128
|
+
preassembled_result = action.preassemble_result(context)
|
|
130
129
|
if preassembled_result:
|
|
131
130
|
# Check if these items already exist, with last_operation matching action and input fingerprints.
|
|
132
131
|
already_present = [ws.find_by_id(item) for item in preassembled_result.items]
|
|
@@ -155,7 +154,7 @@ def check_for_existing_result(
|
|
|
155
154
|
|
|
156
155
|
|
|
157
156
|
def run_action_operation(
|
|
158
|
-
context:
|
|
157
|
+
context: ActionContext,
|
|
159
158
|
action_input: ActionInput,
|
|
160
159
|
operation: Operation,
|
|
161
160
|
) -> ActionResult:
|
|
@@ -205,7 +204,7 @@ class SkipItem(Exception):
|
|
|
205
204
|
"""
|
|
206
205
|
|
|
207
206
|
|
|
208
|
-
def _run_for_each_item(context:
|
|
207
|
+
def _run_for_each_item(context: ActionContext, input: ActionInput) -> ActionResult:
|
|
209
208
|
"""
|
|
210
209
|
Helper to process each input item. If non-fatal errors are encountered on any item,
|
|
211
210
|
they are reported and processing continues with the next item.
|
|
@@ -341,7 +340,7 @@ def save_action_result(
|
|
|
341
340
|
|
|
342
341
|
|
|
343
342
|
def run_action_with_caching(
|
|
344
|
-
|
|
343
|
+
exec_context: ExecContext, action_input: ActionInput
|
|
345
344
|
) -> tuple[ActionResult, list[StorePath], list[StorePath]]:
|
|
346
345
|
"""
|
|
347
346
|
Run an action, including validation, only rerunning if `rerun` requested or
|
|
@@ -350,57 +349,64 @@ def run_action_with_caching(
|
|
|
350
349
|
|
|
351
350
|
Note: Mutates the input but only to add `context` to each item.
|
|
352
351
|
"""
|
|
353
|
-
action =
|
|
354
|
-
settings =
|
|
352
|
+
action = exec_context.action
|
|
353
|
+
settings = exec_context.settings
|
|
355
354
|
ws = settings.workspace
|
|
356
355
|
|
|
357
|
-
# For convenience, we include the context to each item too (this helps so per-item
|
|
358
|
-
# functions don't have to take context args everywhere).
|
|
359
|
-
for item in action_input.items:
|
|
360
|
-
item.context = context
|
|
361
|
-
|
|
362
356
|
# Assemble the operation and validate the action input.
|
|
363
|
-
operation = validate_action_input(
|
|
357
|
+
operation = validate_action_input(exec_context, ws, action, action_input)
|
|
364
358
|
|
|
365
359
|
# Log what we're about to run.
|
|
366
360
|
log_action(action, action_input, operation)
|
|
367
361
|
|
|
368
|
-
#
|
|
369
|
-
|
|
362
|
+
# Consolidate all the context.
|
|
363
|
+
context = ActionContext(
|
|
364
|
+
exec_context=exec_context, operation=operation, action_input=action_input
|
|
365
|
+
)
|
|
370
366
|
|
|
371
|
-
|
|
372
|
-
#
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
archived_store_paths = []
|
|
367
|
+
try:
|
|
368
|
+
# Hack to add the context to each item.
|
|
369
|
+
# We do this before cache preassemble check.
|
|
370
|
+
action_input.set_context(context)
|
|
376
371
|
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
"%s Skipped: `%s` completed with %s %s",
|
|
380
|
-
EMOJI_SKIP,
|
|
381
|
-
action.name,
|
|
382
|
-
len(result.items),
|
|
383
|
-
plural("item", len(result.items)),
|
|
384
|
-
)
|
|
385
|
-
else:
|
|
386
|
-
# Run it!
|
|
387
|
-
result = run_action_operation(context, action_input, operation)
|
|
388
|
-
result_store_paths, archived_store_paths = save_action_result(
|
|
389
|
-
ws,
|
|
390
|
-
result,
|
|
391
|
-
action_input,
|
|
392
|
-
as_tmp=settings.tmp_output,
|
|
393
|
-
no_format=settings.no_format,
|
|
394
|
-
)
|
|
372
|
+
# Check if a previous run already produced the result.
|
|
373
|
+
existing_result = check_for_existing_result(context)
|
|
395
374
|
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
375
|
+
if existing_result and not settings.rerun:
|
|
376
|
+
# Use the cached result.
|
|
377
|
+
result = existing_result
|
|
378
|
+
result_store_paths = [StorePath(not_none(item.store_path)) for item in result.items]
|
|
379
|
+
archived_store_paths = []
|
|
380
|
+
|
|
381
|
+
PrintHooks.before_done_message()
|
|
382
|
+
log.message(
|
|
383
|
+
"%s Skipped: `%s` completed with %s %s",
|
|
384
|
+
EMOJI_SKIP,
|
|
385
|
+
action.name,
|
|
386
|
+
len(result.items),
|
|
387
|
+
plural("item", len(result.items)),
|
|
388
|
+
)
|
|
389
|
+
else:
|
|
390
|
+
# Run it!
|
|
391
|
+
result = run_action_operation(context, action_input, operation)
|
|
392
|
+
result_store_paths, archived_store_paths = save_action_result(
|
|
393
|
+
ws,
|
|
394
|
+
result,
|
|
395
|
+
action_input,
|
|
396
|
+
as_tmp=settings.tmp_output,
|
|
397
|
+
no_format=settings.no_format,
|
|
398
|
+
)
|
|
399
|
+
|
|
400
|
+
PrintHooks.before_done_message()
|
|
401
|
+
log.message(
|
|
402
|
+
"%s Action: `%s` completed with %s %s",
|
|
403
|
+
EMOJI_SUCCESS,
|
|
404
|
+
action.name,
|
|
405
|
+
len(result.items),
|
|
406
|
+
plural("item", len(result.items)),
|
|
407
|
+
)
|
|
408
|
+
finally:
|
|
409
|
+
action_input.clear_context()
|
|
404
410
|
|
|
405
411
|
return result, result_store_paths, archived_store_paths
|
|
406
412
|
|
kash/exec/fetch_url_items.py
CHANGED
|
@@ -2,7 +2,7 @@ from dataclasses import dataclass
|
|
|
2
2
|
|
|
3
3
|
from kash.config.logger import get_logger
|
|
4
4
|
from kash.exec.preconditions import is_url_resource
|
|
5
|
-
from kash.model.items_model import Item, ItemType
|
|
5
|
+
from kash.model.items_model import Format, Item, ItemType
|
|
6
6
|
from kash.model.paths_model import StorePath
|
|
7
7
|
from kash.utils.common.format_utils import fmt_loc
|
|
8
8
|
from kash.utils.common.url import Url, is_url
|
|
@@ -36,7 +36,15 @@ def fetch_url_item(
|
|
|
36
36
|
save_content: bool = True,
|
|
37
37
|
refetch: bool = False,
|
|
38
38
|
cache: bool = True,
|
|
39
|
+
overwrite: bool = True,
|
|
39
40
|
) -> FetchItemResult:
|
|
41
|
+
"""
|
|
42
|
+
Fetch or load an URL or path. For a URL, will fetch the content and metadata and save
|
|
43
|
+
as an item in the workspace.
|
|
44
|
+
|
|
45
|
+
Returns:
|
|
46
|
+
The fetched or loaded item, already saved to the workspace.
|
|
47
|
+
"""
|
|
40
48
|
from kash.workspaces import current_ws
|
|
41
49
|
|
|
42
50
|
ws = current_ws()
|
|
@@ -51,11 +59,22 @@ def fetch_url_item(
|
|
|
51
59
|
else:
|
|
52
60
|
raise InvalidInput(f"Not a URL or URL resource: {fmt_loc(locator)}")
|
|
53
61
|
|
|
54
|
-
return fetch_url_item_content(
|
|
62
|
+
return fetch_url_item_content(
|
|
63
|
+
item,
|
|
64
|
+
save_content=save_content,
|
|
65
|
+
refetch=refetch,
|
|
66
|
+
cache=cache,
|
|
67
|
+
overwrite=overwrite,
|
|
68
|
+
)
|
|
55
69
|
|
|
56
70
|
|
|
57
71
|
def fetch_url_item_content(
|
|
58
|
-
item: Item,
|
|
72
|
+
item: Item,
|
|
73
|
+
*,
|
|
74
|
+
save_content: bool = True,
|
|
75
|
+
refetch: bool = False,
|
|
76
|
+
cache: bool = True,
|
|
77
|
+
overwrite: bool = True,
|
|
59
78
|
) -> FetchItemResult:
|
|
60
79
|
"""
|
|
61
80
|
Fetch content and metadata for a URL using a media service if we
|
|
@@ -67,8 +86,11 @@ def fetch_url_item_content(
|
|
|
67
86
|
|
|
68
87
|
If `cache` is true, the content is also cached in the local file cache.
|
|
69
88
|
|
|
70
|
-
|
|
71
|
-
|
|
89
|
+
If `overwrite` is true, the item is saved at the same location every time.
|
|
90
|
+
This is useful to keep resource filenames consistent.
|
|
91
|
+
|
|
92
|
+
Returns:
|
|
93
|
+
The fetched or loaded item, already saved to the workspace.
|
|
72
94
|
"""
|
|
73
95
|
from kash.media_base.media_services import get_media_metadata
|
|
74
96
|
from kash.web_content.canon_url import canonicalize_url
|
|
@@ -109,7 +131,10 @@ def fetch_url_item_content(
|
|
|
109
131
|
url_item = item.merged_copy(url_item)
|
|
110
132
|
else:
|
|
111
133
|
page_data = fetch_page_content(url, refetch=refetch, cache=cache)
|
|
112
|
-
url_item =
|
|
134
|
+
url_item = Item(
|
|
135
|
+
type=ItemType.resource,
|
|
136
|
+
format=Format.url,
|
|
137
|
+
url=url,
|
|
113
138
|
title=page_data.title or item.title,
|
|
114
139
|
description=page_data.description or item.description,
|
|
115
140
|
thumbnail_url=page_data.thumbnail_url or item.thumbnail_url,
|
|
@@ -128,10 +153,10 @@ def fetch_url_item_content(
|
|
|
128
153
|
log.warning("Failed to fetch page data: title is missing: %s", item.url)
|
|
129
154
|
|
|
130
155
|
# Now save the updated URL item and also the content item if we have one.
|
|
131
|
-
ws.save(url_item)
|
|
156
|
+
ws.save(url_item, overwrite=overwrite)
|
|
132
157
|
assert url_item.store_path
|
|
133
158
|
if content_item:
|
|
134
|
-
ws.save(content_item)
|
|
159
|
+
ws.save(content_item, overwrite=overwrite)
|
|
135
160
|
assert content_item.store_path
|
|
136
161
|
log.info(
|
|
137
162
|
"Saved both URL and content item: %s, %s",
|
|
@@ -144,4 +169,6 @@ def fetch_url_item_content(
|
|
|
144
169
|
was_cached = bool(
|
|
145
170
|
not page_data or (page_data.cache_result and page_data.cache_result.was_cached)
|
|
146
171
|
)
|
|
147
|
-
return FetchItemResult(
|
|
172
|
+
return FetchItemResult(
|
|
173
|
+
item=content_item or url_item, was_cached=was_cached, page_data=page_data
|
|
174
|
+
)
|
kash/exec/preconditions.py
CHANGED
|
@@ -69,7 +69,7 @@ def is_instructions(item: Item) -> bool:
|
|
|
69
69
|
|
|
70
70
|
@kash_precondition
|
|
71
71
|
def is_url_resource(item: Item) -> bool:
|
|
72
|
-
return bool(item.type == ItemType.resource and item.url)
|
|
72
|
+
return bool(item.type == ItemType.resource and item.format == Format.url and item.url)
|
|
73
73
|
|
|
74
74
|
|
|
75
75
|
@kash_precondition
|
|
@@ -126,7 +126,7 @@ def has_markdown_with_html_body(item: Item) -> bool:
|
|
|
126
126
|
|
|
127
127
|
@kash_precondition
|
|
128
128
|
def has_fullpage_html_body(item: Item) -> bool:
|
|
129
|
-
return bool(
|
|
129
|
+
return bool(has_html_compatible_body(item) and item.body and is_fullpage_html(item.body))
|
|
130
130
|
|
|
131
131
|
|
|
132
132
|
@kash_precondition
|
kash/exec/resolve_args.py
CHANGED
|
@@ -118,10 +118,13 @@ def import_locator_args(
|
|
|
118
118
|
*locators_or_strs: UnresolvedLocator,
|
|
119
119
|
as_type: ItemType = ItemType.resource,
|
|
120
120
|
reimport: bool = False,
|
|
121
|
+
with_sidematter: bool = False,
|
|
121
122
|
) -> list[StorePath]:
|
|
122
123
|
"""
|
|
123
124
|
Import locators into the current workspace.
|
|
124
125
|
"""
|
|
125
126
|
locators = [resolve_locator_arg(loc) for loc in locators_or_strs]
|
|
126
127
|
ws = current_ws()
|
|
127
|
-
return ws.import_items(
|
|
128
|
+
return ws.import_items(
|
|
129
|
+
*locators, as_type=as_type, reimport=reimport, with_sidematter=with_sidematter
|
|
130
|
+
)
|
kash/exec/runtime_settings.py
CHANGED
kash/file_storage/file_store.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import functools
|
|
2
2
|
import os
|
|
3
|
+
import shutil
|
|
3
4
|
import threading
|
|
4
5
|
import time
|
|
5
6
|
from collections.abc import Callable, Generator
|
|
@@ -9,12 +10,12 @@ from typing import Concatenate, ParamSpec, TypeVar
|
|
|
9
10
|
|
|
10
11
|
from funlog import format_duration, log_calls
|
|
11
12
|
from prettyfmt import fmt_lines, fmt_path
|
|
12
|
-
from
|
|
13
|
+
from sidematter_format import copy_sidematter, move_sidematter, remove_sidematter
|
|
14
|
+
from strif import copyfile_atomic, hash_file
|
|
13
15
|
from typing_extensions import override
|
|
14
16
|
|
|
15
17
|
from kash.config.logger import get_log_settings, get_logger
|
|
16
18
|
from kash.config.text_styles import EMOJI_SAVED
|
|
17
|
-
from kash.file_storage.item_file_format import read_item, write_item
|
|
18
19
|
from kash.file_storage.metadata_dirs import MetadataDirs
|
|
19
20
|
from kash.file_storage.store_filenames import (
|
|
20
21
|
folder_for_type,
|
|
@@ -83,11 +84,6 @@ class FileStore(Workspace):
|
|
|
83
84
|
def base_dir(self) -> Path:
|
|
84
85
|
return self.base_dir_path
|
|
85
86
|
|
|
86
|
-
@property
|
|
87
|
-
@override
|
|
88
|
-
def assets_dir(self) -> Path:
|
|
89
|
-
return self.base_dir / "assets"
|
|
90
|
-
|
|
91
87
|
@synchronized
|
|
92
88
|
@log_calls(level="warning", if_slower_than=2.0)
|
|
93
89
|
def reload(self, auto_init: bool = True):
|
|
@@ -345,16 +341,23 @@ class FileStore(Workspace):
|
|
|
345
341
|
|
|
346
342
|
return StorePath(store_path), old_store_path
|
|
347
343
|
|
|
348
|
-
|
|
344
|
+
@synchronized
|
|
345
|
+
def assign_store_path(self, item: Item) -> Path:
|
|
349
346
|
"""
|
|
350
|
-
|
|
351
|
-
usual save mechanism and write directly to the store yourself, at the location
|
|
352
|
-
the item usually would be saved.
|
|
347
|
+
Pick a new store path for the item and mutate `item.store_path`.
|
|
353
348
|
|
|
354
|
-
|
|
355
|
-
|
|
349
|
+
This is useful if you need to write to the store yourself, at the location
|
|
350
|
+
the item usually would be saved, and also want the path to be fixed.
|
|
351
|
+
|
|
352
|
+
This is idempotent. If you also write to the file, call `mark_as_saved()`
|
|
353
|
+
to indicate that the file is now saved. Otherwise the item should be saved
|
|
354
|
+
with `save()`.
|
|
355
|
+
|
|
356
|
+
Returns the absolute path, for convenience if you wish to write to the file
|
|
357
|
+
directly.
|
|
356
358
|
"""
|
|
357
359
|
store_path, _old_store_path = self.store_path_for(item)
|
|
360
|
+
item.store_path = str(store_path)
|
|
358
361
|
return self.base_dir / store_path
|
|
359
362
|
|
|
360
363
|
def _tmp_path_for(self, item: Item) -> StorePath:
|
|
@@ -455,6 +458,8 @@ class FileStore(Workspace):
|
|
|
455
458
|
# Save as a text item with frontmatter.
|
|
456
459
|
if item.external_path:
|
|
457
460
|
item.body = Path(item.external_path).read_text()
|
|
461
|
+
from kash.file_storage.item_file_format import write_item
|
|
462
|
+
|
|
458
463
|
write_item(item, full_path, normalize=not no_format)
|
|
459
464
|
except OSError as e:
|
|
460
465
|
log.error("Error saving item: %s", e)
|
|
@@ -499,6 +504,8 @@ class FileStore(Workspace):
|
|
|
499
504
|
"""
|
|
500
505
|
Load item at the given path.
|
|
501
506
|
"""
|
|
507
|
+
from kash.file_storage.item_file_format import read_item
|
|
508
|
+
|
|
502
509
|
return read_item(self.base_dir / store_path, self.base_dir)
|
|
503
510
|
|
|
504
511
|
def hash(self, store_path: StorePath) -> str:
|
|
@@ -513,6 +520,7 @@ class FileStore(Workspace):
|
|
|
513
520
|
*,
|
|
514
521
|
as_type: ItemType | None = None,
|
|
515
522
|
reimport: bool = False,
|
|
523
|
+
with_sidematter: bool = False,
|
|
516
524
|
) -> StorePath:
|
|
517
525
|
"""
|
|
518
526
|
Add resources from files or URLs. If a locator is a path, copy it into the store.
|
|
@@ -520,7 +528,10 @@ class FileStore(Workspace):
|
|
|
520
528
|
are not imported again and the existing store path is returned.
|
|
521
529
|
If `as_type` is specified, it will be used to override the item type, otherwise
|
|
522
530
|
we go with our best guess.
|
|
531
|
+
If `with_sidematter` is true, will copy any sidematter files (metadata/assets) to
|
|
532
|
+
the destination.
|
|
523
533
|
"""
|
|
534
|
+
from kash.file_storage.item_file_format import read_item
|
|
524
535
|
from kash.web_content.canon_url import canonicalize_url
|
|
525
536
|
|
|
526
537
|
if isinstance(locator, StorePath) and not reimport:
|
|
@@ -580,6 +591,9 @@ class FileStore(Workspace):
|
|
|
580
591
|
# we'll pick a new store path.
|
|
581
592
|
store_path = self.save(item)
|
|
582
593
|
log.info("Imported text file: %s", item.as_str())
|
|
594
|
+
# If requested, also copy any sidematter files (metadata/assets) to match destination.
|
|
595
|
+
if with_sidematter:
|
|
596
|
+
copy_sidematter(path, self.base_dir / store_path, copy_original=False)
|
|
583
597
|
else:
|
|
584
598
|
# Binary or other files we just copy over as-is, preserving the name.
|
|
585
599
|
# We know the extension is recognized.
|
|
@@ -588,7 +602,10 @@ class FileStore(Workspace):
|
|
|
588
602
|
raise FileExists(f"Resource already in store: {fmt_loc(store_path)}")
|
|
589
603
|
|
|
590
604
|
log.message("Importing resource: %s", fmt_loc(path))
|
|
591
|
-
|
|
605
|
+
if with_sidematter:
|
|
606
|
+
copy_sidematter(path, self.base_dir / store_path)
|
|
607
|
+
else:
|
|
608
|
+
copyfile_atomic(path, self.base_dir / store_path, make_parents=True)
|
|
592
609
|
|
|
593
610
|
# Optimization: Don't import an identical file twice.
|
|
594
611
|
if old_store_path:
|
|
@@ -599,7 +616,10 @@ class FileStore(Workspace):
|
|
|
599
616
|
"Imported resource is identical to the previous import: %s",
|
|
600
617
|
fmt_loc(old_store_path),
|
|
601
618
|
)
|
|
602
|
-
|
|
619
|
+
if with_sidematter:
|
|
620
|
+
remove_sidematter(self.base_dir / store_path)
|
|
621
|
+
else:
|
|
622
|
+
os.unlink(self.base_dir / store_path)
|
|
603
623
|
store_path = old_store_path
|
|
604
624
|
log.message("Imported resource: %s", fmt_loc(store_path))
|
|
605
625
|
return store_path
|
|
@@ -609,16 +629,20 @@ class FileStore(Workspace):
|
|
|
609
629
|
*locators: Locator,
|
|
610
630
|
as_type: ItemType | None = None,
|
|
611
631
|
reimport: bool = False,
|
|
632
|
+
with_sidematter: bool = False,
|
|
612
633
|
) -> list[StorePath]:
|
|
613
634
|
return [
|
|
614
|
-
self.import_item(
|
|
635
|
+
self.import_item(
|
|
636
|
+
locator, as_type=as_type, reimport=reimport, with_sidematter=with_sidematter
|
|
637
|
+
)
|
|
638
|
+
for locator in locators
|
|
615
639
|
]
|
|
616
640
|
|
|
617
|
-
def import_and_load(self, locator: UnresolvedLocator) -> Item:
|
|
641
|
+
def import_and_load(self, locator: UnresolvedLocator, with_sidematter: bool = False) -> Item:
|
|
618
642
|
"""
|
|
619
643
|
Import a locator and return the item.
|
|
620
644
|
"""
|
|
621
|
-
store_path = self.import_item(locator)
|
|
645
|
+
store_path = self.import_item(locator, with_sidematter=with_sidematter)
|
|
622
646
|
return self.load(store_path)
|
|
623
647
|
|
|
624
648
|
def _filter_selection_paths(self):
|
|
@@ -660,7 +684,12 @@ class FileStore(Workspace):
|
|
|
660
684
|
# TODO: Update metadata of all relations that point to this path too.
|
|
661
685
|
|
|
662
686
|
def archive(
|
|
663
|
-
self,
|
|
687
|
+
self,
|
|
688
|
+
store_path: StorePath,
|
|
689
|
+
*,
|
|
690
|
+
missing_ok: bool = False,
|
|
691
|
+
quiet: bool = False,
|
|
692
|
+
with_sidematter: bool = False,
|
|
664
693
|
) -> StorePath:
|
|
665
694
|
"""
|
|
666
695
|
Archive the item by moving it into the archive directory.
|
|
@@ -672,20 +701,24 @@ class FileStore(Workspace):
|
|
|
672
701
|
fmt_loc(self.dirs.archive_dir),
|
|
673
702
|
)
|
|
674
703
|
orig_path = self.base_dir / store_path
|
|
675
|
-
|
|
704
|
+
full_archive_path = self.base_dir / self.dirs.archive_dir / store_path
|
|
676
705
|
if missing_ok and not orig_path.exists():
|
|
677
706
|
log.message("Item to archive not found so moving on: %s", fmt_loc(orig_path))
|
|
678
707
|
return store_path
|
|
679
708
|
if not orig_path.exists():
|
|
680
709
|
log.warning("Item to archive not found: %s", fmt_loc(orig_path))
|
|
681
710
|
return store_path
|
|
682
|
-
|
|
711
|
+
if with_sidematter:
|
|
712
|
+
move_sidematter(orig_path, full_archive_path)
|
|
713
|
+
else:
|
|
714
|
+
os.makedirs(full_archive_path.parent, exist_ok=True)
|
|
715
|
+
shutil.move(orig_path, full_archive_path)
|
|
683
716
|
self._remove_references([store_path])
|
|
684
717
|
|
|
685
718
|
archive_path = StorePath(self.dirs.archive_dir / store_path)
|
|
686
719
|
return archive_path
|
|
687
720
|
|
|
688
|
-
def unarchive(self, store_path: StorePath) -> StorePath:
|
|
721
|
+
def unarchive(self, store_path: StorePath, with_sidematter: bool = False) -> StorePath:
|
|
689
722
|
"""
|
|
690
723
|
Unarchive the item by moving back out of the archive directory.
|
|
691
724
|
Path may be with or without the archive dir prefix.
|
|
@@ -695,7 +728,10 @@ class FileStore(Workspace):
|
|
|
695
728
|
if full_input_path.is_relative_to(full_archive_path):
|
|
696
729
|
store_path = StorePath(relpath(full_input_path, full_archive_path))
|
|
697
730
|
original_path = self.base_dir / store_path
|
|
698
|
-
|
|
731
|
+
if with_sidematter:
|
|
732
|
+
move_sidematter(full_input_path, original_path)
|
|
733
|
+
else:
|
|
734
|
+
shutil.move(full_input_path, original_path)
|
|
699
735
|
return StorePath(store_path)
|
|
700
736
|
|
|
701
737
|
@synchronized
|