fss-cli 0.11.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fss_cli/__init__.py +24 -0
- fss_cli/_accounts.py +29 -0
- fss_cli/_app.py +663 -0
- fss_cli/_cat.py +391 -0
- fss_cli/_command.py +368 -0
- fss_cli/_concurrent.py +181 -0
- fss_cli/_content.py +94 -0
- fss_cli/_copy_file.py +65 -0
- fss_cli/_cp.py +740 -0
- fss_cli/_diagnostics.py +12 -0
- fss_cli/_du.py +90 -0
- fss_cli/_find.py +109 -0
- fss_cli/_head_tail.py +128 -0
- fss_cli/_info.py +179 -0
- fss_cli/_listing.py +289 -0
- fss_cli/_logging.py +72 -0
- fss_cli/_ls.py +435 -0
- fss_cli/_manifest.py +264 -0
- fss_cli/_metadata.py +70 -0
- fss_cli/_mkdir.py +97 -0
- fss_cli/_mv.py +146 -0
- fss_cli/_path.py +62 -0
- fss_cli/_post_copy.py +191 -0
- fss_cli/_progress.py +132 -0
- fss_cli/_recursive_cp.py +908 -0
- fss_cli/_recursive_rm.py +335 -0
- fss_cli/_results.py +115 -0
- fss_cli/_rm.py +149 -0
- fss_cli/_rmdir.py +148 -0
- fss_cli/_size.py +95 -0
- fss_cli/_sources.py +178 -0
- fss_cli/_stat.py +196 -0
- fss_cli/_test.py +59 -0
- fss_cli/_tree.py +208 -0
- fss_cli/_unlink.py +90 -0
- fss_cli/_walk.py +141 -0
- fss_cli-0.11.0.dist-info/METADATA +144 -0
- fss_cli-0.11.0.dist-info/RECORD +40 -0
- fss_cli-0.11.0.dist-info/WHEEL +4 -0
- fss_cli-0.11.0.dist-info/licenses/LICENSE +661 -0
fss_cli/_tree.py
ADDED
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
"""Walk validation and Unicode rendering for the central ``tree`` callback."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import TYPE_CHECKING, TypeGuard
|
|
7
|
+
|
|
8
|
+
from ._command import (
|
|
9
|
+
_collate,
|
|
10
|
+
_Failure,
|
|
11
|
+
_MappedOperand,
|
|
12
|
+
_run_single_operand_text,
|
|
13
|
+
)
|
|
14
|
+
from ._metadata import valid_display_text
|
|
15
|
+
from ._path import _lexical_join, _lexical_root
|
|
16
|
+
from ._walk import _IncompatibleListingError, _walk
|
|
17
|
+
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
from collections.abc import Mapping, Sequence
|
|
20
|
+
|
|
21
|
+
from fsspec.asyn import AsyncFileSystem
|
|
22
|
+
|
|
23
|
+
from ._app import AsyncFilesystemSource
|
|
24
|
+
|
|
25
|
+
_WALK_ROW_LENGTH = 3
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class _TreeRequest:
|
|
30
|
+
maxdepth: int | None
|
|
31
|
+
operand: _MappedOperand
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True)
|
|
35
|
+
class _WalkRow:
|
|
36
|
+
root: str
|
|
37
|
+
directories: tuple[str, ...]
|
|
38
|
+
files: tuple[str, ...]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _valid_root(value: object) -> TypeGuard[str]:
|
|
42
|
+
return type(value) is str and valid_display_text(value)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _valid_entry(value: object, *, root_file: bool = False) -> TypeGuard[str]:
|
|
46
|
+
return type(value) is str and (bool(value) or root_file) and "/" not in value and valid_display_text(value)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _entries(
|
|
50
|
+
values: Sequence[object],
|
|
51
|
+
*,
|
|
52
|
+
root_file: bool = False,
|
|
53
|
+
) -> tuple[str, ...] | None:
|
|
54
|
+
entries: list[str] = []
|
|
55
|
+
for value in values:
|
|
56
|
+
if not _valid_entry(value, root_file=root_file):
|
|
57
|
+
return None
|
|
58
|
+
entries.append(value)
|
|
59
|
+
if len(set(entries)) != len(entries):
|
|
60
|
+
return None
|
|
61
|
+
return tuple(entries)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _row(value: object) -> _WalkRow | None:
|
|
65
|
+
if type(value) is not tuple or len(value) != _WALK_ROW_LENGTH:
|
|
66
|
+
return None
|
|
67
|
+
root, directories, files = value
|
|
68
|
+
if not (_valid_root(root) and type(directories) is list and type(files) is list):
|
|
69
|
+
return None
|
|
70
|
+
typed_directories = _entries(directories)
|
|
71
|
+
typed_files = _entries(files, root_file=True)
|
|
72
|
+
if typed_directories is None or typed_files is None:
|
|
73
|
+
return None
|
|
74
|
+
if set(typed_directories).intersection(typed_files):
|
|
75
|
+
return None
|
|
76
|
+
return _WalkRow(root, typed_directories, typed_files)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _index_rows(values: list[object]) -> tuple[_WalkRow, dict[str, _WalkRow]] | None:
|
|
80
|
+
indexed: dict[str, _WalkRow] = {}
|
|
81
|
+
first = None
|
|
82
|
+
for value in values:
|
|
83
|
+
row = _row(value)
|
|
84
|
+
if row is None:
|
|
85
|
+
return None
|
|
86
|
+
if first is None:
|
|
87
|
+
first = row
|
|
88
|
+
canonical = _lexical_root(row.root)
|
|
89
|
+
if canonical in indexed:
|
|
90
|
+
return None
|
|
91
|
+
indexed[canonical] = row
|
|
92
|
+
if first is None:
|
|
93
|
+
return None
|
|
94
|
+
return first, indexed
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _valid_root_file(rows: Mapping[str, _WalkRow], root: _WalkRow) -> bool:
|
|
98
|
+
rows_with_empty_file = [row for row in rows.values() if "" in row.files]
|
|
99
|
+
if not rows_with_empty_file:
|
|
100
|
+
return True
|
|
101
|
+
return rows_with_empty_file == [root] and root.directories == () and root.files == ("",) and len(rows) == 1
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _all_rows_reachable(rows: Mapping[str, _WalkRow], root: _WalkRow) -> bool:
|
|
105
|
+
reachable = {_lexical_root(root.root)}
|
|
106
|
+
pending = [root]
|
|
107
|
+
while pending:
|
|
108
|
+
parent = pending.pop()
|
|
109
|
+
for directory in parent.directories:
|
|
110
|
+
child_key = _lexical_root(_lexical_join(parent.root, directory))
|
|
111
|
+
child = rows.get(child_key)
|
|
112
|
+
if child is not None and child_key not in reachable:
|
|
113
|
+
reachable.add(child_key)
|
|
114
|
+
pending.append(child)
|
|
115
|
+
return reachable == rows.keys()
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _validated_rows(
|
|
119
|
+
request: _TreeRequest,
|
|
120
|
+
values: list[object],
|
|
121
|
+
) -> Mapping[str, _WalkRow] | None:
|
|
122
|
+
indexed = _index_rows(values)
|
|
123
|
+
if indexed is None:
|
|
124
|
+
return None
|
|
125
|
+
root, rows = indexed
|
|
126
|
+
|
|
127
|
+
root_key = _lexical_root(root.root)
|
|
128
|
+
if root_key != _lexical_root(request.operand.path):
|
|
129
|
+
return None
|
|
130
|
+
if not _valid_root_file(rows, root):
|
|
131
|
+
return None
|
|
132
|
+
if not _all_rows_reachable(rows, root):
|
|
133
|
+
return None
|
|
134
|
+
return rows
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _sorted_entries(entries: tuple[str, ...]) -> list[str]:
|
|
138
|
+
return sorted(entries, key=_collate)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _render_tree(request: _TreeRequest, rows: Mapping[str, _WalkRow]) -> str:
|
|
142
|
+
lines = [request.operand.path]
|
|
143
|
+
root = rows[_lexical_root(request.operand.path)]
|
|
144
|
+
if root.files == ("",):
|
|
145
|
+
return f"{request.operand.path}\n"
|
|
146
|
+
|
|
147
|
+
stack: list[tuple[_WalkRow, int, str, str, bool, bool]] = []
|
|
148
|
+
|
|
149
|
+
def push_children(row: _WalkRow, depth: int, prefix: str) -> None:
|
|
150
|
+
if request.maxdepth is not None and depth >= request.maxdepth:
|
|
151
|
+
return
|
|
152
|
+
directories = _sorted_entries(row.directories)
|
|
153
|
+
files = _sorted_entries(row.files)
|
|
154
|
+
children = [(name, True) for name in directories]
|
|
155
|
+
children.extend((name, False) for name in files)
|
|
156
|
+
for index in range(len(children) - 1, -1, -1):
|
|
157
|
+
name, is_directory = children[index]
|
|
158
|
+
is_last = index == len(children) - 1
|
|
159
|
+
stack.append((row, depth, prefix, name, is_directory, is_last))
|
|
160
|
+
|
|
161
|
+
push_children(root, 0, "")
|
|
162
|
+
while stack:
|
|
163
|
+
parent, depth, prefix, name, is_directory, is_last = stack.pop()
|
|
164
|
+
connector = "└── " if is_last else "├── "
|
|
165
|
+
lines.append(f"{prefix}{connector}{name}")
|
|
166
|
+
if not is_directory:
|
|
167
|
+
continue
|
|
168
|
+
child = rows.get(_lexical_root(_lexical_join(parent.root, name)))
|
|
169
|
+
if child is None:
|
|
170
|
+
continue
|
|
171
|
+
continuation = " " if is_last else "│ "
|
|
172
|
+
push_children(child, depth + 1, prefix + continuation)
|
|
173
|
+
|
|
174
|
+
return "\n".join(lines) + "\n"
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
async def _walk_tree(
|
|
178
|
+
request: _TreeRequest,
|
|
179
|
+
filesystem: AsyncFileSystem,
|
|
180
|
+
) -> str | _Failure:
|
|
181
|
+
try:
|
|
182
|
+
listed = await _walk(
|
|
183
|
+
filesystem,
|
|
184
|
+
request.operand.path,
|
|
185
|
+
maxdepth=1 if request.maxdepth == 0 else request.maxdepth,
|
|
186
|
+
)
|
|
187
|
+
except _IncompatibleListingError:
|
|
188
|
+
return _Failure(request.operand)
|
|
189
|
+
except Exception as error: # noqa: BLE001 - classify awaited backend failure.
|
|
190
|
+
return _Failure(request.operand, backend_error=error)
|
|
191
|
+
values: list[object] = [(row.root, list(row.directories), list(row.files)) for row in listed]
|
|
192
|
+
rows = _validated_rows(request, values)
|
|
193
|
+
if rows is None:
|
|
194
|
+
return _Failure(request.operand)
|
|
195
|
+
return _render_tree(request, rows)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
async def _run_tree(
|
|
199
|
+
command: str,
|
|
200
|
+
request: _TreeRequest,
|
|
201
|
+
sources: Mapping[str, AsyncFilesystemSource],
|
|
202
|
+
) -> None:
|
|
203
|
+
await _run_single_operand_text(
|
|
204
|
+
command,
|
|
205
|
+
request.operand,
|
|
206
|
+
sources,
|
|
207
|
+
lambda filesystem: _walk_tree(request, filesystem),
|
|
208
|
+
)
|
fss_cli/_unlink.py
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Typed async execution for ``unlink``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from typing import TYPE_CHECKING
|
|
8
|
+
|
|
9
|
+
from ._command import (
|
|
10
|
+
_drain_current_operation,
|
|
11
|
+
_Failure,
|
|
12
|
+
_MappedOperand,
|
|
13
|
+
_raise_operand_failures,
|
|
14
|
+
_render_backend_failure,
|
|
15
|
+
_render_operand_diagnostic,
|
|
16
|
+
_run_mapped_command,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
from fsspec.asyn import AsyncFileSystem
|
|
21
|
+
|
|
22
|
+
from ._app import AsyncFilesystemSource
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True)
|
|
26
|
+
class _UnlinkFailure(_Failure):
|
|
27
|
+
"""A failure raised on the single-file removal path.
|
|
28
|
+
|
|
29
|
+
Structurally identical to the base failure; the distinct type is how ``rm``
|
|
30
|
+
tells a file-path failure from a directory-path one, because the two render
|
|
31
|
+
different diagnostic vocabularies for the same condition.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
async def _confirmed_rm_file( # noqa: PLR0911 - explicit outcome branches.
|
|
36
|
+
operand: _MappedOperand,
|
|
37
|
+
filesystem: AsyncFileSystem,
|
|
38
|
+
) -> _UnlinkFailure | None:
|
|
39
|
+
"""Remove one source-reported file and confirm absence."""
|
|
40
|
+
try:
|
|
41
|
+
info = await _drain_current_operation(filesystem._info(operand.path))
|
|
42
|
+
except Exception as error: # noqa: BLE001 - classify awaited backend failure.
|
|
43
|
+
return _UnlinkFailure(operand, backend_error=error)
|
|
44
|
+
|
|
45
|
+
if not isinstance(info, Mapping) or not isinstance(info.get("type"), str):
|
|
46
|
+
return _UnlinkFailure(operand, incompatible="result")
|
|
47
|
+
|
|
48
|
+
result_type = info["type"]
|
|
49
|
+
if result_type == "directory":
|
|
50
|
+
return _UnlinkFailure(operand, incompatible="directory")
|
|
51
|
+
if result_type != "file":
|
|
52
|
+
return _UnlinkFailure(operand, incompatible="result")
|
|
53
|
+
|
|
54
|
+
try:
|
|
55
|
+
await _drain_current_operation(filesystem._rm_file(operand.path))
|
|
56
|
+
except Exception as error: # noqa: BLE001 - mutation may have partially applied.
|
|
57
|
+
return _UnlinkFailure(operand, backend_error=error, uncertain=True)
|
|
58
|
+
|
|
59
|
+
try:
|
|
60
|
+
await _drain_current_operation(filesystem._info(operand.path))
|
|
61
|
+
except FileNotFoundError:
|
|
62
|
+
return None
|
|
63
|
+
except Exception as error: # noqa: BLE001 - never hide non-not-found errors.
|
|
64
|
+
return _UnlinkFailure(operand, backend_error=error, uncertain=True)
|
|
65
|
+
else:
|
|
66
|
+
return _UnlinkFailure(operand, uncertain=True)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _render_failure(command: str, failure: _UnlinkFailure) -> None:
|
|
70
|
+
if failure.uncertain:
|
|
71
|
+
_render_operand_diagnostic(command, failure.operand, "uncertain mutation state")
|
|
72
|
+
elif failure.incompatible == "directory":
|
|
73
|
+
_render_operand_diagnostic(command, failure.operand, "is a directory")
|
|
74
|
+
elif failure.incompatible == "result" or failure.backend_error is None:
|
|
75
|
+
_render_operand_diagnostic(command, failure.operand, "incompatible result")
|
|
76
|
+
else:
|
|
77
|
+
_render_backend_failure(command, failure.operand, failure.backend_error)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
async def _run_unlink(
|
|
81
|
+
command: str,
|
|
82
|
+
operand: _MappedOperand,
|
|
83
|
+
sources: Mapping[str, AsyncFilesystemSource],
|
|
84
|
+
) -> None:
|
|
85
|
+
async def operation(filesystems: Mapping[str, AsyncFileSystem]) -> None:
|
|
86
|
+
failure = await _confirmed_rm_file(operand, filesystems[operand.name])
|
|
87
|
+
if failure is not None:
|
|
88
|
+
_raise_operand_failures(command, (failure,), _render_failure)
|
|
89
|
+
|
|
90
|
+
await _run_mapped_command(command, (operand,), sources, operation)
|
fss_cli/_walk.py
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""Breadth-first directory walks over bounded concurrent ``_ls`` listings.
|
|
2
|
+
|
|
3
|
+
fsspec's inherited ``_walk`` lists one directory at a time, and its nested
|
|
4
|
+
calls do not forward ``on_error``: with ``on_error="raise"`` an unreadable
|
|
5
|
+
*descendant* is silently treated as empty, so a walk can report success with
|
|
6
|
+
content missing. Commands that walk a tree therefore list it here instead:
|
|
7
|
+
every directory of one depth is listed concurrently under the shared bound,
|
|
8
|
+
and a listing failure at **any** depth raises.
|
|
9
|
+
|
|
10
|
+
Rows match fsspec's ``_walk(detail=True)`` format. Each row contains the listed
|
|
11
|
+
root and child directory and file metadata, keyed by basename. A file-like
|
|
12
|
+
entry for the root itself uses the key ``""``. This module checks the fields
|
|
13
|
+
needed to build a row. Commands validate the remaining fields under ADR 0006.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from dataclasses import dataclass
|
|
19
|
+
from functools import partial
|
|
20
|
+
from typing import TYPE_CHECKING, cast
|
|
21
|
+
|
|
22
|
+
from ._command import _call
|
|
23
|
+
from ._concurrent import _outcome_value, _run_bounded
|
|
24
|
+
from ._path import _lexical_join, _same_lexical_path
|
|
25
|
+
from ._results import _BadResultError, snapshot_listing
|
|
26
|
+
|
|
27
|
+
if TYPE_CHECKING:
|
|
28
|
+
from collections.abc import Callable, Mapping
|
|
29
|
+
|
|
30
|
+
from fsspec.asyn import AsyncFileSystem
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class _IncompatibleListingError(Exception):
|
|
34
|
+
"""A listing whose shape cannot form a walk row."""
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class _ListingLimitError(Exception):
|
|
38
|
+
"""A listing longer than the caller's remaining entry capacity."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class _ListedRow:
|
|
43
|
+
"""One listed directory, shaped like an fsspec ``_walk(detail=True)`` row."""
|
|
44
|
+
|
|
45
|
+
root: str
|
|
46
|
+
directories: Mapping[str, Mapping[object, object]]
|
|
47
|
+
files: Mapping[str, Mapping[object, object]]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _listed_row(
|
|
51
|
+
root: str,
|
|
52
|
+
listing: object,
|
|
53
|
+
capacity: int | None,
|
|
54
|
+
) -> tuple[_ListedRow, tuple[str, ...]]:
|
|
55
|
+
"""Freeze one ``_ls(detail=True)`` result into a row and its child paths.
|
|
56
|
+
|
|
57
|
+
The listing length is checked against ``capacity`` before any entry is
|
|
58
|
+
read, so an oversized listing is refused without materializing it.
|
|
59
|
+
"""
|
|
60
|
+
if not isinstance(listing, list):
|
|
61
|
+
raise _IncompatibleListingError
|
|
62
|
+
if capacity is not None and len(listing) > capacity:
|
|
63
|
+
raise _ListingLimitError
|
|
64
|
+
try:
|
|
65
|
+
indexed = snapshot_listing(root, listing)
|
|
66
|
+
except _BadResultError as error:
|
|
67
|
+
raise _IncompatibleListingError from error
|
|
68
|
+
directories: dict[str, Mapping[object, object]] = {}
|
|
69
|
+
files: dict[str, Mapping[object, object]] = {}
|
|
70
|
+
children: list[str] = []
|
|
71
|
+
for value in indexed.values():
|
|
72
|
+
info = cast("Mapping[object, object]", value)
|
|
73
|
+
pathname = cast("str", info["name"]).rstrip("/")
|
|
74
|
+
if _same_lexical_path(pathname, root):
|
|
75
|
+
collection, key = files, ""
|
|
76
|
+
else:
|
|
77
|
+
key = pathname.rsplit("/", 1)[-1]
|
|
78
|
+
collection = directories if info["type"] == "directory" else files
|
|
79
|
+
collection[key] = info
|
|
80
|
+
if collection is directories:
|
|
81
|
+
children.append(_lexical_join(root, key))
|
|
82
|
+
return _ListedRow(root, directories, files), tuple(children)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
async def _walk(
|
|
86
|
+
filesystem: AsyncFileSystem,
|
|
87
|
+
root: str,
|
|
88
|
+
*,
|
|
89
|
+
maxdepth: int | None = None,
|
|
90
|
+
accept: Callable[[_ListedRow], None] | None = None,
|
|
91
|
+
capacity: Callable[[], int] | None = None,
|
|
92
|
+
) -> list[_ListedRow]:
|
|
93
|
+
"""List ``root`` and its descendants breadth-first.
|
|
94
|
+
|
|
95
|
+
Every directory of one depth is listed concurrently under the shared
|
|
96
|
+
bound. Rows are frozen and passed to ``accept`` in breadth-first order,
|
|
97
|
+
before any directory of the next depth is listed, so a caller rejecting a
|
|
98
|
+
row stops the walk before it reads further.
|
|
99
|
+
|
|
100
|
+
Args:
|
|
101
|
+
filesystem: The async filesystem to list.
|
|
102
|
+
root: The directory (or file) to walk, passed to ``_ls`` literally.
|
|
103
|
+
maxdepth: Number of directory levels to list; ``None`` is unbounded.
|
|
104
|
+
accept: Optional validator for each row; raising stops the walk.
|
|
105
|
+
capacity: Optional remaining-entry budget checked before each listing
|
|
106
|
+
is materialized.
|
|
107
|
+
|
|
108
|
+
Returns:
|
|
109
|
+
Every listed row in breadth-first order.
|
|
110
|
+
|
|
111
|
+
Raises:
|
|
112
|
+
_IncompatibleListingError: A listing has an unusable shape.
|
|
113
|
+
_ListingLimitError: A listing exceeds the remaining capacity.
|
|
114
|
+
Exception: The first listing failure in breadth-first order.
|
|
115
|
+
"""
|
|
116
|
+
rows: list[_ListedRow] = []
|
|
117
|
+
level = [root]
|
|
118
|
+
depth = 1
|
|
119
|
+
while level:
|
|
120
|
+
outcomes = await _run_bounded(
|
|
121
|
+
[partial(_call, filesystem, "_ls", directory, detail=True) for directory in level]
|
|
122
|
+
)
|
|
123
|
+
next_level: list[str] = []
|
|
124
|
+
# Settle in breadth-first order, so an earlier malformed listing wins
|
|
125
|
+
# over a later sibling's listing failure.
|
|
126
|
+
for directory, outcome in zip(level, outcomes, strict=True):
|
|
127
|
+
listing = _outcome_value(outcome)
|
|
128
|
+
row, children = _listed_row(
|
|
129
|
+
directory,
|
|
130
|
+
listing,
|
|
131
|
+
None if capacity is None else capacity(),
|
|
132
|
+
)
|
|
133
|
+
if accept is not None:
|
|
134
|
+
accept(row)
|
|
135
|
+
rows.append(row)
|
|
136
|
+
next_level.extend(children)
|
|
137
|
+
if maxdepth is not None and depth >= maxdepth:
|
|
138
|
+
break
|
|
139
|
+
level = next_level
|
|
140
|
+
depth += 1
|
|
141
|
+
return rows
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: fss-cli
|
|
3
|
+
Version: 0.11.0
|
|
4
|
+
Summary: Library-only POSIX-shaped commands for async fsspec filesystems
|
|
5
|
+
Keywords: fsspec,cli,posix,typer,filesystem
|
|
6
|
+
Author: Shiny Brar (he/il)
|
|
7
|
+
Author-email: Shiny Brar (he/il) <charanjotbrar@gmail.com>
|
|
8
|
+
License-Expression: AGPL-3.0-or-later
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
14
|
+
Classifier: Operating System :: MacOS
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
21
|
+
Classifier: Topic :: System :: Filesystems
|
|
22
|
+
Classifier: Topic :: Utilities
|
|
23
|
+
Requires-Dist: fsspec>=2026.6.0
|
|
24
|
+
Requires-Dist: rich>=13.8.0
|
|
25
|
+
Requires-Dist: typer>=0.27.0
|
|
26
|
+
Requires-Python: >=3.10
|
|
27
|
+
Project-URL: Homepage, https://github.com/opencadc/fsspec-tools
|
|
28
|
+
Project-URL: Documentation, https://opencadc.github.io/fsspec-tools/
|
|
29
|
+
Project-URL: Repository, https://github.com/opencadc/fsspec-tools
|
|
30
|
+
Project-URL: Issues, https://github.com/opencadc/fsspec-tools/issues
|
|
31
|
+
Project-URL: Changelog, https://github.com/opencadc/fsspec-tools/blob/main/libs/fss-cli/CHANGELOG.md
|
|
32
|
+
Description-Content-Type: text/markdown
|
|
33
|
+
|
|
34
|
+
# fss-cli
|
|
35
|
+
|
|
36
|
+
[](https://github.com/opencadc/fsspec-tools/actions/workflows/ci.yml)
|
|
37
|
+
[](https://www.python.org/)
|
|
38
|
+
[](LICENSE)
|
|
39
|
+
|
|
40
|
+
`fss-cli` provides POSIX-shaped [Typer](https://typer.tiangolo.com/) commands
|
|
41
|
+
for async [`fsspec`](https://github.com/fsspec/filesystem_spec) filesystems.
|
|
42
|
+
Your application configures the filesystems and embeds the commands in its CLI.
|
|
43
|
+
The package installs no executable or module entry point.
|
|
44
|
+
|
|
45
|
+
Releases up to 0.10.0 were named `fsspec-cli`, imported as `fsspec_cli`. See
|
|
46
|
+
[ADR 0010](../../docs/adr/0010-rename-fsspec-cli-to-fss-cli.md).
|
|
47
|
+
|
|
48
|
+
## Install
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
uv add "git+https://github.com/opencadc/fsspec-tools@main#subdirectory=libs/fss-cli"
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Quick start
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
from contextlib import asynccontextmanager
|
|
58
|
+
|
|
59
|
+
import fsspec
|
|
60
|
+
import typer
|
|
61
|
+
from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper
|
|
62
|
+
from fss_cli import App
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@asynccontextmanager
|
|
66
|
+
async def data_source():
|
|
67
|
+
# One fresh async-capable filesystem per command invocation.
|
|
68
|
+
yield AsyncFileSystemWrapper(fsspec.filesystem("memory"))
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
app = typer.Typer()
|
|
72
|
+
app.add_typer(App({"data": data_source}).typer_app, name="fs")
|
|
73
|
+
|
|
74
|
+
if __name__ == "__main__":
|
|
75
|
+
app()
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Operands use `name:/path` to select a configured source:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
python app.py fs ls data:/
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
The example uses an in-memory filesystem. The
|
|
85
|
+
[overview](https://opencadc.github.io/fsspec-tools/cli/) shows how to configure
|
|
86
|
+
local disk and a remote VOSpace archive. With sources named `local` and `archive`,
|
|
87
|
+
copy a file between them with:
|
|
88
|
+
|
|
89
|
+
```bash
|
|
90
|
+
python app.py fs cp local:/results.csv archive:/2026/results.csv
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
## Commands
|
|
94
|
+
|
|
95
|
+
`ls` · `du` · `find` · `tree` · `size` · `test` · `info` · `stat` ·
|
|
96
|
+
`head` · `tail` · `cat` · `cp` · `mv` · `mkdir` · `rmdir` · `unlink` · `rm`.
|
|
97
|
+
|
|
98
|
+
Recursive `cp -R` is available by default. Enable recursive `rm -R` explicitly:
|
|
99
|
+
|
|
100
|
+
```python
|
|
101
|
+
App({"data": data_source}, capabilities={"recursion": {"remove": True}})
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
## Public API
|
|
105
|
+
|
|
106
|
+
`App`, `AppCapabilities`, `RecursionCapabilities`, `AsyncFilesystemSource`,
|
|
107
|
+
`CommandCallback`, and `CommandContext`.
|
|
108
|
+
|
|
109
|
+
## Documentation
|
|
110
|
+
|
|
111
|
+
| For | Read |
|
|
112
|
+
| --- | --- |
|
|
113
|
+
| Embedding it: sources, lifecycle, capabilities, exit statuses | [Integration guide](https://opencadc.github.io/fsspec-tools/cli/integration/) |
|
|
114
|
+
| What each command does | [Command reference](https://opencadc.github.io/fsspec-tools/cli/commands/) |
|
|
115
|
+
| The public API | [API reference](https://opencadc.github.io/fsspec-tools/cli/api-reference/) |
|
|
116
|
+
| The command contract and its rationale | [`docs/design/`](docs/design/) |
|
|
117
|
+
| Architecture decisions | [`docs/adr/`](../../docs/adr/) |
|
|
118
|
+
|
|
119
|
+
## Scope
|
|
120
|
+
|
|
121
|
+
`fss-cli` uses familiar shell layouts for backend metadata and marks missing
|
|
122
|
+
values as unknown. It does not claim POSIX, GNU, BSD/macOS, or all-fsspec
|
|
123
|
+
compatibility. Supported host platforms are Linux and macOS.
|
|
124
|
+
|
|
125
|
+
## License
|
|
126
|
+
|
|
127
|
+
Copyright (c) 2026 National Research Council of Canada / Government of Canada.
|
|
128
|
+
|
|
129
|
+
`fss-cli` is distributed under the terms of the
|
|
130
|
+
[GNU Affero General Public License v3.0 or later](LICENSE) (AGPL-3.0-or-later).
|
|
131
|
+
`fss-cli` 0.10.0 and earlier releases were published under the BSD 3-Clause
|
|
132
|
+
License.
|
|
133
|
+
|
|
134
|
+
## Long listings and repeated copies
|
|
135
|
+
|
|
136
|
+
`ls -l` keeps shell column order even when metadata is sparse. Unknown scalar
|
|
137
|
+
fields show `-`, unknown permission bits show `?`, and old dates show the year.
|
|
138
|
+
VOSpace access groups share one field (`r=OSSOS,w=NONE`) rather than implying
|
|
139
|
+
POSIX group ownership; raw access and lock metadata remain available in `info`.
|
|
140
|
+
|
|
141
|
+
Recursive `cp -R` skips files only after compatible MD5 metadata or staged
|
|
142
|
+
SHA-256 comparison establishes equality. Equal size alone is insufficient.
|
|
143
|
+
Without content checksums, comparison downloads both files. If their contents
|
|
144
|
+
differ, the upload reuses the staged source bytes.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
fss_cli/__init__.py,sha256=rDS1Ud7EN7h23W8vOayqeELLvv8yBsdRj8F3aXZYas8,520
|
|
2
|
+
fss_cli/_accounts.py,sha256=VEuXGonYn3NINHye2Sv8XX0DV84iLtslDbpE-Bpkrg0,819
|
|
3
|
+
fss_cli/_app.py,sha256=ONAJgoRK9ZKCCm661uchV2x6JPhMYY0xlXsf5Aczeyw,22546
|
|
4
|
+
fss_cli/_cat.py,sha256=oUjJBTpQShMvksEGZlcc7K4KS5ud8XuK-Wb9_oegTYc,12484
|
|
5
|
+
fss_cli/_command.py,sha256=tv2VpnIQihYTB2S8N45iVTbCvrCWGcsOVUb066o3RX4,11993
|
|
6
|
+
fss_cli/_concurrent.py,sha256=L70Omh5wOq8H5JGidKWldWpVcQQHaQ1Jgtnc44AKyXI,6832
|
|
7
|
+
fss_cli/_content.py,sha256=eQeLl3HAJqVCAO3IT0cgSjy8uVOf5i4nGccvDkSgp2M,3202
|
|
8
|
+
fss_cli/_copy_file.py,sha256=9wdrDyZvrORyHBSbuLn6rrDBupzH9LLmmk9G1pFqCzc,1912
|
|
9
|
+
fss_cli/_cp.py,sha256=i6ZAkhjumzpo9dP_3i5dUDeqqJ8YObltIsQ65eNOOjs,24854
|
|
10
|
+
fss_cli/_diagnostics.py,sha256=05JPZ0U-zZqN8MoPNXG06fiouVQQ_kzbKnj9HMsQPbE,347
|
|
11
|
+
fss_cli/_du.py,sha256=McMYQ8r-YoLYnbKcIjImZFmngccRGpWeB_oejhFMJg8,2668
|
|
12
|
+
fss_cli/_find.py,sha256=kGAJuC7fpyYhso4o0jbpPF4s7HMtbca6TYrwnndF334,3358
|
|
13
|
+
fss_cli/_head_tail.py,sha256=Y6Bpk69zjEd3D2xIjR9CK_M5TIOMglCv2jkzAcZqG8o,3946
|
|
14
|
+
fss_cli/_info.py,sha256=ocWr-G__32FBWoxX9ueKo6sl_6UP5AYFsxTbq8R87eE,5373
|
|
15
|
+
fss_cli/_listing.py,sha256=HP96xZx7ZRaxJjTvRO7KnMpIO3u3uAgyyhYZ8CYrp2I,9352
|
|
16
|
+
fss_cli/_logging.py,sha256=T-frFsVEWWA6rm3KeDAi2fNyUiWBm-zC9tKLR9PODf0,2272
|
|
17
|
+
fss_cli/_ls.py,sha256=pxy2BrLGMgZKzHd21CqRyEgF1abHBo8Szo_v0Y26riU,13959
|
|
18
|
+
fss_cli/_manifest.py,sha256=iMDQQ7belP9TzMX5FN1zTVy_wFnJ7YQRV6BQcLdQ-rM,8068
|
|
19
|
+
fss_cli/_metadata.py,sha256=jhFZndn-vOqpoEB-wbUTm5m8PK9V2g8wr8s6RwEsv2o,2246
|
|
20
|
+
fss_cli/_mkdir.py,sha256=l4TDDYJshADcy7C3H38BcCccVbCF931W90fsKzNolMI,2972
|
|
21
|
+
fss_cli/_mv.py,sha256=7f27UZjiPXYO72D5plnOFu63c3baSBjCgUjnVCHh-fY,4502
|
|
22
|
+
fss_cli/_path.py,sha256=le9_z48UVLTxSjlJ5ciAM6O8DNOXFKUT-edi16pKygQ,1723
|
|
23
|
+
fss_cli/_post_copy.py,sha256=aXnt_bDsIwL5AFGlmkDmzvo6nAaY7qILa56NNFXVYSw,6777
|
|
24
|
+
fss_cli/_progress.py,sha256=E_pc6QBwSmTNf3iSsFTeBRAUV1hLded3ZTa9mALFW6Q,4531
|
|
25
|
+
fss_cli/_recursive_cp.py,sha256=WNVtLuGFNm5NZYtZhN51WliHCP0iuuA8g9T6h96RR3Y,33604
|
|
26
|
+
fss_cli/_recursive_rm.py,sha256=Rm_9B9AES81aCbfKsXQ57qf2-5UArYQ-7tg7FuhMQ9w,10635
|
|
27
|
+
fss_cli/_results.py,sha256=uNkc_28nQkcCnaJPioHxlcBiDJnqJ7j2TnCfP5-NgQQ,4484
|
|
28
|
+
fss_cli/_rm.py,sha256=-sNFtCu7T9edhKNc4YXPNp4EZEWsubpRx0HcX6uT-aE,4848
|
|
29
|
+
fss_cli/_rmdir.py,sha256=CvrDQP2UrEMUT2c7t2_g_O7RJAgdWSfwoeZ2ARNbE7o,5249
|
|
30
|
+
fss_cli/_size.py,sha256=IRi1ALxu4R6Hkq1uMKnwzJ0z6Gf4DOXNpWHq4BANBIA,2895
|
|
31
|
+
fss_cli/_sources.py,sha256=QJDLz0f0UNuspu28-imMKyyycsT3McarLiD_Yyh-kls,6684
|
|
32
|
+
fss_cli/_stat.py,sha256=N6rI5ac5M2uw_jEwtnXzJdRmJYCxZStj60b0xbbSwpg,5863
|
|
33
|
+
fss_cli/_test.py,sha256=qfyYeRmoibS1Hr9uJqnqXbqo8M31VvyvKDOuQPbF8vY,1614
|
|
34
|
+
fss_cli/_tree.py,sha256=Fuldi4jIJnmkus25R7eLOlyT4mxx5kaCoI9wnQM-h-s,6525
|
|
35
|
+
fss_cli/_unlink.py,sha256=5cp94CidKvouXuIKDow4nSzc8hrK7ePWLBZ4zU4agoU,3299
|
|
36
|
+
fss_cli/_walk.py,sha256=4zuoFRAKZqFxze-a5E8q5f4lVqRgtkMipOtRVb8pm84,5272
|
|
37
|
+
fss_cli-0.11.0.dist-info/licenses/LICENSE,sha256=dql8h4yceoMhuzlcK0TT_i-NgTFNIZsgE47Q4t3dUYI,34520
|
|
38
|
+
fss_cli-0.11.0.dist-info/WHEEL,sha256=si2p8vW03RnhyBnhWl4tj-ObJa78SpE-c9HsEtblGR0,80
|
|
39
|
+
fss_cli-0.11.0.dist-info/METADATA,sha256=UJw4pfU6ec0v5h-f5QBCk3lgIswcfdjZ6o_Gr6fH3CI,5410
|
|
40
|
+
fss_cli-0.11.0.dist-info/RECORD,,
|