sharefetch 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sharefetch/__init__.py +11 -0
- sharefetch/__main__.py +8 -0
- sharefetch/auth.py +343 -0
- sharefetch/cli.py +782 -0
- sharefetch/config.py +374 -0
- sharefetch/engine.py +1055 -0
- sharefetch/errors.py +57 -0
- sharefetch/events.py +129 -0
- sharefetch/graph.py +474 -0
- sharefetch/gui.py +1170 -0
- sharefetch/hashes.py +123 -0
- sharefetch/listing.py +493 -0
- sharefetch/manifest.py +198 -0
- sharefetch/pacing.py +211 -0
- sharefetch/paths.py +245 -0
- sharefetch/py.typed +0 -0
- sharefetch/resolve.py +660 -0
- sharefetch/state.py +707 -0
- sharefetch/tokencache.py +214 -0
- sharefetch/transfer.py +815 -0
- sharefetch/urls.py +195 -0
- sharefetch/verify.py +378 -0
- sharefetch-1.0.0.dist-info/METADATA +193 -0
- sharefetch-1.0.0.dist-info/RECORD +28 -0
- sharefetch-1.0.0.dist-info/WHEEL +5 -0
- sharefetch-1.0.0.dist-info/entry_points.txt +5 -0
- sharefetch-1.0.0.dist-info/licenses/LICENSE +21 -0
- sharefetch-1.0.0.dist-info/top_level.txt +1 -0
sharefetch/manifest.py
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"""The run's user-facing record: one CSV row per enumerated item.
|
|
2
|
+
|
|
3
|
+
The manifest answers, for a person who was not watching the run, what the tool
|
|
4
|
+
was asked to fetch and what happened to each item. Every enumerated item gets a
|
|
5
|
+
row whatever its status, so a run finishing with failures produces a manifest
|
|
6
|
+
recording those failures next to the files that succeeded. The filename, the
|
|
7
|
+
encoding and the ten columns are fixed.
|
|
8
|
+
|
|
9
|
+
The file is written UTF-8 with a BOM, because it exists to be opened in Excel
|
|
10
|
+
by double-click and Excel reads a BOM-less UTF-8 CSV as the system codepage.
|
|
11
|
+
|
|
12
|
+
Nothing here removes a file. The rows go to ``manifest.csv.tmp`` which is
|
|
13
|
+
renamed onto ``manifest.csv`` by ``os.replace``, so a run interrupted mid-write
|
|
14
|
+
leaves the previous manifest whole and never a truncated one.
|
|
15
|
+
|
|
16
|
+
Two column readings this module settles, because the fixed list names the
|
|
17
|
+
columns without defining them:
|
|
18
|
+
|
|
19
|
+
``local_path``
|
|
20
|
+
Written absolute, resolved against the destination the caller passes. The
|
|
21
|
+
``relative_path`` column already carries the drive-relative form, so a
|
|
22
|
+
second relative path would duplicate it for every item except a sanitised
|
|
23
|
+
one. An absolute path pastes into Explorer. The state file's own
|
|
24
|
+
``local_path`` field stays destination-relative and is unaffected.
|
|
25
|
+
|
|
26
|
+
``verification_reason``
|
|
27
|
+
Carries the failing check's reason where verification failed, and the
|
|
28
|
+
advisory findings where it passed with warnings. Check 5 warns and never
|
|
29
|
+
fails, and this is the only column in the fixed ten that can report it.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
import csv
|
|
33
|
+
import logging
|
|
34
|
+
import os
|
|
35
|
+
from pathlib import Path
|
|
36
|
+
|
|
37
|
+
from .config import MANIFEST_FILENAME, TMP_SUFFIX
|
|
38
|
+
from .errors import ManifestError
|
|
39
|
+
from .paths import long_path
|
|
40
|
+
from .state import ItemState, StateStore
|
|
41
|
+
|
|
42
|
+
logger = logging.getLogger(__name__)
|
|
43
|
+
|
|
44
|
+
# The fixed ten columns, in order. The header row and every data row are built
|
|
45
|
+
# from this tuple, so the two cannot drift.
|
|
46
|
+
MANIFEST_COLUMNS = (
|
|
47
|
+
"relative_path",
|
|
48
|
+
"local_path",
|
|
49
|
+
"size_bytes",
|
|
50
|
+
"ctag",
|
|
51
|
+
"etag",
|
|
52
|
+
"status",
|
|
53
|
+
"verification_status",
|
|
54
|
+
"verification_reason",
|
|
55
|
+
"original_name",
|
|
56
|
+
"completed_at",
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
# Excel and LibreOffice treat a cell opening with one of these as a formula
|
|
60
|
+
# rather than as text. A SharePoint library can hold a file whose name starts
|
|
61
|
+
# with any of them, so the condition is reported at WARNING and the value is
|
|
62
|
+
# still written byte for byte. Rewriting it would make the manifest disagree
|
|
63
|
+
# with the state file and with the name on disk, which is the one thing a
|
|
64
|
+
# record of a download may not do.
|
|
65
|
+
FORMULA_LEAD_CHARACTERS = ("=", "+", "-", "@", "\t", "\r")
|
|
66
|
+
|
|
67
|
+
# The keys verify.VerificationResult.to_dict() writes. The manifest reads the
|
|
68
|
+
# dictionary the state file carries rather than importing the dataclass,
|
|
69
|
+
# because a record loaded from a previous run is a plain dictionary.
|
|
70
|
+
_VERIFICATION_STATUS_KEY = "status"
|
|
71
|
+
_VERIFICATION_REASON_KEY = "reason"
|
|
72
|
+
_VERIFICATION_WARNINGS_KEY = "warnings"
|
|
73
|
+
|
|
74
|
+
# Separates several advisory findings inside the one reason cell. A space
|
|
75
|
+
# follows so Excel wraps the cell on the separator.
|
|
76
|
+
_WARNING_SEPARATOR = "; "
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _cell(value: object) -> str:
|
|
80
|
+
"""Return the text written for one field.
|
|
81
|
+
|
|
82
|
+
``None`` becomes an empty cell. Every other value is rendered by ``str``,
|
|
83
|
+
so an integer size is written without a thousands separator and a boolean
|
|
84
|
+
never reaches a cell at all.
|
|
85
|
+
"""
|
|
86
|
+
if value is None:
|
|
87
|
+
return ""
|
|
88
|
+
return str(value)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _verification_cells(verification: object) -> tuple[str, str]:
|
|
92
|
+
"""Return the ``verification_status`` and ``verification_reason`` cells.
|
|
93
|
+
|
|
94
|
+
``verification`` is the dictionary
|
|
95
|
+
:meth:`verify.VerificationResult.to_dict` produced, as the state file
|
|
96
|
+
carries it, or ``None`` for an item that never reached verification. A
|
|
97
|
+
record that is neither is reported at WARNING and written as two empty
|
|
98
|
+
cells, because a manifest that omits a row is worse than one that omits a
|
|
99
|
+
verdict.
|
|
100
|
+
"""
|
|
101
|
+
if verification is None:
|
|
102
|
+
return "", ""
|
|
103
|
+
if not isinstance(verification, dict):
|
|
104
|
+
logger.warning(
|
|
105
|
+
"verification record is a %s, expected an object; its manifest cells are left empty",
|
|
106
|
+
type(verification).__name__,
|
|
107
|
+
)
|
|
108
|
+
return "", ""
|
|
109
|
+
|
|
110
|
+
status = _cell(verification.get(_VERIFICATION_STATUS_KEY))
|
|
111
|
+
reason = _cell(verification.get(_VERIFICATION_REASON_KEY))
|
|
112
|
+
if reason:
|
|
113
|
+
return status, reason
|
|
114
|
+
|
|
115
|
+
warnings = verification.get(_VERIFICATION_WARNINGS_KEY)
|
|
116
|
+
if isinstance(warnings, (list, tuple)):
|
|
117
|
+
return status, _WARNING_SEPARATOR.join(_cell(warning) for warning in warnings)
|
|
118
|
+
if warnings is not None:
|
|
119
|
+
logger.warning(
|
|
120
|
+
"verification warnings are a %s, expected a list; "
|
|
121
|
+
"the manifest reason cell is left empty",
|
|
122
|
+
type(warnings).__name__,
|
|
123
|
+
)
|
|
124
|
+
return status, ""
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _row(relative_path: str, item: ItemState, dest: Path) -> list[str]:
|
|
128
|
+
"""Build one data row, in ``MANIFEST_COLUMNS`` order."""
|
|
129
|
+
values = {
|
|
130
|
+
"relative_path": relative_path,
|
|
131
|
+
"local_path": str(dest / item.local_path),
|
|
132
|
+
"size_bytes": item.size,
|
|
133
|
+
"ctag": item.ctag,
|
|
134
|
+
"etag": item.etag,
|
|
135
|
+
"status": item.status,
|
|
136
|
+
"original_name": item.original_name,
|
|
137
|
+
"completed_at": item.completed_at,
|
|
138
|
+
}
|
|
139
|
+
verification_status, verification_reason = _verification_cells(item.verification)
|
|
140
|
+
values["verification_status"] = verification_status
|
|
141
|
+
values["verification_reason"] = verification_reason
|
|
142
|
+
return [_cell(values[column]) for column in MANIFEST_COLUMNS]
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _warn_on_formula_leads(rows: list[list[str]]) -> None:
|
|
146
|
+
"""Report cells a spreadsheet will evaluate as a formula.
|
|
147
|
+
|
|
148
|
+
The count is logged once for the whole manifest, with the first offending
|
|
149
|
+
cell named, so a library holding a hundred such names produces one line and
|
|
150
|
+
not a hundred. The values themselves are written unchanged.
|
|
151
|
+
"""
|
|
152
|
+
offenders = [cell for row in rows for cell in row if cell.startswith(FORMULA_LEAD_CHARACTERS)]
|
|
153
|
+
if not offenders:
|
|
154
|
+
return
|
|
155
|
+
logger.warning(
|
|
156
|
+
"%d manifest cell(s) open with a character a spreadsheet evaluates as a formula, "
|
|
157
|
+
"the first being %r; the values are written unchanged and the file is safe to "
|
|
158
|
+
"read as text",
|
|
159
|
+
len(offenders),
|
|
160
|
+
offenders[0],
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def write_manifest(store: StateStore, dest: Path) -> Path:
|
|
165
|
+
"""Write ``<dest>/manifest.csv`` and return the path written.
|
|
166
|
+
|
|
167
|
+
Every item the store holds gets a row, ordered by its drive-relative path,
|
|
168
|
+
so two runs over an unchanged tree produce byte-identical manifests
|
|
169
|
+
whatever order the enumeration returned.
|
|
170
|
+
|
|
171
|
+
Raises :class:`errors.ManifestError` where the file cannot be written. The
|
|
172
|
+
caller decides what a failure means for the run's exit status; the
|
|
173
|
+
downloaded tree is already on disk and is unaffected either way.
|
|
174
|
+
"""
|
|
175
|
+
dest = Path(dest)
|
|
176
|
+
manifest_path = dest / MANIFEST_FILENAME
|
|
177
|
+
tmp_path = dest / (MANIFEST_FILENAME + TMP_SUFFIX)
|
|
178
|
+
|
|
179
|
+
items = store.all_items()
|
|
180
|
+
rows = [_row(relative_path, items[relative_path], dest) for relative_path in sorted(items)]
|
|
181
|
+
_warn_on_formula_leads(rows)
|
|
182
|
+
|
|
183
|
+
try:
|
|
184
|
+
os.makedirs(long_path(dest), exist_ok=True) # noqa: PTH103
|
|
185
|
+
with open( # noqa: PTH123
|
|
186
|
+
long_path(tmp_path), "w", encoding="utf-8-sig", newline=""
|
|
187
|
+
) as handle:
|
|
188
|
+
writer = csv.writer(handle)
|
|
189
|
+
writer.writerow(MANIFEST_COLUMNS)
|
|
190
|
+
writer.writerows(rows)
|
|
191
|
+
handle.flush()
|
|
192
|
+
os.fsync(handle.fileno())
|
|
193
|
+
os.replace(long_path(tmp_path), long_path(manifest_path)) # noqa: PTH105
|
|
194
|
+
except OSError as exc:
|
|
195
|
+
raise ManifestError(f"the manifest {manifest_path} could not be written: {exc}") from exc
|
|
196
|
+
|
|
197
|
+
logger.info("manifest written to %s with %d item(s)", manifest_path, len(rows))
|
|
198
|
+
return manifest_path
|
sharefetch/pacing.py
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
"""Traffic discipline: one shared pacer, the backoff curve and ``Retry-After``.
|
|
2
|
+
|
|
3
|
+
A single ``Pacer`` instance is shared by every worker and by enumeration.
|
|
4
|
+
Per-worker pacing doubles the effective request rate at concurrency 2, so the
|
|
5
|
+
sharing is the mechanism this module exists to provide, and the inter-request
|
|
6
|
+
delay, the byte-rate ceiling and the throttle hold are all measured globally
|
|
7
|
+
across that one instance.
|
|
8
|
+
|
|
9
|
+
``sleep``, ``clock`` and ``rand`` are injected so the behaviour is testable
|
|
10
|
+
without elapsing real time. Nothing in this module calls ``time.sleep``
|
|
11
|
+
directly.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import logging
|
|
15
|
+
import random
|
|
16
|
+
import re
|
|
17
|
+
import threading
|
|
18
|
+
import time
|
|
19
|
+
from collections.abc import Callable
|
|
20
|
+
from datetime import timezone
|
|
21
|
+
from email.utils import parsedate_to_datetime
|
|
22
|
+
|
|
23
|
+
from .config import (
|
|
24
|
+
BACKOFF_BASE_SECONDS,
|
|
25
|
+
BACKOFF_CAP_SECONDS,
|
|
26
|
+
RETRY_AFTER_CLAMP,
|
|
27
|
+
check_delay,
|
|
28
|
+
check_limit_rate,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
logger = logging.getLogger(__name__)
|
|
32
|
+
|
|
33
|
+
# RFC 7231 delay-seconds is one or more digits and nothing else. Anything that
|
|
34
|
+
# is not this shape is tried as an HTTP-date.
|
|
35
|
+
_DELAY_SECONDS = re.compile(r"^\d+$")
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class Pacer:
|
|
39
|
+
"""Global inter-request delay, byte-rate ceiling and throttle hold.
|
|
40
|
+
|
|
41
|
+
One instance is shared by every worker. ``acquire`` is called once per
|
|
42
|
+
request, ``consume`` once per chunk read, and ``hold`` once per ``429`` or
|
|
43
|
+
``503``. A hold set from one worker is observed by every other worker
|
|
44
|
+
inside ``acquire``.
|
|
45
|
+
|
|
46
|
+
The lock guards ``_next_request``, ``_hold_until`` and the token bucket. It
|
|
47
|
+
is never held across a sleep, so a worker meeting a throttle can publish
|
|
48
|
+
its hold while another worker is waiting out a delay.
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
def __init__(
|
|
52
|
+
self,
|
|
53
|
+
delay: float,
|
|
54
|
+
limit_rate_kbps: int | None,
|
|
55
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
56
|
+
clock: Callable[[], float] = time.monotonic,
|
|
57
|
+
) -> None:
|
|
58
|
+
self._delay = float(check_delay(delay))
|
|
59
|
+
limit = check_limit_rate(limit_rate_kbps)
|
|
60
|
+
self._sleep = sleep
|
|
61
|
+
self._clock = clock
|
|
62
|
+
self._lock = threading.Lock()
|
|
63
|
+
|
|
64
|
+
# The next instant at which a request may leave. Initialised to the
|
|
65
|
+
# far past so the first acquire never waits.
|
|
66
|
+
self._next_request = float("-inf")
|
|
67
|
+
self._hold_until = float("-inf")
|
|
68
|
+
|
|
69
|
+
# A ceiling of None leaves the bucket off and makes consume a no-op.
|
|
70
|
+
# Capacity is one second of allowance, so a burst of up to one
|
|
71
|
+
# second's worth of bytes passes without a wait. The 1024 conversion
|
|
72
|
+
# from KB/s to bytes per second matches config.Settings.chunk_bytes.
|
|
73
|
+
self._rate: float | None = None if limit is None else float(limit) * 1024
|
|
74
|
+
self._capacity = 0.0 if self._rate is None else self._rate
|
|
75
|
+
self._tokens = self._capacity
|
|
76
|
+
self._bucket_updated = self._clock()
|
|
77
|
+
|
|
78
|
+
@property
|
|
79
|
+
def delay(self) -> float:
|
|
80
|
+
"""The configured inter-request delay in seconds."""
|
|
81
|
+
return self._delay
|
|
82
|
+
|
|
83
|
+
@property
|
|
84
|
+
def rate_bytes_per_second(self) -> float | None:
|
|
85
|
+
"""The configured byte-rate ceiling, or ``None`` when the ceiling is off."""
|
|
86
|
+
return self._rate
|
|
87
|
+
|
|
88
|
+
def acquire(self) -> None:
|
|
89
|
+
"""Block until the inter-request delay and any active hold have elapsed.
|
|
90
|
+
|
|
91
|
+
Claiming the next slot happens under the lock, so two workers calling
|
|
92
|
+
at the same instant are staggered by the delay instead of both firing.
|
|
93
|
+
The hold is re-read after every sleep, so a hold published while this
|
|
94
|
+
worker was waiting is still observed.
|
|
95
|
+
"""
|
|
96
|
+
while True:
|
|
97
|
+
with self._lock:
|
|
98
|
+
now = self._clock()
|
|
99
|
+
ready_at = max(self._next_request, self._hold_until)
|
|
100
|
+
if ready_at <= now:
|
|
101
|
+
self._next_request = now + self._delay
|
|
102
|
+
return
|
|
103
|
+
wait = ready_at - now
|
|
104
|
+
logger.debug("pacer waiting %.3fs before the next request", wait)
|
|
105
|
+
self._sleep(wait)
|
|
106
|
+
|
|
107
|
+
def consume(self, n: int) -> None:
|
|
108
|
+
"""Block as long as the byte-rate ceiling requires after reading ``n`` bytes.
|
|
109
|
+
|
|
110
|
+
Returns immediately when no ceiling is configured. A chunk larger than
|
|
111
|
+
one second of allowance is charged in full and waited out in full, so
|
|
112
|
+
the ceiling holds for any chunk size.
|
|
113
|
+
"""
|
|
114
|
+
if self._rate is None or n <= 0:
|
|
115
|
+
return
|
|
116
|
+
with self._lock:
|
|
117
|
+
now = self._clock()
|
|
118
|
+
elapsed = now - self._bucket_updated
|
|
119
|
+
if elapsed > 0:
|
|
120
|
+
self._tokens = min(self._capacity, self._tokens + elapsed * self._rate)
|
|
121
|
+
self._bucket_updated = now
|
|
122
|
+
# The balance is allowed to go negative. The debt is visible to
|
|
123
|
+
# every other worker, so two workers sharing one ceiling each wait
|
|
124
|
+
# out the combined overdraft rather than half of it.
|
|
125
|
+
self._tokens -= n
|
|
126
|
+
wait = 0.0 if self._tokens >= 0 else -self._tokens / self._rate
|
|
127
|
+
if wait > 0:
|
|
128
|
+
logger.debug("pacer waiting %.3fs to stay under the byte-rate ceiling", wait)
|
|
129
|
+
self._sleep(wait)
|
|
130
|
+
|
|
131
|
+
def hold(self, seconds: float) -> None:
|
|
132
|
+
"""Set a global floor that every worker observes inside ``acquire``.
|
|
133
|
+
|
|
134
|
+
The floor only ever moves later, so a short hold never shortens a hold
|
|
135
|
+
already in force.
|
|
136
|
+
"""
|
|
137
|
+
if seconds <= 0:
|
|
138
|
+
return
|
|
139
|
+
with self._lock:
|
|
140
|
+
until = self._clock() + seconds
|
|
141
|
+
if until <= self._hold_until:
|
|
142
|
+
logger.debug(
|
|
143
|
+
"pacer hold of %.3fs is shorter than the hold already in force", seconds
|
|
144
|
+
)
|
|
145
|
+
return
|
|
146
|
+
self._hold_until = until
|
|
147
|
+
logger.info("pacer holding every worker for %.3fs", seconds)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def backoff_delay(attempt: int, rand: Callable[[float, float], float] = random.uniform) -> float:
|
|
151
|
+
"""Full-jitter exponential backoff for ``attempt``, counted from 1.
|
|
152
|
+
|
|
153
|
+
The jitter is applied so two workers recovering from the same outage do
|
|
154
|
+
not resynchronise.
|
|
155
|
+
"""
|
|
156
|
+
if attempt < 1:
|
|
157
|
+
raise ValueError(f"attempt is counted from 1, received {attempt}")
|
|
158
|
+
computed = min(BACKOFF_BASE_SECONDS * 2 ** (attempt - 1), BACKOFF_CAP_SECONDS)
|
|
159
|
+
return rand(0.0, computed)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def parse_retry_after(value: str | None, now: float | None = None) -> float | None:
|
|
163
|
+
"""Return the ``Retry-After`` wait in seconds, or ``None`` when unusable.
|
|
164
|
+
|
|
165
|
+
Both forms RFC 7231 allows are accepted: delay-seconds as a bare integer,
|
|
166
|
+
and an HTTP-date. A missing, empty or unparseable value returns ``None``,
|
|
167
|
+
which the caller treats as a fall-through to ``backoff_delay``. The result
|
|
168
|
+
is clamped to ``config.RETRY_AFTER_CLAMP``, so a date already in the past
|
|
169
|
+
gives 0.0 and an implausibly distant one gives the ceiling.
|
|
170
|
+
|
|
171
|
+
``now`` is epoch seconds and defaults to ``time.time()``. It is a wall
|
|
172
|
+
clock rather than a monotonic one because an HTTP-date is an absolute
|
|
173
|
+
instant.
|
|
174
|
+
"""
|
|
175
|
+
if value is None:
|
|
176
|
+
return None
|
|
177
|
+
text = value.strip()
|
|
178
|
+
if not text:
|
|
179
|
+
return None
|
|
180
|
+
|
|
181
|
+
if _DELAY_SECONDS.match(text):
|
|
182
|
+
return _clamp(float(text))
|
|
183
|
+
|
|
184
|
+
seconds = _seconds_until_http_date(text, now)
|
|
185
|
+
if seconds is None:
|
|
186
|
+
logger.warning("Retry-After header %r is neither delay-seconds nor an HTTP-date", text)
|
|
187
|
+
return None
|
|
188
|
+
return _clamp(seconds)
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _seconds_until_http_date(text: str, now: float | None) -> float | None:
|
|
192
|
+
try:
|
|
193
|
+
when = parsedate_to_datetime(text)
|
|
194
|
+
except (ValueError, TypeError):
|
|
195
|
+
return None
|
|
196
|
+
if when is None: # pragma: no cover - older stdlib returned None here
|
|
197
|
+
return None
|
|
198
|
+
if when.tzinfo is None:
|
|
199
|
+
# RFC 7231 requires GMT. A date arriving without a zone is read as UTC
|
|
200
|
+
# rather than as local time, which would shift it by the offset.
|
|
201
|
+
when = when.replace(tzinfo=timezone.utc)
|
|
202
|
+
reference = time.time() if now is None else now
|
|
203
|
+
return when.timestamp() - reference
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _clamp(seconds: float) -> float:
|
|
207
|
+
low, high = RETRY_AFTER_CLAMP
|
|
208
|
+
clamped = min(max(seconds, low), high)
|
|
209
|
+
if clamped != seconds:
|
|
210
|
+
logger.info("Retry-After of %.3fs clamped to %.3fs", seconds, clamped)
|
|
211
|
+
return clamped
|
sharefetch/paths.py
ADDED
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
"""Mapping a Graph drive-relative path onto a local path Windows accepts.
|
|
2
|
+
|
|
3
|
+
The remote name space allows names the local one refuses. A drive item may be
|
|
4
|
+
named ``report:final.txt``, may end in a dot or a space, may collide with a
|
|
5
|
+
reserved device name such as ``CON``, and may exceed the 255-character ceiling a
|
|
6
|
+
single Windows path component carries. This module holds every rule that closes
|
|
7
|
+
that gap, and it holds them once, so the enumerator, the transfer loop, the
|
|
8
|
+
state store and the manifest writer all agree on where an item lands.
|
|
9
|
+
|
|
10
|
+
The module is pure: it opens no socket, reads no clock and touches no file. The
|
|
11
|
+
``\\\\?\\`` prefix ``long_path`` produces is applied at the filesystem boundary
|
|
12
|
+
and nowhere else; the internal representation stays an ordinary
|
|
13
|
+
``pathlib.Path``.
|
|
14
|
+
|
|
15
|
+
Every substitution is logged at INFO, so a run's log recovers the mapping back
|
|
16
|
+
to the original name even where the state file is discarded.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
import ctypes
|
|
20
|
+
import logging
|
|
21
|
+
import os
|
|
22
|
+
from dataclasses import dataclass
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
from .config import (
|
|
26
|
+
MAX_PATH_SEGMENT,
|
|
27
|
+
WINDOWS_FORBIDDEN_CHARS,
|
|
28
|
+
WINDOWS_RESERVED_NAMES,
|
|
29
|
+
)
|
|
30
|
+
from .config import REASON_PATH_TOO_LONG as _REASON_PATH_TOO_LONG
|
|
31
|
+
from .errors import ValidationError
|
|
32
|
+
|
|
33
|
+
logger = logging.getLogger(__name__)
|
|
34
|
+
|
|
35
|
+
REASON_PATH_TOO_LONG = _REASON_PATH_TOO_LONG
|
|
36
|
+
REPLACEMENT = "_"
|
|
37
|
+
COLLISION_MARKER = "~"
|
|
38
|
+
CONTROL_CHARACTER_CEILING = 0x20
|
|
39
|
+
TRAILING_STRIP_CHARS = ". "
|
|
40
|
+
|
|
41
|
+
# Beyond the forbidden set config.py names: a backslash is a path separator on
|
|
42
|
+
# Windows, so a backslash surviving inside a segment would silently split one
|
|
43
|
+
# component into two. A separator never appears inside a segment, and replacing
|
|
44
|
+
# it makes that hold rather than leaving it assumed.
|
|
45
|
+
_FORBIDDEN_CHARACTERS = frozenset(
|
|
46
|
+
set(WINDOWS_FORBIDDEN_CHARS) | {"\\"} | {chr(code) for code in range(CONTROL_CHARACTER_CEILING)}
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(frozen=True)
|
|
51
|
+
class MappedPath:
|
|
52
|
+
"""Where one drive item lands below the destination root.
|
|
53
|
+
|
|
54
|
+
``local_path`` is relative to the destination root and carries the platform
|
|
55
|
+
separator. ``sanitised`` is true where the local path differs from the
|
|
56
|
+
drive-relative path for any reason, including a collision suffix.
|
|
57
|
+
``original_name`` carries the drive-relative path exactly as supplied, and
|
|
58
|
+
is set only where ``sanitised`` is true, so the manifest and the state file
|
|
59
|
+
can map the local name back to the remote one.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
local_path: Path
|
|
63
|
+
sanitised: bool
|
|
64
|
+
original_name: str | None
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def sanitise_segment(name: str) -> tuple[str, bool]:
|
|
68
|
+
"""Return one path component Windows accepts, and whether it changed.
|
|
69
|
+
|
|
70
|
+
The rules apply in a fixed order: replace every forbidden and control
|
|
71
|
+
character, strip the trailing run of dots and spaces, prefix a reserved
|
|
72
|
+
device name, then substitute a single underscore for an empty result.
|
|
73
|
+
|
|
74
|
+
Raises ``ValidationError`` where the result exceeds
|
|
75
|
+
``config.MAX_PATH_SEGMENT`` characters. The result is never truncated,
|
|
76
|
+
because a truncated name no longer identifies the item it came from.
|
|
77
|
+
"""
|
|
78
|
+
replaced = "".join(REPLACEMENT if char in _FORBIDDEN_CHARACTERS else char for char in name)
|
|
79
|
+
trimmed = replaced.rstrip(TRAILING_STRIP_CHARS)
|
|
80
|
+
|
|
81
|
+
stem = trimmed.partition(".")[0]
|
|
82
|
+
prefixed = REPLACEMENT + trimmed if stem.upper() in WINDOWS_RESERVED_NAMES else trimmed
|
|
83
|
+
|
|
84
|
+
result = prefixed if prefixed else REPLACEMENT
|
|
85
|
+
|
|
86
|
+
if len(result) > MAX_PATH_SEGMENT:
|
|
87
|
+
raise ValidationError(
|
|
88
|
+
f"{REASON_PATH_TOO_LONG}: path component is {len(result)} characters after "
|
|
89
|
+
f"sanitisation, above the {MAX_PATH_SEGMENT}-character Windows limit that the "
|
|
90
|
+
f"long-path prefix does not lift; the name is never truncated because truncation "
|
|
91
|
+
f"destroys identity"
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
changed = result != name
|
|
95
|
+
if changed:
|
|
96
|
+
logger.info("path component %r mapped to %r", name, result)
|
|
97
|
+
return result, changed
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def map_drive_path(drive_relative: str, taken: set[str]) -> MappedPath:
|
|
101
|
+
"""Map a drive-relative path onto a local path below the destination root.
|
|
102
|
+
|
|
103
|
+
``taken`` holds the lowercased string form of every local path already
|
|
104
|
+
assigned in this run. A candidate colliding with one of them gains a ``~1``,
|
|
105
|
+
``~2`` suffix before its extension, incrementing until free, so the result
|
|
106
|
+
is deterministic in enumeration order. The caller owns ``taken`` and adds
|
|
107
|
+
the returned path to it; this function never mutates it.
|
|
108
|
+
|
|
109
|
+
Raises ``ValidationError`` where the path is empty or where a component
|
|
110
|
+
exceeds ``config.MAX_PATH_SEGMENT``.
|
|
111
|
+
"""
|
|
112
|
+
raw_segments = [segment for segment in drive_relative.split("/") if segment]
|
|
113
|
+
if not raw_segments:
|
|
114
|
+
raise ValidationError(
|
|
115
|
+
f"drive-relative path {drive_relative!r} holds no path component, so it names no "
|
|
116
|
+
f"location below the destination root"
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
segments: list[str] = []
|
|
120
|
+
sanitised = False
|
|
121
|
+
for raw_segment in raw_segments:
|
|
122
|
+
mapped_segment, changed = sanitise_segment(raw_segment)
|
|
123
|
+
sanitised = sanitised or changed
|
|
124
|
+
segments.append(mapped_segment)
|
|
125
|
+
|
|
126
|
+
local_path = Path(*segments)
|
|
127
|
+
if str(local_path).lower() in taken:
|
|
128
|
+
local_path = _disambiguate(segments, taken)
|
|
129
|
+
sanitised = True
|
|
130
|
+
|
|
131
|
+
return MappedPath(
|
|
132
|
+
local_path=local_path,
|
|
133
|
+
sanitised=sanitised,
|
|
134
|
+
original_name=drive_relative if sanitised else None,
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _disambiguate(segments: list[str], taken: set[str]) -> Path:
|
|
139
|
+
"""Return the first free ``~N`` variant of the last component.
|
|
140
|
+
|
|
141
|
+
The loop terminates because ``taken`` is finite and every iteration
|
|
142
|
+
produces a distinct candidate, so at most ``len(taken) + 1`` candidates are
|
|
143
|
+
tested.
|
|
144
|
+
"""
|
|
145
|
+
parents = segments[:-1]
|
|
146
|
+
leaf = segments[-1]
|
|
147
|
+
|
|
148
|
+
dot = leaf.rfind(".")
|
|
149
|
+
if dot > 0:
|
|
150
|
+
stem, suffix = leaf[:dot], leaf[dot:]
|
|
151
|
+
else:
|
|
152
|
+
stem, suffix = leaf, ""
|
|
153
|
+
|
|
154
|
+
index = 1
|
|
155
|
+
while True:
|
|
156
|
+
candidate_leaf = f"{stem}{COLLISION_MARKER}{index}{suffix}"
|
|
157
|
+
if len(candidate_leaf) > MAX_PATH_SEGMENT:
|
|
158
|
+
raise ValidationError(
|
|
159
|
+
f"{REASON_PATH_TOO_LONG}: resolving a name collision would make the path "
|
|
160
|
+
f"component {len(candidate_leaf)} characters, above the "
|
|
161
|
+
f"{MAX_PATH_SEGMENT}-character Windows limit"
|
|
162
|
+
)
|
|
163
|
+
candidate = Path(*parents, candidate_leaf)
|
|
164
|
+
if str(candidate).lower() not in taken:
|
|
165
|
+
logger.info(
|
|
166
|
+
"path component %r renamed to %r to resolve a collision", leaf, candidate_leaf
|
|
167
|
+
)
|
|
168
|
+
return candidate
|
|
169
|
+
index += 1
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def long_path(p: Path) -> str:
|
|
173
|
+
"""Return the string form every filesystem call in the package uses.
|
|
174
|
+
|
|
175
|
+
On Windows the ``\\\\?\\`` prefix is applied unconditionally, because
|
|
176
|
+
``LongPathsEnabled`` cannot be assumed on a managed machine. The prefix
|
|
177
|
+
disables normalisation, so the path is made absolute and backslash-
|
|
178
|
+
normalised first. On any other platform the path is returned unchanged.
|
|
179
|
+
"""
|
|
180
|
+
if os.name != "nt":
|
|
181
|
+
return str(p)
|
|
182
|
+
resolved = Path(os.path.abspath(str(p))) # noqa: PTH100
|
|
183
|
+
text = str(resolved)
|
|
184
|
+
if text.startswith("\\\\?\\"):
|
|
185
|
+
return text
|
|
186
|
+
if text.startswith("\\\\"):
|
|
187
|
+
return "\\\\?\\UNC\\" + text[2:]
|
|
188
|
+
return "\\\\?\\" + text
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
# The Windows hidden attribute lives here rather than in tokencache.py so that
|
|
192
|
+
# state.py can set it without importing an authentication library. Saving
|
|
193
|
+
# download state has no business depending on msal.
|
|
194
|
+
FILE_ATTRIBUTE_HIDDEN = 0x02
|
|
195
|
+
FILE_ATTRIBUTE_NORMAL = 0x80
|
|
196
|
+
INVALID_FILE_ATTRIBUTES = -1
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def windows_attributes(path: Path) -> int | None:
|
|
200
|
+
"""Return the Win32 attributes of ``path``, or ``None`` when they cannot be read."""
|
|
201
|
+
current = ctypes.windll.kernel32.GetFileAttributesW(str(path))
|
|
202
|
+
if current == INVALID_FILE_ATTRIBUTES:
|
|
203
|
+
return None
|
|
204
|
+
return current
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def hide_file(path: Path) -> None:
|
|
208
|
+
"""Set the Windows hidden attribute on ``path``.
|
|
209
|
+
|
|
210
|
+
A no-op off Windows, where a dot prefix carries the same meaning and no
|
|
211
|
+
attribute exists. A failure is logged at WARNING and never raised: the
|
|
212
|
+
attribute is a tidiness measure and no caller depends on it.
|
|
213
|
+
"""
|
|
214
|
+
if os.name != "nt":
|
|
215
|
+
return
|
|
216
|
+
current = windows_attributes(path)
|
|
217
|
+
if current is None:
|
|
218
|
+
logger.warning("could not read the file attributes of %s", path)
|
|
219
|
+
return
|
|
220
|
+
if current & FILE_ATTRIBUTE_HIDDEN:
|
|
221
|
+
return
|
|
222
|
+
if not ctypes.windll.kernel32.SetFileAttributesW(str(path), current | FILE_ATTRIBUTE_HIDDEN):
|
|
223
|
+
logger.warning("could not set the hidden attribute on %s", path)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def unhide_file(path: Path) -> None:
|
|
227
|
+
"""Clear the Windows hidden attribute so a truncating rewrite can open the file.
|
|
228
|
+
|
|
229
|
+
``FilePersistence.save`` opens the cache with ``O_CREAT | O_TRUNC``, which the
|
|
230
|
+
C runtime maps to ``CREATE_ALWAYS``. Windows refuses that call with
|
|
231
|
+
``ERROR_ACCESS_DENIED`` when the existing file is hidden, so the attribute is
|
|
232
|
+
cleared for the duration of the write and restored immediately afterwards.
|
|
233
|
+
Both calls happen while ``PersistedTokenCache.modify`` holds its cross-process
|
|
234
|
+
lock, so no other writer observes the file unhidden.
|
|
235
|
+
"""
|
|
236
|
+
if os.name != "nt":
|
|
237
|
+
return
|
|
238
|
+
current = windows_attributes(path)
|
|
239
|
+
if current is None or not current & FILE_ATTRIBUTE_HIDDEN:
|
|
240
|
+
return
|
|
241
|
+
cleared = current & ~FILE_ATTRIBUTE_HIDDEN
|
|
242
|
+
if cleared == 0:
|
|
243
|
+
cleared = FILE_ATTRIBUTE_NORMAL
|
|
244
|
+
if not ctypes.windll.kernel32.SetFileAttributesW(str(path), cleared):
|
|
245
|
+
logger.warning("could not clear the hidden attribute on %s", path)
|
sharefetch/py.typed
ADDED
|
File without changes
|