pushframe 5.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pushframe/__init__.py +4 -0
- pushframe/api/__init__.py +0 -0
- pushframe/api/accountApi.py +72 -0
- pushframe/api/activityApi.py +87 -0
- pushframe/api/assetApi.py +194 -0
- pushframe/api/baseApi.py +6 -0
- pushframe/api/frameApi.py +271 -0
- pushframe/api/notificationApi.py +15 -0
- pushframe/api/peopleApi.py +25 -0
- pushframe/api/playlistApi.py +9 -0
- pushframe/aura.py +182 -0
- pushframe/aws/__init__.py +0 -0
- pushframe/aws/awsclient.py +23 -0
- pushframe/aws/s3client.py +40 -0
- pushframe/aws/sqsclient.py +33 -0
- pushframe/cache.py +50 -0
- pushframe/cli.py +1134 -0
- pushframe/client.py +267 -0
- pushframe/exif.py +147 -0
- pushframe/export.py +53 -0
- pushframe/google/__init__.py +43 -0
- pushframe/google/bootstrap.py +134 -0
- pushframe/google/cache.py +167 -0
- pushframe/google/client.py +140 -0
- pushframe/google/enumerate.py +270 -0
- pushframe/google/manifest.py +111 -0
- pushframe/google/parsers.py +345 -0
- pushframe/google/redaction.py +33 -0
- pushframe/google/vault.py +126 -0
- pushframe/gsync.py +463 -0
- pushframe/migration.py +86 -0
- pushframe/models/__init__.py +0 -0
- pushframe/models/activity.py +79 -0
- pushframe/models/asset.py +159 -0
- pushframe/models/frame.py +105 -0
- pushframe/models/meta.py +11 -0
- pushframe/models/person.py +24 -0
- pushframe/models/user.py +22 -0
- pushframe/ratelimit.py +222 -0
- pushframe/reconcile.py +384 -0
- pushframe/sync.py +1105 -0
- pushframe/utils/dt.py +15 -0
- pushframe/utils/io.py +23 -0
- pushframe/utils/settings.py +59 -0
- pushframe-5.0.0.dist-info/METADATA +53 -0
- pushframe-5.0.0.dist-info/RECORD +49 -0
- pushframe-5.0.0.dist-info/WHEEL +4 -0
- pushframe-5.0.0.dist-info/entry_points.txt +2 -0
- pushframe-5.0.0.dist-info/licenses/LICENSE +31 -0
pushframe/sync.py
ADDED
|
@@ -0,0 +1,1105 @@
|
|
|
1
|
+
"""Sync engine: pure dry-run core (Phase 7) plus the single mutating
|
|
2
|
+
execute path (Phase 8, batched in quick task 260708-fyr).
|
|
3
|
+
|
|
4
|
+
Contains a recursive local-directory scanner + content hasher
|
|
5
|
+
(`scan_directory`) and a pure diff function (`compute_plan`) that classifies
|
|
6
|
+
a frame's assets into upload / delete / unchanged against local hashes --
|
|
7
|
+
both remain pure, no I/O beyond reading local file bytes, no mutation.
|
|
8
|
+
|
|
9
|
+
`execute_plan()` is this module's ONLY mutating function -- the sole place
|
|
10
|
+
that calls select_asset/S3 upload/batch_update (for `to_upload`) and
|
|
11
|
+
remove_asset (for `to_delete`). It never references the hard-delete
|
|
12
|
+
primitive (the other, non-frame-scoped Asset removal call on `AssetApi`);
|
|
13
|
+
the delete loop calls `remove_asset` exclusively (D-06 structural
|
|
14
|
+
isolation, grep-verified absent from this module including comments).
|
|
15
|
+
Uploads are attempted before any delete (D-09), and a single item's
|
|
16
|
+
failure is caught, recorded with its identity, and the loop continues
|
|
17
|
+
rather than aborting (D-08), with results reported back as a separated
|
|
18
|
+
`ExecutionResult` (D-10).
|
|
19
|
+
|
|
20
|
+
Both `select_asset` and `batch_update` are native Pushd BATCH endpoints
|
|
21
|
+
(the official app sends a whole collection in one `{"assets":[...]}` call
|
|
22
|
+
rather than one call per asset -- see
|
|
23
|
+
`.planning/debug/resolved/select-asset-401-unauthorized.md`). `execute_plan`
|
|
24
|
+
chunks `to_upload`/`to_delete` at `WRITE_BATCH_SIZE` and, per upload chunk,
|
|
25
|
+
performs per-file S3 prep (S3 is AWS, not the Pushd anti-abuse surface, so
|
|
26
|
+
those uploads stay per-file) followed by exactly ONE `select_asset` call and
|
|
27
|
+
ONE `batch_update` call carrying every prepped file in the chunk -- not the
|
|
28
|
+
double `select_asset` + per-file round-trip the original single-item flow
|
|
29
|
+
used. Per-file attribution is recovered from `batch_update`'s
|
|
30
|
+
`successes[].local_identifier`: any prepped file whose local_identifier is
|
|
31
|
+
absent from `successes` is attributed a failure. A delete chunk issues one
|
|
32
|
+
`remove_asset` call for every asset id in the chunk; since `remove_asset`
|
|
33
|
+
returns only a failure COUNT (never per-item), a chunk-level failure
|
|
34
|
+
attributes ALL deletes in that chunk (coarser than upload attribution --
|
|
35
|
+
documented tradeoff, T-fyr-01).
|
|
36
|
+
"""
|
|
37
|
+
from __future__ import annotations
|
|
38
|
+
|
|
39
|
+
import time
|
|
40
|
+
import uuid
|
|
41
|
+
from dataclasses import dataclass, field
|
|
42
|
+
from pathlib import Path
|
|
43
|
+
|
|
44
|
+
import httpx
|
|
45
|
+
import pillow_heif
|
|
46
|
+
from PIL import Image
|
|
47
|
+
from loguru import logger
|
|
48
|
+
|
|
49
|
+
from pushframe.aws.s3client import get_md5
|
|
50
|
+
from pushframe.client import AuraError, AuthenticationError, RateLimitError
|
|
51
|
+
from pushframe.models.asset import AssetPartial, AssetPartialId
|
|
52
|
+
from pushframe.ratelimit import BudgetExhausted
|
|
53
|
+
from pushframe.utils.dt import format_dt_to_aura, get_utc_now
|
|
54
|
+
|
|
55
|
+
# D-09/FMT-03: without this call, Image.open() on a .heic path raises
|
|
56
|
+
# before any UTI lookup could even run -- Pillow has no built-in HEIF
|
|
57
|
+
# decoder. Registered here at pushframe.sync import time (not inside
|
|
58
|
+
# Aura.__init__) so _prep_upload stays usable, and testable, without
|
|
59
|
+
# constructing an Aura instance.
|
|
60
|
+
pillow_heif.register_heif_opener()
|
|
61
|
+
|
|
62
|
+
# Only these extensions are eligible for content-hash diffing (D-02). Phase 6
|
|
63
|
+
# confirmed md5_hash is populated for photo assets but null for video assets,
|
|
64
|
+
# so videos/non-images are excluded here rather than diffed unsafely.
|
|
65
|
+
ELIGIBLE_EXTENSIONS = frozenset({'.jpg', '.jpeg', '.png', '.heic'})
|
|
66
|
+
|
|
67
|
+
# D-11: maps the DECODED image's real format (Pillow's `image.format`,
|
|
68
|
+
# read from the same `Image.open()` call `_prep_upload` already performs
|
|
69
|
+
# for dimensions) to the Apple UTI the API expects in `data_uti` -- not
|
|
70
|
+
# the filename or its extension. A `.jpg` that is really a PNG is typed
|
|
71
|
+
# `public.png`; the bytes decide, the name is no longer a trust anchor.
|
|
72
|
+
# Keyed on the exact strings Pillow returns on this installation (verified
|
|
73
|
+
# empirically in Phase 11 Plan 04's Task 2, not assumed): 'JPEG', 'PNG',
|
|
74
|
+
# 'HEIF'.
|
|
75
|
+
#
|
|
76
|
+
# D-12: deliberately narrower than any format Pillow/pillow-heif could
|
|
77
|
+
# theoretically decode -- exactly JPEG/PNG/HEIF, matching
|
|
78
|
+
# ELIGIBLE_EXTENSIONS. WebP was considered and deferred: Google Photos
|
|
79
|
+
# does serve WebP in some paths, but shipping it here would be unverified
|
|
80
|
+
# write surface before Phase 12 has established what bytes the album
|
|
81
|
+
# mechanism actually delivers. An unmapped decoded format -- WebP or
|
|
82
|
+
# anything else -- is never given a fallback UTI; `_prep_upload` fails
|
|
83
|
+
# closed with a message naming the real decoded format, exactly as an
|
|
84
|
+
# unmapped extension did before this table existed.
|
|
85
|
+
_DATA_UTI_BY_IMAGE_FORMAT = {
|
|
86
|
+
'JPEG': 'public.jpeg',
|
|
87
|
+
'PNG': 'public.png',
|
|
88
|
+
'HEIF': 'public.heic',
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
# Seconds to pause before each write network call (select_asset /
|
|
92
|
+
# batch_update / remove_asset) so a bulk apply is paced rather than fired as
|
|
93
|
+
# a rapid burst. Root cause of the select-asset-401-unauthorized session:
|
|
94
|
+
# the Pushd anti-abuse layer trips on burst write volume (a 120-file apply
|
|
95
|
+
# fired ~300ms apart, each upload issuing 2x select_asset + 1x batch_update,
|
|
96
|
+
# got 401 on every call then escalated to a login lockout) while the
|
|
97
|
+
# human-paced phone app is never flagged. Batching (WRITE_BATCH_SIZE) fixes
|
|
98
|
+
# the root cause -- call COUNT -- by collapsing ~3N Pushd write calls to ~2
|
|
99
|
+
# per chunk; this throttle remains as a second, independent layer of
|
|
100
|
+
# defense that paces whatever write calls remain (2 per upload chunk, 1 per
|
|
101
|
+
# delete chunk) so even a batched apply is never fired as a rapid burst. It
|
|
102
|
+
# is deliberately conservative (a full apply is a rare, batch operation, not
|
|
103
|
+
# a latency-sensitive path). Callers can override via `throttle_seconds` (0
|
|
104
|
+
# disables) and inject `sleep` for tests.
|
|
105
|
+
WRITE_THROTTLE_SECONDS = 0.5
|
|
106
|
+
|
|
107
|
+
# Maximum number of items (uploads or deletes) batched into a single
|
|
108
|
+
# select_asset/batch_update/remove_asset call. The official Aura Android app
|
|
109
|
+
# has no observed hard client-side cap and does not appear to chunk at all in
|
|
110
|
+
# normal use, but a single call carrying hundreds of assets is untested
|
|
111
|
+
# territory -- it could re-trip the anti-abuse layer on payload size/shape
|
|
112
|
+
# rather than call volume, or hit an undocumented server-side limit. 50 is a
|
|
113
|
+
# conservative chunk size: small enough to stay well within any plausible
|
|
114
|
+
# server limit, large enough that the ~3N-to-~2-per-chunk call-count
|
|
115
|
+
# reduction is realized in practice. Injectable via `batch_size` so offline
|
|
116
|
+
# tests can exercise the chunk-boundary logic with a small size.
|
|
117
|
+
WRITE_BATCH_SIZE = 50
|
|
118
|
+
|
|
119
|
+
# Seconds to pause BETWEEN write chunks (separate from WRITE_THROTTLE_SECONDS,
|
|
120
|
+
# which paces individual network calls). This spaces out the per-chunk write
|
|
121
|
+
# bursts over the whole apply so the run looks human-paced rather than firing
|
|
122
|
+
# every chunk back-to-back -- a third layer of defense on top of batching
|
|
123
|
+
# (call count) and the per-call throttle. Motivated live: after the batched
|
|
124
|
+
# fix, a first 50-file chunk succeeded but the immediately-following second
|
|
125
|
+
# chunk still tripped the anti-abuse layer, consistent with a cumulative
|
|
126
|
+
# write-volume-over-time limit, not a per-call one. Realized via the injected
|
|
127
|
+
# `sleep` in 1s steps and surfaced through `on_wait` so the CLI can show a
|
|
128
|
+
# live countdown instead of a frozen progress bar. Injectable via
|
|
129
|
+
# `chunk_delay_seconds` (0 disables). Skipped before the first write chunk.
|
|
130
|
+
WRITE_CHUNK_DELAY_SECONDS = 5.0
|
|
131
|
+
|
|
132
|
+
# Number of write items that must fail in an unbroken run before execute_plan
|
|
133
|
+
# aborts the whole batch. Root cause of this session's REOPENED gap: the Pushd
|
|
134
|
+
# anti-abuse trip does NOT reliably announce itself with the 429/475 that
|
|
135
|
+
# `RateLimitError` catches. The live regression showed it as a plain HTTP 401
|
|
136
|
+
# on every write after the 7th succeeded, so execute_plan caught each of the
|
|
137
|
+
# 103 post-trip failures per-item (D-08) and kept hammering the throttling
|
|
138
|
+
# server ~103 more times instead of backing off. A RUN of consecutive failures
|
|
139
|
+
# is the reliable, STATUS-CODE-AGNOSTIC signature of a systemic cut-off, while a
|
|
140
|
+
# genuine isolated failure (one bad/expired asset ref, one permissions edge) is
|
|
141
|
+
# a single failure surrounded by successes and never reaches the threshold.
|
|
142
|
+
# Deliberately counts ANY caught per-item write failure (plain 401, 5xx,
|
|
143
|
+
# network error, ValueError) rather than sniffing the status -- the whole
|
|
144
|
+
# lesson of this session is that the trip cannot be recognised from the HTTP
|
|
145
|
+
# status alone. This is a BACKSTOP below the RateLimitError fast-path: a 429/475
|
|
146
|
+
# still aborts on its FIRST occurrence via its own branch; this catches trips
|
|
147
|
+
# the status code hides. N=5 balances catching the lockout early (5 wasted calls
|
|
148
|
+
# vs 103) against tolerating a small unlucky cluster of independent failures.
|
|
149
|
+
MAX_CONSECUTIVE_WRITE_FAILURES = 5
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
# Cost (in write-budget tokens) of the ONE re-login `execute_plan`'s 401
|
|
153
|
+
# retry performs per run (REL-01..04, D-06). Reads demonstrably kept
|
|
154
|
+
# working throughout the v2.0 write lockouts, so the verify probe this
|
|
155
|
+
# retry runs before re-sending anything is free -- but the LOGIN endpoint
|
|
156
|
+
# is on the anti-abuse surface too, and the v2.0 incident escalated from
|
|
157
|
+
# write 401s into a login lockout (see
|
|
158
|
+
# `.planning/debug/resolved/select-asset-401-unauthorized.md`), so the
|
|
159
|
+
# re-login is charged like any other write rather than treated as free
|
|
160
|
+
# housekeeping.
|
|
161
|
+
RETRY_RELOGIN_REQUEST_COST = 1
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _is_http_401(exc: BaseException) -> bool:
|
|
165
|
+
"""True only when `exc` is an `httpx.HTTPStatusError` whose response
|
|
166
|
+
carries HTTP 401 -- the plain-401 form of a write's anti-abuse trip
|
|
167
|
+
(REL-01/REL-04, D-01/D-02). Any other status or exception type is not
|
|
168
|
+
eligible for the verify-then-retry path below."""
|
|
169
|
+
return isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 401
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _probe_landed(asset_probe, local_identifiers: list[str]) -> tuple[set, set, dict]:
|
|
173
|
+
"""Verify, per item, whether a local_identifier from a 401'd write chunk
|
|
174
|
+
actually landed server-side BEFORE the retry re-sends anything (REL-02,
|
|
175
|
+
D-01).
|
|
176
|
+
|
|
177
|
+
Classification is deliberately asymmetric (PITFALLS.md Pitfall 4): the
|
|
178
|
+
probe returning normally means the item is `landed`; the probe raising
|
|
179
|
+
an `httpx.HTTPStatusError` with status 404 means it is genuinely
|
|
180
|
+
`absent`; ANY other raised exception (a different status, a network
|
|
181
|
+
error, a malformed response) is `inconclusive` and is NEVER treated as
|
|
182
|
+
absent -- re-sending on an ambiguous probe is exactly how the 58 stuck
|
|
183
|
+
placeholder rows this phase is separately reconciling were created.
|
|
184
|
+
|
|
185
|
+
:param asset_probe: Callable `(local_identifier: str) -> Any` -- the
|
|
186
|
+
already-existing read endpoint (`AssetApi.get_asset_by_local_identifier`
|
|
187
|
+
by default). Its return value is discarded; only whether it raises
|
|
188
|
+
(and how) is examined.
|
|
189
|
+
:return: `(landed, absent, inconclusive)` -- `landed`/`absent` are sets
|
|
190
|
+
of local_identifier strings; `inconclusive` maps a local_identifier
|
|
191
|
+
to the `str()` of the exception that made its probe ambiguous.
|
|
192
|
+
"""
|
|
193
|
+
landed: set = set()
|
|
194
|
+
absent: set = set()
|
|
195
|
+
inconclusive: dict = {}
|
|
196
|
+
for local_identifier in local_identifiers:
|
|
197
|
+
try:
|
|
198
|
+
asset_probe(local_identifier)
|
|
199
|
+
landed.add(local_identifier)
|
|
200
|
+
except httpx.HTTPStatusError as e:
|
|
201
|
+
if e.response.status_code == 404:
|
|
202
|
+
absent.add(local_identifier)
|
|
203
|
+
else:
|
|
204
|
+
inconclusive[local_identifier] = str(e)
|
|
205
|
+
except Exception as e:
|
|
206
|
+
inconclusive[local_identifier] = str(e)
|
|
207
|
+
return landed, absent, inconclusive
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
class ConsecutiveWriteFailureError(AuraError):
|
|
211
|
+
"""Raised by `execute_plan` when `max_consecutive_failures` write items
|
|
212
|
+
fail in an unbroken run -- the signature of an account lockout / systemic
|
|
213
|
+
cut-off that did NOT announce itself with a 429/475 (`RateLimitError`),
|
|
214
|
+
e.g. the plain-HTTP-401 form of the Pushd anti-abuse trip seen in the
|
|
215
|
+
select-asset-401-unauthorized session's live regression.
|
|
216
|
+
|
|
217
|
+
Distinct from `RateLimitError` so the CLI surfaces a separate "run of
|
|
218
|
+
failures -- probable lockout, stop and investigate" message rather than the
|
|
219
|
+
throttle-specific Retry-After wording. Carries the run length (`count`), the
|
|
220
|
+
last per-item error string (`last_error`), and the partial `ExecutionResult`
|
|
221
|
+
accumulated before the abort (`result`) so the caller can report what
|
|
222
|
+
succeeded first.
|
|
223
|
+
"""
|
|
224
|
+
|
|
225
|
+
def __init__(self, count: int, last_error: str, result: "ExecutionResult"):
|
|
226
|
+
self.count = count
|
|
227
|
+
self.last_error = last_error
|
|
228
|
+
self.result = result
|
|
229
|
+
super().__init__(
|
|
230
|
+
f'Aborted after {count} consecutive write failures -- this is the '
|
|
231
|
+
f'signature of an account lockout or a systemic cut-off (the Pushd '
|
|
232
|
+
f'anti-abuse trip can surface as a run of plain HTTP 401s with no '
|
|
233
|
+
f'Retry-After), not isolated per-item errors. Stop and investigate '
|
|
234
|
+
f'before retrying; continued calls may extend a lockout. '
|
|
235
|
+
f'Last error: {last_error}'
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
@dataclass
|
|
240
|
+
class ScanResult:
|
|
241
|
+
local_hashes: dict[str, list[Path]]
|
|
242
|
+
skipped_non_image: int
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def scan_directory(root: Path) -> ScanResult:
|
|
246
|
+
"""Recursively walk `root` (D-01) and content-hash every eligible image
|
|
247
|
+
file, grouping local paths by base64-MD5 hash (D-05: byte-identical
|
|
248
|
+
files collapse to one hash key/logical want).
|
|
249
|
+
|
|
250
|
+
Non-eligible files (videos, dotfiles, arbitrary junk) are counted in
|
|
251
|
+
`skipped_non_image` and never error (D-02, D-03). `Path.rglob` does not
|
|
252
|
+
recurse into symlinked directories, and symlinks are excluded
|
|
253
|
+
explicitly below (in addition to the `is_file()` check) -- bounding
|
|
254
|
+
traversal to real files under the user's own directory (T-07-01,
|
|
255
|
+
WR-01: a symlink to a file directly inside the scanned root would
|
|
256
|
+
otherwise still be matched by `rglob('*')` and `is_file()` follows the
|
|
257
|
+
symlink, silently reading and hashing content from outside `root`).
|
|
258
|
+
|
|
259
|
+
Raises `NotADirectoryError` if `root` does not exist or is not a
|
|
260
|
+
directory (CR-01): `Path.rglob` silently yields nothing for a missing
|
|
261
|
+
or non-directory path, which would otherwise be indistinguishable from
|
|
262
|
+
a genuinely empty directory and produce a misleading "delete everything"
|
|
263
|
+
plan downstream.
|
|
264
|
+
"""
|
|
265
|
+
if not root.is_dir():
|
|
266
|
+
raise NotADirectoryError(f'{root} is not an existing directory')
|
|
267
|
+
|
|
268
|
+
local_hashes: dict[str, list[Path]] = {}
|
|
269
|
+
skipped_non_image = 0
|
|
270
|
+
|
|
271
|
+
for p in root.rglob('*'):
|
|
272
|
+
if p.is_symlink() or not p.is_file():
|
|
273
|
+
continue
|
|
274
|
+
|
|
275
|
+
if p.suffix.lower() not in ELIGIBLE_EXTENSIONS:
|
|
276
|
+
skipped_non_image += 1
|
|
277
|
+
continue
|
|
278
|
+
|
|
279
|
+
data = p.read_bytes()
|
|
280
|
+
h = get_md5(data)
|
|
281
|
+
local_hashes.setdefault(h, []).append(p)
|
|
282
|
+
|
|
283
|
+
return ScanResult(local_hashes, skipped_non_image)
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
@dataclass
|
|
287
|
+
class SyncPlan:
|
|
288
|
+
to_upload: list[Path] = field(default_factory=list)
|
|
289
|
+
# Mode-agnostic REMOVAL-CANDIDATE list (D-07): frame assets no longer
|
|
290
|
+
# wanted locally. It deliberately keeps the name `to_delete` -- which
|
|
291
|
+
# primitive actually acts on it (hide vs. hard delete) is the executor's
|
|
292
|
+
# choice, not the diff's.
|
|
293
|
+
to_delete: list = field(default_factory=list)
|
|
294
|
+
# Frame assets that ARE wanted locally but are currently hidden on the
|
|
295
|
+
# frame -- the re-show candidates (D-05).
|
|
296
|
+
to_reshow: list = field(default_factory=list)
|
|
297
|
+
unchanged: int = 0
|
|
298
|
+
# Frame assets no longer wanted locally that are ALREADY hidden: nothing
|
|
299
|
+
# left to do, so they must never re-enter to_delete on later runs (D-06).
|
|
300
|
+
already_hidden: int = 0
|
|
301
|
+
skipped_non_image: int = 0
|
|
302
|
+
frame_no_hash: int = 0
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def compute_plan(local_hashes: dict[str, list[Path]], frame_assets: list, skipped_non_image: int = 0) -> SyncPlan:
|
|
306
|
+
"""Diff local content hashes against a frame's assets (SYNC-01).
|
|
307
|
+
|
|
308
|
+
Pure function -- no I/O, no network, no mutation of its inputs --
|
|
309
|
+
mirroring `resolve_frame`'s pure dataclass-result shape. There is
|
|
310
|
+
deliberately no execute/mutating counterpart here; the dry-run
|
|
311
|
+
guarantee is structural.
|
|
312
|
+
|
|
313
|
+
Local duplicates (per `scan_directory`'s dedup) already collapse to one
|
|
314
|
+
logical want per hash (D-05), so each unique hash demands exactly one
|
|
315
|
+
frame copy. Frame-side assets are matched count-for-count against that
|
|
316
|
+
demand (D-06 multiset asymmetry): surplus frame copies beyond local
|
|
317
|
+
demand become delete candidates, they are NOT deduped as a group.
|
|
318
|
+
Hashless frame assets (e.g. videos) are excluded from both unchanged
|
|
319
|
+
and delete, and counted separately in `frame_no_hash`.
|
|
320
|
+
|
|
321
|
+
Every hash-bearing frame asset is classified two ways at once -- whether
|
|
322
|
+
the local directory still wants it, and whether it is currently visible on
|
|
323
|
+
the frame (`asset.selected`, which `FrameApi.get_assets` joins from
|
|
324
|
+
`asset_settings` so it means THIS FRAME's visibility):
|
|
325
|
+
|
|
326
|
+
| local | visible | outcome |
|
|
327
|
+
|-----------|---------|------------------------------------------|
|
|
328
|
+
| present | hidden | `to_reshow` -- bring it back (D-05) |
|
|
329
|
+
| present | visible | `unchanged` |
|
|
330
|
+
| gone | visible | `to_delete` -- removal candidate |
|
|
331
|
+
| gone | hidden | `already_hidden` -- no-op (D-06) |
|
|
332
|
+
|
|
333
|
+
A hidden asset consumes local demand exactly like a visible one, so a
|
|
334
|
+
photo that is merely hidden is re-shown rather than uploaded a second
|
|
335
|
+
time (D-06).
|
|
336
|
+
|
|
337
|
+
This function stays PURE and MODE-AGNOSTIC: it never consults the removal
|
|
338
|
+
mode. It reports what each asset's state *is*; choosing hide-vs-delete for
|
|
339
|
+
`to_delete` belongs to the executor.
|
|
340
|
+
"""
|
|
341
|
+
demand = {h: 1 for h in local_hashes}
|
|
342
|
+
to_delete: list = []
|
|
343
|
+
to_reshow: list = []
|
|
344
|
+
unchanged = 0
|
|
345
|
+
already_hidden = 0
|
|
346
|
+
frame_no_hash = 0
|
|
347
|
+
|
|
348
|
+
for asset in frame_assets:
|
|
349
|
+
if not asset.md5_hash:
|
|
350
|
+
frame_no_hash += 1
|
|
351
|
+
continue
|
|
352
|
+
|
|
353
|
+
if demand.get(asset.md5_hash, 0) > 0:
|
|
354
|
+
# Wanted locally. Consume the demand either way (D-06 dedup) --
|
|
355
|
+
# a hidden copy still counts as present, so it is never
|
|
356
|
+
# re-uploaded, only re-shown.
|
|
357
|
+
demand[asset.md5_hash] -= 1
|
|
358
|
+
if asset.selected:
|
|
359
|
+
unchanged += 1
|
|
360
|
+
else:
|
|
361
|
+
to_reshow.append(asset)
|
|
362
|
+
else:
|
|
363
|
+
# No longer wanted locally. Already hidden means there is nothing
|
|
364
|
+
# left to do; re-listing it would re-issue the same hide forever.
|
|
365
|
+
if asset.selected:
|
|
366
|
+
to_delete.append(asset)
|
|
367
|
+
else:
|
|
368
|
+
already_hidden += 1
|
|
369
|
+
|
|
370
|
+
to_upload = [local_hashes[h][0] for h, remaining in demand.items() if remaining > 0]
|
|
371
|
+
|
|
372
|
+
return SyncPlan(
|
|
373
|
+
to_upload=to_upload,
|
|
374
|
+
to_delete=to_delete,
|
|
375
|
+
to_reshow=to_reshow,
|
|
376
|
+
unchanged=unchanged,
|
|
377
|
+
already_hidden=already_hidden,
|
|
378
|
+
skipped_non_image=skipped_non_image,
|
|
379
|
+
frame_no_hash=frame_no_hash,
|
|
380
|
+
)
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
@dataclass
|
|
384
|
+
class ExecutionResult:
|
|
385
|
+
upload_succeeded: int = 0
|
|
386
|
+
delete_succeeded: int = 0
|
|
387
|
+
reshow_succeeded: int = 0
|
|
388
|
+
upload_failures: list = field(default_factory=list) # list[tuple[Path, str]]
|
|
389
|
+
delete_failures: list = field(default_factory=list) # list[tuple[str, str]]
|
|
390
|
+
reshow_failures: list = field(default_factory=list) # list[tuple[str, str]]
|
|
391
|
+
# REL-01/REL-03/D-08 visibility counters for the 401 verify-then-retry
|
|
392
|
+
# path -- both default to 0 so every existing caller/test is unaffected.
|
|
393
|
+
chunks_retried: int = 0
|
|
394
|
+
items_already_landed: int = 0
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
# The three tiers of "this photo is no longer wanted locally" (D-01/D-03).
|
|
398
|
+
# execute_plan applies exactly ONE of these per run, chosen by `removal_mode`,
|
|
399
|
+
# so the destructive primitive is unreachable unless a caller names it:
|
|
400
|
+
#
|
|
401
|
+
# hide exclude_asset reversible, non-destructive -- the DEFAULT
|
|
402
|
+
# delete remove_asset disassociates from this frame only
|
|
403
|
+
# hard_delete delete_asset irreversible, destroys the asset account-wide
|
|
404
|
+
#
|
|
405
|
+
# `hide` and `delete` are native batch endpoints (one request per chunk);
|
|
406
|
+
# `hard_delete` is asset-scoped with no batch form, hence the per-asset loop
|
|
407
|
+
# -- which is also why it is charged the budget per asset (Pitfall 3).
|
|
408
|
+
_REMOVAL_PRIMITIVE = {
|
|
409
|
+
'hide': lambda aura, frame_id, assets: aura.frame_api.exclude_asset(
|
|
410
|
+
frame_id, [AssetPartialId(id=asset.id) for asset in assets]),
|
|
411
|
+
'delete': lambda aura, frame_id, assets: aura.frame_api.remove_asset(
|
|
412
|
+
frame_id, [AssetPartialId(id=asset.id) for asset in assets]),
|
|
413
|
+
'hard_delete': lambda aura, frame_id, assets: [
|
|
414
|
+
aura.asset_api.delete_asset(asset) for asset in assets],
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
# Requests a removal chunk actually costs, per mode -- batch endpoints are one
|
|
418
|
+
# call regardless of chunk size; hard_delete is one call per asset.
|
|
419
|
+
_REMOVAL_REQUEST_COST = {
|
|
420
|
+
'hide': lambda chunk: 1,
|
|
421
|
+
'delete': lambda chunk: 1,
|
|
422
|
+
'hard_delete': lambda chunk: len(chunk),
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _chunked(items: list, size: int):
|
|
427
|
+
"""Yield successive `size`-length slices of `items` (the final slice may
|
|
428
|
+
be shorter). `size` is expected to be a positive int (`WRITE_BATCH_SIZE`
|
|
429
|
+
or an injected override); a chunk boundary is where the Pushd write-call
|
|
430
|
+
count collapses from ~3N to ~2 per chunk."""
|
|
431
|
+
for i in range(0, len(items), size):
|
|
432
|
+
yield items[i:i + size]
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
def _prep_upload(path: Path, s3_client) -> AssetPartial:
|
|
436
|
+
"""Per-file S3 prep phase for a single new local file: decode the image
|
|
437
|
+
once to read both its dimensions and its real format (D-11), resolve
|
|
438
|
+
the Apple UTI from that decoded format -- not the filename -- upload
|
|
439
|
+
the raw bytes to S3, and build the `AssetPartial` that will be sent in
|
|
440
|
+
the chunk's batched `batch_update` call. Because the UTI comes from the
|
|
441
|
+
decoded bytes rather than the extension, a `.jpg` that is really a PNG
|
|
442
|
+
can no longer be mislabeled server-side; the bytes decide. Raises
|
|
443
|
+
(fails closed) if the decoded format is unmapped, `Image.open` fails,
|
|
444
|
+
or the S3 upload fails -- the caller catches this per file so one bad
|
|
445
|
+
file never blocks the rest of the chunk.
|
|
446
|
+
"""
|
|
447
|
+
local_identifier = str(uuid.uuid4())
|
|
448
|
+
with Image.open(path) as image:
|
|
449
|
+
width, height = image.size
|
|
450
|
+
data_uti = _DATA_UTI_BY_IMAGE_FORMAT.get(image.format)
|
|
451
|
+
if data_uti is None:
|
|
452
|
+
raise ValueError(f'Unsupported image format: {image.format} ({path.name})')
|
|
453
|
+
|
|
454
|
+
filename, md5 = s3_client.upload_file(path.read_bytes(), path.suffix)
|
|
455
|
+
|
|
456
|
+
return local_identifier, AssetPartial(
|
|
457
|
+
local_identifier=local_identifier,
|
|
458
|
+
file_name=filename,
|
|
459
|
+
md5_hash=md5,
|
|
460
|
+
height=height,
|
|
461
|
+
width=width,
|
|
462
|
+
taken_at=format_dt_to_aura(get_utc_now()),
|
|
463
|
+
data_uti=data_uti,
|
|
464
|
+
selected=True,
|
|
465
|
+
upload_priority=0,
|
|
466
|
+
)
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def execute_plan(plan: SyncPlan, aura, frame_id: str, *, s3_client, sqs_client,
|
|
470
|
+
throttle_seconds: float = WRITE_THROTTLE_SECONDS, sleep=time.sleep,
|
|
471
|
+
max_consecutive_failures: int = MAX_CONSECUTIVE_WRITE_FAILURES,
|
|
472
|
+
progress=lambda *args: None,
|
|
473
|
+
batch_size: int = WRITE_BATCH_SIZE,
|
|
474
|
+
chunk_delay_seconds: float = WRITE_CHUNK_DELAY_SECONDS,
|
|
475
|
+
on_wait=lambda *args: None,
|
|
476
|
+
budget=None, geo_check=None, wait_on_budget: bool = True,
|
|
477
|
+
max_wait_seconds: float = 3600.0, clock=get_utc_now,
|
|
478
|
+
removal_mode: str = 'hide',
|
|
479
|
+
relogin=None, asset_probe=None,
|
|
480
|
+
retry_on_auth_401: bool = True) -> ExecutionResult:
|
|
481
|
+
"""Execute a `SyncPlan` against a live frame -- the module's only
|
|
482
|
+
mutating entry point (D-06/D-08/D-09/D-10), batched (quick task
|
|
483
|
+
260708-fyr) to collapse ~3N Pushd write calls to ~2 per chunk.
|
|
484
|
+
|
|
485
|
+
`plan.to_upload` is processed in sorted-path chunks of up to
|
|
486
|
+
`batch_size`: each chunk performs per-file S3 uploads (S3 is AWS, not
|
|
487
|
+
the Pushd anti-abuse surface, so those stay per-file and precede the
|
|
488
|
+
chunk's batched writes), then exactly ONE `select_asset` call and ONE
|
|
489
|
+
`batch_update` call carrying every successfully-prepped file in the
|
|
490
|
+
chunk. Per-file attribution is recovered from `batch_update`'s
|
|
491
|
+
`successes[].local_identifier`: a prepped file's local_identifier
|
|
492
|
+
absent from `successes` is recorded as a per-file failure. `plan.to_delete`
|
|
493
|
+
is processed in chunks the same way, issuing one `remove_asset` call per
|
|
494
|
+
chunk; since `remove_asset` returns only a failure COUNT (never
|
|
495
|
+
per-item), a chunk-level failure attributes ALL deletes in that chunk
|
|
496
|
+
(coarser than upload attribution -- documented tradeoff, T-fyr-01). All
|
|
497
|
+
uploads are attempted before any delete is attempted (D-09) -- on
|
|
498
|
+
interruption mid-run this leaves the safer partial state (content added,
|
|
499
|
+
nothing removed).
|
|
500
|
+
|
|
501
|
+
Work runs least-destructive-first: uploads, then re-shows, then removals.
|
|
502
|
+
The re-show loop calls `select_asset` on `plan.to_reshow` and ALWAYS runs,
|
|
503
|
+
whatever `removal_mode` is -- bringing back a photo the user restored
|
|
504
|
+
locally is not a removal concern (D-05).
|
|
505
|
+
|
|
506
|
+
The removal loop applies exactly ONE primitive, chosen by `removal_mode`
|
|
507
|
+
from `_REMOVAL_PRIMITIVE` (D-01/D-03). It defaults to `'hide'`, so the
|
|
508
|
+
destructive tiers are unreachable unless a caller names one.
|
|
509
|
+
|
|
510
|
+
`s3_client`/`sqs_client` are injected by the caller (never constructed
|
|
511
|
+
in this module) so this function is offline-testable with fakes --
|
|
512
|
+
no AWS client construction call appears here at all.
|
|
513
|
+
|
|
514
|
+
:param removal_mode: Which primitive acts on `plan.to_delete` -- `'hide'`
|
|
515
|
+
(default, reversible `exclude_asset`), `'delete'` (`remove_asset`,
|
|
516
|
+
frame-scoped) or `'hard_delete'` (`delete_asset`, irreversible and
|
|
517
|
+
account-wide). Preservation is the default because a mistaken sync
|
|
518
|
+
must never destroy photos (D-01/D-03).
|
|
519
|
+
:param plan: The `SyncPlan` (from `compute_plan`) to execute.
|
|
520
|
+
:param aura: An authenticated `Aura` instance.
|
|
521
|
+
:param frame_id: The frame to upload to / delete from.
|
|
522
|
+
:param s3_client: An object providing `upload_file(data, extension) -> (filename, md5)`.
|
|
523
|
+
:param sqs_client: An object providing `get_queue_url(frame_id)` and `receive_message(...)`.
|
|
524
|
+
:param throttle_seconds: Seconds to pause before each write network call
|
|
525
|
+
so a bulk apply is paced rather than fired as a burst -- a second,
|
|
526
|
+
independent layer of defense on top of batching itself (root-cause
|
|
527
|
+
mitigation for the select-asset-401-unauthorized session). 0 disables.
|
|
528
|
+
:param sleep: The sleep function to call (injectable for offline tests
|
|
529
|
+
so they pace-check without real delays).
|
|
530
|
+
:param max_consecutive_failures: Abort the whole batch once this many
|
|
531
|
+
write items fail in an unbroken run (reset on any success) -- the
|
|
532
|
+
backstop for an anti-abuse trip that surfaces as plain HTTP 401s
|
|
533
|
+
rather than the 429/475 `RateLimitError` catches (see
|
|
534
|
+
`MAX_CONSECUTIVE_WRITE_FAILURES`). Status-code agnostic: any caught
|
|
535
|
+
per-item write failure counts. 0 disables the backstop entirely
|
|
536
|
+
(restoring the pure unbounded per-item D-08 behaviour).
|
|
537
|
+
:param progress: Optional reporter called exactly once per RESOLVED
|
|
538
|
+
item -- an upload from `plan.to_upload` or a delete from
|
|
539
|
+
`plan.to_delete` -- as `progress(kind, identifier, ok)`, where
|
|
540
|
+
`kind` is `'upload'` or `'delete'`, `identifier` is the `Path` for
|
|
541
|
+
an upload or the asset id string for a delete, and `ok` is True on
|
|
542
|
+
success / False on a caught per-item failure. Deliberately NOT
|
|
543
|
+
called on the `RateLimitError` abort path below -- that item never
|
|
544
|
+
resolves, the whole batch stops there, so the number of reporter
|
|
545
|
+
calls always equals the number of attempted-and-resolved items.
|
|
546
|
+
Defaults to a no-op so offline tests and any caller that doesn't
|
|
547
|
+
care about live feedback are unaffected.
|
|
548
|
+
:param batch_size: Maximum number of items batched into a single
|
|
549
|
+
select_asset/batch_update/remove_asset call (see
|
|
550
|
+
`WRITE_BATCH_SIZE`). Injectable so offline tests can exercise the
|
|
551
|
+
chunk-boundary logic with a small size.
|
|
552
|
+
:param chunk_delay_seconds: Seconds to pause BETWEEN write chunks (see
|
|
553
|
+
`WRITE_CHUNK_DELAY_SECONDS`) -- distinct from `throttle_seconds`
|
|
554
|
+
(which paces individual calls). Skipped before the first write chunk
|
|
555
|
+
of the run; applied before every subsequent chunk (uploads then
|
|
556
|
+
deletes are paced as one sequence). Realized via `sleep` in 1s steps.
|
|
557
|
+
0 disables. Injectable for offline tests.
|
|
558
|
+
:param on_wait: Optional callback invoked once per second during an
|
|
559
|
+
inter-chunk pause as `on_wait(remaining_seconds)`, so a CLI can render
|
|
560
|
+
a live countdown instead of a frozen bar. Defaults to a no-op.
|
|
561
|
+
:param budget: Optional `pushframe.ratelimit.WriteBudget` (Phase 09,
|
|
562
|
+
ANTI-03/04) gating each write chunk with a client-side token-bucket
|
|
563
|
+
request budget -- a proactive defense making the anti-abuse
|
|
564
|
+
write-lockout structurally hard to hit. When `None` (the default),
|
|
565
|
+
every budget-related touch point below is skipped entirely: this is
|
|
566
|
+
a true byte-for-byte no-op, matching this function's pre-Phase-09
|
|
567
|
+
behavior exactly.
|
|
568
|
+
:param geo_check: Optional zero-arg callable (Phase 09, ANTI-03) invoked
|
|
569
|
+
exactly once, before any write (including for a delete-only plan),
|
|
570
|
+
raising `pushframe.ratelimit.GeoMismatchError` on a VPN/exit-IP
|
|
571
|
+
country mismatch. When `None` (the default), skipped entirely.
|
|
572
|
+
:param wait_on_budget: Forwarded to `budget.acquire(..., wait=...)` --
|
|
573
|
+
when True (the default), a chunk waits for enough tokens to refill
|
|
574
|
+
rather than raising `BudgetExhausted` immediately. Ignored when
|
|
575
|
+
`budget` is `None`.
|
|
576
|
+
:param max_wait_seconds: Forwarded to `budget.acquire(..., max_wait=...)`
|
|
577
|
+
-- the longest a chunk will wait for tokens before raising
|
|
578
|
+
`BudgetExhausted` anyway (default 3600s/1hr). Ignored when `budget`
|
|
579
|
+
is `None`.
|
|
580
|
+
:param clock: Zero-arg function returning the current instant (Phase 09),
|
|
581
|
+
injected so tests can control time deterministically -- mirrors the
|
|
582
|
+
`sleep` seam. Its return VALUE is passed as `budget.acquire(...,
|
|
583
|
+
now=clock())` and `budget.reconcile_tripped(clock())`; defaults to
|
|
584
|
+
`pushframe.utils.dt.get_utc_now`. Only ever called when `budget` is
|
|
585
|
+
not `None` (or `geo_check`, which never touches `clock`).
|
|
586
|
+
:param relogin: Zero-arg callable invoked at most ONCE per run (REL-01/
|
|
587
|
+
REL-04, D-02/D-04) when a write chunk's first attempt returns HTTP
|
|
588
|
+
401 -- the re-login is this function's auth-vs-anti-abuse
|
|
589
|
+
discriminator: if it raises, the 401 is a genuine authentication
|
|
590
|
+
failure (`AuthenticationError` propagates, the run aborts); if it
|
|
591
|
+
succeeds, the original 401 was the anti-abuse trip, not auth, and
|
|
592
|
+
the chunk's items are verified then selectively re-sent. Defaults
|
|
593
|
+
to `aura.login` when `None`, so this is live behaviour by default
|
|
594
|
+
and an injectable seam for offline tests.
|
|
595
|
+
:param asset_probe: Callable `(local_identifier: str) -> Any` used by
|
|
596
|
+
the retry's verify step (REL-02, D-01) to check whether an item
|
|
597
|
+
already landed before re-sending it. Defaults to
|
|
598
|
+
`aura.asset_api.get_asset_by_local_identifier` when `None`. Never
|
|
599
|
+
charged against `budget` (D-06) -- it is a read.
|
|
600
|
+
:param retry_on_auth_401: Kill switch (T-11-03) for the whole 401
|
|
601
|
+
verify-then-retry path. `True` (the default) enables it; `False`
|
|
602
|
+
restores the pre-Phase-11 behaviour where a plain 401 falls straight
|
|
603
|
+
through to the generic per-chunk attribution below.
|
|
604
|
+
:return: An `ExecutionResult` with separated upload/delete success counts and named failures.
|
|
605
|
+
|
|
606
|
+
Raises `RateLimitError` (from the client layer) WITHOUT catching it:
|
|
607
|
+
a 429/475 throttle or lockout aborts the entire batch immediately
|
|
608
|
+
rather than being recorded as one of N per-item failures -- the whole
|
|
609
|
+
point being to stop hammering a throttling server and surface a single
|
|
610
|
+
"back off" message instead of dozens of confusing per-item errors.
|
|
611
|
+
Ordinary per-item/per-chunk failures are still caught and recorded (D-08).
|
|
612
|
+
|
|
613
|
+
Raises `ConsecutiveWriteFailureError` when `max_consecutive_failures`
|
|
614
|
+
write items fail in an unbroken run -- the backstop for an anti-abuse
|
|
615
|
+
trip that surfaces as plain HTTP 401s (indistinguishable at the status
|
|
616
|
+
level from an isolated per-item auth failure) rather than the 429/475
|
|
617
|
+
`RateLimitError` catches. The counter spans BOTH loops and resets on any
|
|
618
|
+
success, so an isolated failure never trips it. The final failing item is
|
|
619
|
+
still recorded in `result` and reported via `progress` before the abort
|
|
620
|
+
(so the reporter-call count still equals the attempted-and-resolved item
|
|
621
|
+
count); items after the abort are never attempted.
|
|
622
|
+
|
|
623
|
+
Raises `AuthenticationError` (REL-04, D-02) when a write chunk's first
|
|
624
|
+
attempt returns HTTP 401 and the resulting re-login itself fails --
|
|
625
|
+
a hard stop, never retried and never re-labelled transient. Nothing in
|
|
626
|
+
the chunk is attributed as a per-item failure in this case; the run
|
|
627
|
+
simply aborts, same as `RateLimitError`.
|
|
628
|
+
"""
|
|
629
|
+
# Geo pre-flight (Phase 09, ANTI-03) -- the VERY FIRST executable
|
|
630
|
+
# statement of the body, before ExecutionResult is even constructed, so
|
|
631
|
+
# a delete-only plan is gated too: no S3 upload, no select_asset, no
|
|
632
|
+
# remove_asset call has happened yet at this point regardless of the
|
|
633
|
+
# plan's shape.
|
|
634
|
+
if geo_check is not None:
|
|
635
|
+
geo_check()
|
|
636
|
+
|
|
637
|
+
# REL-01/D-01: resolve the injectable seams to their live defaults here
|
|
638
|
+
# (not in the signature) so the defaults are live behaviour while the
|
|
639
|
+
# seams stay injectable for offline tests -- the same pattern every
|
|
640
|
+
# other cross-cutting concern in this function uses.
|
|
641
|
+
if relogin is None:
|
|
642
|
+
relogin = aura.login
|
|
643
|
+
if asset_probe is None:
|
|
644
|
+
asset_probe = aura.asset_api.get_asset_by_local_identifier
|
|
645
|
+
|
|
646
|
+
result = ExecutionResult()
|
|
647
|
+
consecutive_failures = 0
|
|
648
|
+
# D-04: at most ONE re-login for the whole run, shared by all three
|
|
649
|
+
# write loops -- no re-login storm. Set the first (and only) time a
|
|
650
|
+
# chunk's 401 retry successfully re-authenticates.
|
|
651
|
+
relogin_done = False
|
|
652
|
+
|
|
653
|
+
def throttle() -> None:
|
|
654
|
+
if throttle_seconds > 0:
|
|
655
|
+
sleep(throttle_seconds)
|
|
656
|
+
|
|
657
|
+
def note_failure(last_error: str) -> None:
|
|
658
|
+
# Bump the shared consecutive-failure run and abort the whole batch
|
|
659
|
+
# once it crosses the threshold. Status-code agnostic on purpose: the
|
|
660
|
+
# trip cannot be recognised from the HTTP status alone (this session's
|
|
661
|
+
# core lesson), so ANY caught per-item write failure counts toward the
|
|
662
|
+
# run. RateLimitError never reaches here -- it is re-raised above and
|
|
663
|
+
# aborts on its first occurrence.
|
|
664
|
+
nonlocal consecutive_failures
|
|
665
|
+
consecutive_failures += 1
|
|
666
|
+
if 0 < max_consecutive_failures <= consecutive_failures:
|
|
667
|
+
if budget is not None:
|
|
668
|
+
# Phase 09 ANTI-04: a systemic cut-off just tripped (the
|
|
669
|
+
# backstop for the plain-401 form of the anti-abuse lockout
|
|
670
|
+
# that RateLimitError's 429/475 branch can't catch) -- force
|
|
671
|
+
# the local budget to reflect reality even though it wasn't
|
|
672
|
+
# what ran the bucket dry.
|
|
673
|
+
budget.reconcile_tripped(clock())
|
|
674
|
+
budget.save()
|
|
675
|
+
raise ConsecutiveWriteFailureError(consecutive_failures, last_error, result)
|
|
676
|
+
|
|
677
|
+
first_write_chunk = True
|
|
678
|
+
|
|
679
|
+
def interchunk_pause() -> None:
|
|
680
|
+
# Human-pacing pause BETWEEN write chunks -- separate from throttle()
|
|
681
|
+
# (which paces individual network calls). Skipped before the very
|
|
682
|
+
# first write chunk of the run, so uploads then deletes are spaced as
|
|
683
|
+
# one continuous sequence. Realized via the injected `sleep` in 1s
|
|
684
|
+
# steps, calling on_wait(remaining) each second so the CLI can show a
|
|
685
|
+
# live countdown rather than a frozen bar. chunk_delay_seconds=0
|
|
686
|
+
# disables the pause entirely.
|
|
687
|
+
nonlocal first_write_chunk
|
|
688
|
+
if first_write_chunk:
|
|
689
|
+
first_write_chunk = False
|
|
690
|
+
return
|
|
691
|
+
remaining = chunk_delay_seconds
|
|
692
|
+
while remaining > 0:
|
|
693
|
+
on_wait(remaining)
|
|
694
|
+
step = 1.0 if remaining >= 1.0 else remaining
|
|
695
|
+
sleep(step)
|
|
696
|
+
remaining -= step
|
|
697
|
+
|
|
698
|
+
queue_url = sqs_client.get_queue_url(frame_id) if plan.to_upload else None
|
|
699
|
+
|
|
700
|
+
for chunk in _chunked(sorted(plan.to_upload), batch_size):
|
|
701
|
+
interchunk_pause()
|
|
702
|
+
if budget is not None:
|
|
703
|
+
# 2 requests per upload chunk (select_asset + batch_update) --
|
|
704
|
+
# acquired BEFORE the per-file S3-prep loop so a BudgetExhausted
|
|
705
|
+
# stop never wastes an S3 upload on a chunk that won't be written.
|
|
706
|
+
budget.acquire(2, wait=wait_on_budget, max_wait=max_wait_seconds,
|
|
707
|
+
now=clock(), sleep=sleep, on_wait=on_wait)
|
|
708
|
+
prepped: list = [] # list[tuple[Path, local_identifier, AssetPartial]]
|
|
709
|
+
for path in chunk:
|
|
710
|
+
try:
|
|
711
|
+
local_identifier, partial = _prep_upload(path, s3_client)
|
|
712
|
+
prepped.append((path, local_identifier, partial))
|
|
713
|
+
except Exception as e:
|
|
714
|
+
result.upload_failures.append((path, str(e)))
|
|
715
|
+
progress('upload', path, False)
|
|
716
|
+
note_failure(str(e))
|
|
717
|
+
|
|
718
|
+
if not prepped:
|
|
719
|
+
continue
|
|
720
|
+
|
|
721
|
+
chunk_retried = False # D-04: at most one verify-and-retry per chunk
|
|
722
|
+
try:
|
|
723
|
+
try:
|
|
724
|
+
throttle()
|
|
725
|
+
aura.frame_api.select_asset(
|
|
726
|
+
frame_id, [AssetPartialId(local_identifier=lid) for (_, lid, _) in prepped]
|
|
727
|
+
)
|
|
728
|
+
# Best-effort/observational poll only -- never gates success or
|
|
729
|
+
# failure (Pitfall 3); at most once per chunk, not per file.
|
|
730
|
+
message = sqs_client.receive_message(queue_url, wait_time_seconds=5)
|
|
731
|
+
logger.debug(f'Best-effort SQS poll after chunk select_asset: {message}')
|
|
732
|
+
|
|
733
|
+
throttle()
|
|
734
|
+
batch_result = aura.asset_api.batch_update([partial for (_, _, partial) in prepped])
|
|
735
|
+
succeeded = {s.local_identifier for s in batch_result.successes}
|
|
736
|
+
|
|
737
|
+
for path, local_identifier, _ in prepped:
|
|
738
|
+
if local_identifier in succeeded:
|
|
739
|
+
result.upload_succeeded += 1
|
|
740
|
+
consecutive_failures = 0
|
|
741
|
+
progress('upload', path, True)
|
|
742
|
+
else:
|
|
743
|
+
reason = 'file not acknowledged in batch_update successes'
|
|
744
|
+
result.upload_failures.append((path, reason))
|
|
745
|
+
progress('upload', path, False)
|
|
746
|
+
note_failure(reason)
|
|
747
|
+
except httpx.HTTPStatusError as e:
|
|
748
|
+
# REL-01/REL-02/REL-04, D-01/D-02/D-04: a plain HTTP 401 on
|
|
749
|
+
# this chunk's write(s). Re-raise (falling through to the
|
|
750
|
+
# generic Pushd-failure branch below, unchanged) whenever
|
|
751
|
+
# this branch is not the tool for the job -- the feature is
|
|
752
|
+
# disabled, the status isn't 401, or this chunk already used
|
|
753
|
+
# its one retry.
|
|
754
|
+
if not retry_on_auth_401 or not _is_http_401(e) or chunk_retried:
|
|
755
|
+
raise
|
|
756
|
+
chunk_retried = True
|
|
757
|
+
result.chunks_retried += 1
|
|
758
|
+
|
|
759
|
+
if not relogin_done:
|
|
760
|
+
# This run's ONE shared re-login -- the auth-vs-anti-abuse
|
|
761
|
+
# discriminator (D-02). A failure here is a genuine
|
|
762
|
+
# authentication failure: hard stop, never retried.
|
|
763
|
+
if budget is not None:
|
|
764
|
+
# D-06: the re-login is charged (RETRY_RELOGIN_REQUEST_COST,
|
|
765
|
+
# see its own rationale comment above) even though the
|
|
766
|
+
# verify probe below is free -- same acquire() shape
|
|
767
|
+
# every other write already uses (D-07), so
|
|
768
|
+
# --no-wait/--max-wait/--ignore-budget govern this
|
|
769
|
+
# exactly like a first attempt, with no bespoke path.
|
|
770
|
+
budget.acquire(RETRY_RELOGIN_REQUEST_COST, wait=wait_on_budget,
|
|
771
|
+
max_wait=max_wait_seconds, now=clock(),
|
|
772
|
+
sleep=sleep, on_wait=on_wait)
|
|
773
|
+
try:
|
|
774
|
+
relogin()
|
|
775
|
+
except Exception as relogin_exc:
|
|
776
|
+
raise AuthenticationError(relogin_exc) from relogin_exc
|
|
777
|
+
relogin_done = True
|
|
778
|
+
else:
|
|
779
|
+
# D-02: this run's ONE re-login already succeeded (on an
|
|
780
|
+
# earlier chunk) -- a further 401 here is NOT
|
|
781
|
+
# authentication, it is the anti-abuse trip. No second
|
|
782
|
+
# re-login, no verify probe, no resend -- attribute every
|
|
783
|
+
# prepped item as a per-file failure exactly like the
|
|
784
|
+
# generic branch, so ConsecutiveWriteFailureError's
|
|
785
|
+
# backstop still applies.
|
|
786
|
+
for path, _, _ in prepped:
|
|
787
|
+
reason = ('HTTP 401 after a successful re-login -- classified as an '
|
|
788
|
+
'anti-abuse trip, not authentication')
|
|
789
|
+
result.upload_failures.append((path, reason))
|
|
790
|
+
progress('upload', path, False)
|
|
791
|
+
note_failure(reason)
|
|
792
|
+
if budget is not None:
|
|
793
|
+
budget.save()
|
|
794
|
+
continue
|
|
795
|
+
|
|
796
|
+
# Re-login just succeeded -- verify every prepped item BEFORE
|
|
797
|
+
# re-sending anything (REL-02, D-01): an item the probe
|
|
798
|
+
# confirms already landed must never be re-sent. D-06: this
|
|
799
|
+
# probe is NEVER charged against `budget` -- reads
|
|
800
|
+
# demonstrably kept working throughout the v2.0 write
|
|
801
|
+
# lockouts, and charging them would make a 50-item chunk's
|
|
802
|
+
# verification cost 50 tokens, which would defeat D-01's
|
|
803
|
+
# whole point (a verify step so expensive nobody enables it).
|
|
804
|
+
by_lid = {lid: path for (path, lid, _) in prepped}
|
|
805
|
+
landed, absent, inconclusive = _probe_landed(asset_probe, list(by_lid))
|
|
806
|
+
|
|
807
|
+
for lid in landed:
|
|
808
|
+
result.items_already_landed += 1
|
|
809
|
+
result.upload_succeeded += 1
|
|
810
|
+
consecutive_failures = 0
|
|
811
|
+
progress('upload', by_lid[lid], True)
|
|
812
|
+
for lid, probe_error in inconclusive.items():
|
|
813
|
+
reason = f'verify probe after re-login was inconclusive: {probe_error}'
|
|
814
|
+
result.upload_failures.append((by_lid[lid], reason))
|
|
815
|
+
progress('upload', by_lid[lid], False)
|
|
816
|
+
note_failure(reason)
|
|
817
|
+
|
|
818
|
+
if absent:
|
|
819
|
+
# Re-send ONLY the genuinely-absent subset, iterating
|
|
820
|
+
# `prepped` in its existing (sorted-path) order so the
|
|
821
|
+
# re-sent list's attribution lines up with what the plan
|
|
822
|
+
# printed.
|
|
823
|
+
resend = [item for item in prepped if item[1] in absent]
|
|
824
|
+
if budget is not None:
|
|
825
|
+
# D-05: a retry is charged at FULL first-attempt cost
|
|
826
|
+
# (2 -- select_asset + batch_update), not a discount.
|
|
827
|
+
# The requests really were sent and really did hit
|
|
828
|
+
# the anti-abuse surface, and Phase 9 already
|
|
829
|
+
# observed that failed attempts appear to consume
|
|
830
|
+
# server-side budget too -- charging the retry keeps
|
|
831
|
+
# the local estimate honest at the price of a
|
|
832
|
+
# shorter effective run. Same acquire() call shape
|
|
833
|
+
# as the first attempt (D-07): no bespoke charge
|
|
834
|
+
# path, no retry-specific wait policy.
|
|
835
|
+
budget.acquire(2, wait=wait_on_budget, max_wait=max_wait_seconds,
|
|
836
|
+
now=clock(), sleep=sleep, on_wait=on_wait)
|
|
837
|
+
try:
|
|
838
|
+
throttle()
|
|
839
|
+
aura.frame_api.select_asset(
|
|
840
|
+
frame_id, [AssetPartialId(local_identifier=lid) for (_, lid, _) in resend]
|
|
841
|
+
)
|
|
842
|
+
throttle()
|
|
843
|
+
resend_batch_result = aura.asset_api.batch_update(
|
|
844
|
+
[partial for (_, _, partial) in resend])
|
|
845
|
+
resend_succeeded = {s.local_identifier for s in resend_batch_result.successes}
|
|
846
|
+
for path, lid, _ in resend:
|
|
847
|
+
if lid in resend_succeeded:
|
|
848
|
+
result.upload_succeeded += 1
|
|
849
|
+
consecutive_failures = 0
|
|
850
|
+
progress('upload', path, True)
|
|
851
|
+
else:
|
|
852
|
+
reason = 'file not acknowledged in batch_update successes (retry)'
|
|
853
|
+
result.upload_failures.append((path, reason))
|
|
854
|
+
progress('upload', path, False)
|
|
855
|
+
note_failure(reason)
|
|
856
|
+
except (RateLimitError, ConsecutiveWriteFailureError):
|
|
857
|
+
raise
|
|
858
|
+
except Exception as resend_exc:
|
|
859
|
+
# No third attempt (D-04) -- attribute every
|
|
860
|
+
# still-unresolved absent item as a per-file failure.
|
|
861
|
+
for path, _, _ in resend:
|
|
862
|
+
reason = f'HTTP 401 retry failed: {resend_exc}'
|
|
863
|
+
result.upload_failures.append((path, reason))
|
|
864
|
+
progress('upload', path, False)
|
|
865
|
+
note_failure(str(resend_exc))
|
|
866
|
+
except RateLimitError:
|
|
867
|
+
# Anti-abuse throttle/lockout: abort the whole batch (do not
|
|
868
|
+
# mask it as one per-item failure and keep hammering).
|
|
869
|
+
if budget is not None:
|
|
870
|
+
budget.reconcile_tripped(clock())
|
|
871
|
+
budget.save()
|
|
872
|
+
raise
|
|
873
|
+
except AuthenticationError:
|
|
874
|
+
# REL-04, D-02: the retry branch above raises this when the
|
|
875
|
+
# run's re-login itself failed -- it must propagate exactly as
|
|
876
|
+
# RateLimitError does, never be re-caught by the broader
|
|
877
|
+
# ConsecutiveWriteFailureError/Exception branches below and
|
|
878
|
+
# never be attributed as a per-file failure.
|
|
879
|
+
raise
|
|
880
|
+
except BudgetExhausted:
|
|
881
|
+
# REL-03, D-07: the retry's re-login/resend `budget.acquire(...)`
|
|
882
|
+
# calls (above, inside the 401 branch) use the exact same call
|
|
883
|
+
# shape as every other write's acquire -- so a BudgetExhausted
|
|
884
|
+
# raised from THEM must propagate exactly as it does from the
|
|
885
|
+
# chunk-start acquire (which sits entirely outside this
|
|
886
|
+
# try/except and is never caught here at all), not be
|
|
887
|
+
# re-labelled a per-file write failure by the generic branch
|
|
888
|
+
# below.
|
|
889
|
+
raise
|
|
890
|
+
except ConsecutiveWriteFailureError:
|
|
891
|
+
# note_failure() above can raise this from WITHIN the per-file
|
|
892
|
+
# attribution loop (all prepped files already individually
|
|
893
|
+
# appended/reported there) -- it must propagate as-is, NOT be
|
|
894
|
+
# re-caught by the generic Exception branch below, which would
|
|
895
|
+
# otherwise mistake the abort for a whole-chunk Pushd failure and
|
|
896
|
+
# double-attribute every prepped file a second time. (Reconcile
|
|
897
|
+
# already happened inside note_failure() before this raised.)
|
|
898
|
+
raise
|
|
899
|
+
except Exception as e:
|
|
900
|
+
# A whole-chunk Pushd failure (select_asset or batch_update
|
|
901
|
+
# raising) attributes ALL prepped files in this chunk as failed
|
|
902
|
+
# -- there is no per-item signal to fall back on.
|
|
903
|
+
for path, _, _ in prepped:
|
|
904
|
+
result.upload_failures.append((path, str(e)))
|
|
905
|
+
progress('upload', path, False)
|
|
906
|
+
note_failure(str(e))
|
|
907
|
+
|
|
908
|
+
if budget is not None:
|
|
909
|
+
# Save after every chunk that returns normally (success OR
|
|
910
|
+
# ordinary caught failure, Phase 09 ANTI-04) -- acquire() already
|
|
911
|
+
# decremented tokens in-memory for the attempted requests
|
|
912
|
+
# regardless of per-item outcome, so this keeps the on-disk
|
|
913
|
+
# estimate from drifting optimistic after an interrupt.
|
|
914
|
+
budget.save()
|
|
915
|
+
|
|
916
|
+
for chunk in _chunked(plan.to_reshow, batch_size):
|
|
917
|
+
interchunk_pause()
|
|
918
|
+
if budget is not None:
|
|
919
|
+
# 1 request per chunk (select_asset is a batch endpoint).
|
|
920
|
+
budget.acquire(1, wait=wait_on_budget, max_wait=max_wait_seconds,
|
|
921
|
+
now=clock(), sleep=sleep, on_wait=on_wait)
|
|
922
|
+
chunk_retried = False # D-04: at most one verify-and-retry per chunk
|
|
923
|
+
try:
|
|
924
|
+
try:
|
|
925
|
+
throttle()
|
|
926
|
+
aura.frame_api.select_asset(frame_id, [AssetPartialId(id=asset.id) for asset in chunk])
|
|
927
|
+
for asset in chunk:
|
|
928
|
+
result.reshow_succeeded += 1
|
|
929
|
+
consecutive_failures = 0
|
|
930
|
+
progress('reshow', asset.id, True)
|
|
931
|
+
except httpx.HTTPStatusError as e:
|
|
932
|
+
# This loop acts on server-assigned asset ids, not a
|
|
933
|
+
# client-minted local_identifier, and re-showing an
|
|
934
|
+
# already-visible asset is a no-op -- identity already
|
|
935
|
+
# exists server-side, so a re-send cannot create a
|
|
936
|
+
# duplicate row and no verify probe is needed here (unlike
|
|
937
|
+
# the upload loop above).
|
|
938
|
+
if not retry_on_auth_401 or not _is_http_401(e) or chunk_retried:
|
|
939
|
+
raise
|
|
940
|
+
chunk_retried = True
|
|
941
|
+
result.chunks_retried += 1
|
|
942
|
+
|
|
943
|
+
if not relogin_done:
|
|
944
|
+
if budget is not None:
|
|
945
|
+
budget.acquire(RETRY_RELOGIN_REQUEST_COST, wait=wait_on_budget,
|
|
946
|
+
max_wait=max_wait_seconds, now=clock(),
|
|
947
|
+
sleep=sleep, on_wait=on_wait)
|
|
948
|
+
try:
|
|
949
|
+
relogin()
|
|
950
|
+
except Exception as relogin_exc:
|
|
951
|
+
raise AuthenticationError(relogin_exc) from relogin_exc
|
|
952
|
+
relogin_done = True
|
|
953
|
+
else:
|
|
954
|
+
for asset in chunk:
|
|
955
|
+
reason = ('HTTP 401 after a successful re-login -- classified as an '
|
|
956
|
+
'anti-abuse trip, not authentication')
|
|
957
|
+
result.reshow_failures.append((asset.id, reason))
|
|
958
|
+
progress('reshow', asset.id, False)
|
|
959
|
+
note_failure(reason)
|
|
960
|
+
if budget is not None:
|
|
961
|
+
budget.save()
|
|
962
|
+
continue
|
|
963
|
+
|
|
964
|
+
# Re-send the whole chunk once (D-04) -- same request count
|
|
965
|
+
# this loop already charges (D-05/D-07).
|
|
966
|
+
if budget is not None:
|
|
967
|
+
budget.acquire(1, wait=wait_on_budget, max_wait=max_wait_seconds,
|
|
968
|
+
now=clock(), sleep=sleep, on_wait=on_wait)
|
|
969
|
+
try:
|
|
970
|
+
throttle()
|
|
971
|
+
aura.frame_api.select_asset(
|
|
972
|
+
frame_id, [AssetPartialId(id=asset.id) for asset in chunk])
|
|
973
|
+
for asset in chunk:
|
|
974
|
+
result.reshow_succeeded += 1
|
|
975
|
+
consecutive_failures = 0
|
|
976
|
+
progress('reshow', asset.id, True)
|
|
977
|
+
except (RateLimitError, ConsecutiveWriteFailureError):
|
|
978
|
+
raise
|
|
979
|
+
except Exception as resend_exc:
|
|
980
|
+
# No third attempt (D-04).
|
|
981
|
+
for asset in chunk:
|
|
982
|
+
reason = f'HTTP 401 retry failed: {resend_exc}'
|
|
983
|
+
result.reshow_failures.append((asset.id, reason))
|
|
984
|
+
progress('reshow', asset.id, False)
|
|
985
|
+
note_failure(str(resend_exc))
|
|
986
|
+
except RateLimitError:
|
|
987
|
+
if budget is not None:
|
|
988
|
+
budget.reconcile_tripped(clock())
|
|
989
|
+
budget.save()
|
|
990
|
+
raise
|
|
991
|
+
except AuthenticationError:
|
|
992
|
+
raise
|
|
993
|
+
except BudgetExhausted:
|
|
994
|
+
raise
|
|
995
|
+
except ConsecutiveWriteFailureError:
|
|
996
|
+
raise
|
|
997
|
+
except Exception as e:
|
|
998
|
+
# select_asset returns only a count, never per-item -- a raised
|
|
999
|
+
# chunk-level failure attributes ALL re-shows in this chunk, the
|
|
1000
|
+
# same coarser attribution the removal loop documents.
|
|
1001
|
+
for asset in chunk:
|
|
1002
|
+
result.reshow_failures.append((asset.id, str(e)))
|
|
1003
|
+
progress('reshow', asset.id, False)
|
|
1004
|
+
note_failure(str(e))
|
|
1005
|
+
|
|
1006
|
+
if budget is not None:
|
|
1007
|
+
budget.save()
|
|
1008
|
+
|
|
1009
|
+
for chunk in _chunked(plan.to_delete, batch_size):
|
|
1010
|
+
interchunk_pause()
|
|
1011
|
+
if budget is not None:
|
|
1012
|
+
# 1 request per chunk for the batch endpoints; one per asset for
|
|
1013
|
+
# hard_delete, which has no batch form (Pitfall 3) -- charging it
|
|
1014
|
+
# 1 would let a hard delete run the budget dry unnoticed and
|
|
1015
|
+
# re-trip the anti-abuse lockout.
|
|
1016
|
+
budget.acquire(_REMOVAL_REQUEST_COST[removal_mode](chunk),
|
|
1017
|
+
wait=wait_on_budget, max_wait=max_wait_seconds,
|
|
1018
|
+
now=clock(), sleep=sleep, on_wait=on_wait)
|
|
1019
|
+
chunk_retried = False # D-04: at most one verify-and-retry per chunk
|
|
1020
|
+
try:
|
|
1021
|
+
try:
|
|
1022
|
+
throttle()
|
|
1023
|
+
_REMOVAL_PRIMITIVE[removal_mode](aura, frame_id, chunk)
|
|
1024
|
+
for asset in chunk:
|
|
1025
|
+
result.delete_succeeded += 1
|
|
1026
|
+
consecutive_failures = 0
|
|
1027
|
+
progress('delete', asset.id, True)
|
|
1028
|
+
except httpx.HTTPStatusError as e:
|
|
1029
|
+
# Same rationale as the re-show loop above: this loop acts
|
|
1030
|
+
# on server-assigned asset ids, hiding/removing an
|
|
1031
|
+
# already-hidden/removed asset is a no-op, so identity
|
|
1032
|
+
# already exists server-side and a re-send cannot create a
|
|
1033
|
+
# duplicate -- no verify probe needed.
|
|
1034
|
+
if not retry_on_auth_401 or not _is_http_401(e) or chunk_retried:
|
|
1035
|
+
raise
|
|
1036
|
+
chunk_retried = True
|
|
1037
|
+
result.chunks_retried += 1
|
|
1038
|
+
|
|
1039
|
+
if not relogin_done:
|
|
1040
|
+
if budget is not None:
|
|
1041
|
+
budget.acquire(RETRY_RELOGIN_REQUEST_COST, wait=wait_on_budget,
|
|
1042
|
+
max_wait=max_wait_seconds, now=clock(),
|
|
1043
|
+
sleep=sleep, on_wait=on_wait)
|
|
1044
|
+
try:
|
|
1045
|
+
relogin()
|
|
1046
|
+
except Exception as relogin_exc:
|
|
1047
|
+
raise AuthenticationError(relogin_exc) from relogin_exc
|
|
1048
|
+
relogin_done = True
|
|
1049
|
+
else:
|
|
1050
|
+
for asset in chunk:
|
|
1051
|
+
reason = ('HTTP 401 after a successful re-login -- classified as an '
|
|
1052
|
+
'anti-abuse trip, not authentication')
|
|
1053
|
+
result.delete_failures.append((asset.id, reason))
|
|
1054
|
+
progress('delete', asset.id, False)
|
|
1055
|
+
note_failure(reason)
|
|
1056
|
+
if budget is not None:
|
|
1057
|
+
budget.save()
|
|
1058
|
+
continue
|
|
1059
|
+
|
|
1060
|
+
# Re-send the whole chunk once (D-04) -- same request count
|
|
1061
|
+
# this loop already charges (D-05/D-07).
|
|
1062
|
+
if budget is not None:
|
|
1063
|
+
budget.acquire(_REMOVAL_REQUEST_COST[removal_mode](chunk),
|
|
1064
|
+
wait=wait_on_budget, max_wait=max_wait_seconds,
|
|
1065
|
+
now=clock(), sleep=sleep, on_wait=on_wait)
|
|
1066
|
+
try:
|
|
1067
|
+
throttle()
|
|
1068
|
+
_REMOVAL_PRIMITIVE[removal_mode](aura, frame_id, chunk)
|
|
1069
|
+
for asset in chunk:
|
|
1070
|
+
result.delete_succeeded += 1
|
|
1071
|
+
consecutive_failures = 0
|
|
1072
|
+
progress('delete', asset.id, True)
|
|
1073
|
+
except (RateLimitError, ConsecutiveWriteFailureError):
|
|
1074
|
+
raise
|
|
1075
|
+
except Exception as resend_exc:
|
|
1076
|
+
# No third attempt (D-04).
|
|
1077
|
+
for asset in chunk:
|
|
1078
|
+
reason = f'HTTP 401 retry failed: {resend_exc}'
|
|
1079
|
+
result.delete_failures.append((asset.id, reason))
|
|
1080
|
+
progress('delete', asset.id, False)
|
|
1081
|
+
note_failure(str(resend_exc))
|
|
1082
|
+
except RateLimitError:
|
|
1083
|
+
if budget is not None:
|
|
1084
|
+
budget.reconcile_tripped(clock())
|
|
1085
|
+
budget.save()
|
|
1086
|
+
raise
|
|
1087
|
+
except AuthenticationError:
|
|
1088
|
+
raise
|
|
1089
|
+
except BudgetExhausted:
|
|
1090
|
+
raise
|
|
1091
|
+
except ConsecutiveWriteFailureError:
|
|
1092
|
+
raise
|
|
1093
|
+
except Exception as e:
|
|
1094
|
+
# remove_asset returns only a count, never per-item -- a raised
|
|
1095
|
+
# chunk-level failure attributes ALL deletes in this chunk
|
|
1096
|
+
# (coarser than upload attribution, T-fyr-01).
|
|
1097
|
+
for asset in chunk:
|
|
1098
|
+
result.delete_failures.append((asset.id, str(e)))
|
|
1099
|
+
progress('delete', asset.id, False)
|
|
1100
|
+
note_failure(str(e))
|
|
1101
|
+
|
|
1102
|
+
if budget is not None:
|
|
1103
|
+
budget.save()
|
|
1104
|
+
|
|
1105
|
+
return result
|