pushframe 5.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pushframe/__init__.py +4 -0
- pushframe/api/__init__.py +0 -0
- pushframe/api/accountApi.py +72 -0
- pushframe/api/activityApi.py +87 -0
- pushframe/api/assetApi.py +194 -0
- pushframe/api/baseApi.py +6 -0
- pushframe/api/frameApi.py +271 -0
- pushframe/api/notificationApi.py +15 -0
- pushframe/api/peopleApi.py +25 -0
- pushframe/api/playlistApi.py +9 -0
- pushframe/aura.py +182 -0
- pushframe/aws/__init__.py +0 -0
- pushframe/aws/awsclient.py +23 -0
- pushframe/aws/s3client.py +40 -0
- pushframe/aws/sqsclient.py +33 -0
- pushframe/cache.py +50 -0
- pushframe/cli.py +1134 -0
- pushframe/client.py +267 -0
- pushframe/exif.py +147 -0
- pushframe/export.py +53 -0
- pushframe/google/__init__.py +43 -0
- pushframe/google/bootstrap.py +134 -0
- pushframe/google/cache.py +167 -0
- pushframe/google/client.py +140 -0
- pushframe/google/enumerate.py +270 -0
- pushframe/google/manifest.py +111 -0
- pushframe/google/parsers.py +345 -0
- pushframe/google/redaction.py +33 -0
- pushframe/google/vault.py +126 -0
- pushframe/gsync.py +463 -0
- pushframe/migration.py +86 -0
- pushframe/models/__init__.py +0 -0
- pushframe/models/activity.py +79 -0
- pushframe/models/asset.py +159 -0
- pushframe/models/frame.py +105 -0
- pushframe/models/meta.py +11 -0
- pushframe/models/person.py +24 -0
- pushframe/models/user.py +22 -0
- pushframe/ratelimit.py +222 -0
- pushframe/reconcile.py +384 -0
- pushframe/sync.py +1105 -0
- pushframe/utils/dt.py +15 -0
- pushframe/utils/io.py +23 -0
- pushframe/utils/settings.py +59 -0
- pushframe-5.0.0.dist-info/METADATA +53 -0
- pushframe-5.0.0.dist-info/RECORD +49 -0
- pushframe-5.0.0.dist-info/WHEEL +4 -0
- pushframe-5.0.0.dist-info/entry_points.txt +2 -0
- pushframe-5.0.0.dist-info/licenses/LICENSE +31 -0
pushframe/gsync.py
ADDED
|
@@ -0,0 +1,463 @@
|
|
|
1
|
+
"""Album → frame mirror engine (phase 18, plans 18-02/18-03).
|
|
2
|
+
|
|
3
|
+
The plan-computing half is pure by construction (CSE-05, the structural
|
|
4
|
+
dry-run rule): `build_demand`, `run_google_sync_plan` and
|
|
5
|
+
`format_plan_report` contain no mutating call and no I/O — everything they
|
|
6
|
+
need (listing, manifest, staged outcome, frame assets) is passed in.
|
|
7
|
+
|
|
8
|
+
The mutating half (`run_google_sync`) composes those with v2.0's
|
|
9
|
+
`execute_plan` behind the CLI's apply gate: Google-side downloads run on the
|
|
10
|
+
plan-01 bounded pool (CSE-01) while every frame write stays inside
|
|
11
|
+
execute_plan's synchronous, WriteBudget-paced path — concurrency never
|
|
12
|
+
crosses the Aura client seam. Removal is HIDE-ONLY (CSE-06, SAFE-03):
|
|
13
|
+
`removal_mode='hide'` is passed unconditionally and no delete tier exists on
|
|
14
|
+
the google-sync surface.
|
|
15
|
+
|
|
16
|
+
The load-bearing rule (CSE-03 / roadmap criterion 3): demand is rebuilt from
|
|
17
|
+
the album LISTING plus the MANIFEST — never from a directory walk of the
|
|
18
|
+
(pruned) cache. A manifest member whose cache file is gone still asserts its
|
|
19
|
+
md5_hash (the manifest MEANS the upload was confirmed, so the frame already
|
|
20
|
+
holds the bytes); its demand entry carries a sentinel path that must never
|
|
21
|
+
enter `to_upload` — if the frame disagrees, that is drift and fails loud.
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import os
|
|
26
|
+
import sys
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
|
|
29
|
+
from tqdm import tqdm
|
|
30
|
+
|
|
31
|
+
from pushframe.google.redaction import redact_link
|
|
32
|
+
from pushframe.sync import compute_plan
|
|
33
|
+
|
|
34
|
+
# Suffix of the sentinel demand path for manifest members whose cache file
|
|
35
|
+
# was pruned: the md5 is asserted from the manifest, the path is fictional
|
|
36
|
+
# and must never be uploaded.
|
|
37
|
+
SENTINEL_SUFFIX = ".absent"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
# SAFE-02: a plan whose hide count exceeds this share of the frame's
|
|
41
|
+
# hash-bearing assets requires an explicit confirmation (env-overridable).
|
|
42
|
+
GOOGLE_SYNC_REMOVAL_THRESHOLD = 0.2
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _threshold() -> float:
|
|
46
|
+
# IDN-04: PUSHFRAME_* primary, AURA_* legacy fallback (one release of grace).
|
|
47
|
+
raw = (os.getenv("PUSHFRAME_GOOGLE_SYNC_REMOVAL_THRESHOLD")
|
|
48
|
+
or os.getenv("AURA_GOOGLE_SYNC_REMOVAL_THRESHOLD"))
|
|
49
|
+
try:
|
|
50
|
+
return float(raw) if raw else GOOGLE_SYNC_REMOVAL_THRESHOLD
|
|
51
|
+
except ValueError:
|
|
52
|
+
return GOOGLE_SYNC_REMOVAL_THRESHOLD
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def default_cache_dir(album_share_token: str) -> Path:
|
|
56
|
+
"""D-03: staging is per-album, keyed by the album's share token —
|
|
57
|
+
`~/.config/pushframe/google-cache/<share_token>/<google_media_id>`."""
|
|
58
|
+
return Path("~/.config/pushframe/google-cache") / album_share_token
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class SafeSyncError(RuntimeError):
|
|
62
|
+
"""Named mirror-safety abort (SAFE-01) — never a silent plan."""
|
|
63
|
+
|
|
64
|
+
@classmethod
|
|
65
|
+
def empty_listing(cls) -> "SafeSyncError":
|
|
66
|
+
return cls(
|
|
67
|
+
"album listing is EMPTY (SAFE-01) — an empty listing cannot be "
|
|
68
|
+
"distinguished from a truncated one and must never be read as "
|
|
69
|
+
"'hide everything on the frame'; refusing to plan"
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
@classmethod
|
|
73
|
+
def truncated_listing(cls) -> "SafeSyncError":
|
|
74
|
+
return cls(
|
|
75
|
+
"album listing is NOT exhausted cleanly (SAFE-01) — a truncated "
|
|
76
|
+
"listing would understate demand and mass-hide the difference; "
|
|
77
|
+
"refusing to plan"
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
@classmethod
|
|
81
|
+
def empty_frame_listing(cls) -> "SafeSyncError":
|
|
82
|
+
return cls(
|
|
83
|
+
"frame asset listing is EMPTY (SAFE-01) — indistinguishable from "
|
|
84
|
+
"the live-observed get_assets drift (16-LIVE-FINDINGS); verify "
|
|
85
|
+
"the frame's assets and re-run"
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
@classmethod
|
|
89
|
+
def manifest_drift(cls) -> "SafeSyncError":
|
|
90
|
+
return cls(
|
|
91
|
+
"manifest claims an upload was confirmed but the frame reports "
|
|
92
|
+
"no matching md5_hash (SAFE-01 drift guard) — the cache was "
|
|
93
|
+
"pruned on that claim; investigate before re-running"
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def build_demand(listing, manifest, staged, cache_dir: Path, *,
|
|
98
|
+
metadata_item_count: int | None = None,
|
|
99
|
+
) -> tuple[dict[str, list[Path]], list[tuple[str, str]], int | None]:
|
|
100
|
+
"""Rebuild the v2.0 demand map from the album listing + manifest.
|
|
101
|
+
|
|
102
|
+
Returns `(demand, failures, videos_skipped)` where `demand` is shaped
|
|
103
|
+
exactly like `scan_directory`'s output (`{md5_hash: [path]}`, one
|
|
104
|
+
logical want per hash) so v2.0's `compute_plan` consumes it unchanged.
|
|
105
|
+
|
|
106
|
+
Per listing item:
|
|
107
|
+
- manifest entry exists → demand key = the manifest's md5_hash; the
|
|
108
|
+
demand path is the staged cache file when present, else the sentinel
|
|
109
|
+
`cache_dir/<id>.absent` (pruned — the frame already holds the bytes).
|
|
110
|
+
- no manifest entry, staged → demand key = the staged md5_hash, real path.
|
|
111
|
+
- otherwise (failed/missing download) → excluded from demand and
|
|
112
|
+
collected in `failures` (SAFE-04: never planned from absent bytes).
|
|
113
|
+
|
|
114
|
+
This function NEVER walks `cache_dir` — the cache is a staging area, not
|
|
115
|
+
a source of truth (CSE-03).
|
|
116
|
+
"""
|
|
117
|
+
from pushframe.google.cache import videos_skipped as _videos_skipped
|
|
118
|
+
|
|
119
|
+
demand: dict[str, list[Path]] = {}
|
|
120
|
+
failures: list[tuple[str, str]] = []
|
|
121
|
+
staged_by_id = staged.staged_by_id if staged is not None else {}
|
|
122
|
+
failed_by_id = dict(staged.failed) if staged is not None else {}
|
|
123
|
+
|
|
124
|
+
for item in listing.items:
|
|
125
|
+
gid = item["id"]
|
|
126
|
+
entry = manifest.entry_for(gid) if manifest is not None else None
|
|
127
|
+
if entry is not None:
|
|
128
|
+
md5 = entry["md5_hash"]
|
|
129
|
+
s = staged_by_id.get(gid)
|
|
130
|
+
path = Path(s["path"]) if s else Path(cache_dir) / f"{gid}{SENTINEL_SUFFIX}"
|
|
131
|
+
demand[md5] = [path]
|
|
132
|
+
elif gid in staged_by_id:
|
|
133
|
+
s = staged_by_id[gid]
|
|
134
|
+
demand[s["md5_hash"]] = [Path(s["path"])]
|
|
135
|
+
else:
|
|
136
|
+
error = failed_by_id.get(gid) or (
|
|
137
|
+
"item is neither manifest-backed nor staged — not downloaded"
|
|
138
|
+
)
|
|
139
|
+
failures.append((gid, error))
|
|
140
|
+
|
|
141
|
+
videos = _videos_skipped(listing, metadata_item_count)
|
|
142
|
+
return demand, failures, videos
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def run_google_sync_plan(listing, manifest, staged, cache_dir: Path,
|
|
146
|
+
frame_assets: list, *,
|
|
147
|
+
metadata_item_count: int | None = None,
|
|
148
|
+
skipped_non_image: int = 0,
|
|
149
|
+
) -> tuple[object, list[tuple[str, str]], int | None]:
|
|
150
|
+
"""The pure plan computation: SAFE-01 gates → demand → v2.0's compute_plan.
|
|
151
|
+
|
|
152
|
+
Returns (SyncPlan, failures, videos_skipped). Raises a named
|
|
153
|
+
SafeSyncError — never returns a plan — when the listing or the frame
|
|
154
|
+
asset listing cannot be trusted.
|
|
155
|
+
"""
|
|
156
|
+
if not listing.items:
|
|
157
|
+
raise SafeSyncError.empty_listing()
|
|
158
|
+
if getattr(listing, "exhausted_cleanly", None) is not True:
|
|
159
|
+
raise SafeSyncError.truncated_listing()
|
|
160
|
+
if not frame_assets:
|
|
161
|
+
raise SafeSyncError.empty_frame_listing()
|
|
162
|
+
|
|
163
|
+
demand, failures, videos = build_demand(
|
|
164
|
+
listing, manifest, staged, cache_dir,
|
|
165
|
+
metadata_item_count=metadata_item_count,
|
|
166
|
+
)
|
|
167
|
+
plan = compute_plan(demand, frame_assets, skipped_non_image=skipped_non_image)
|
|
168
|
+
|
|
169
|
+
# Structural enforcement of the sentinel contract: a sentinel path in
|
|
170
|
+
# to_upload means the manifest claimed a confirmed upload the frame
|
|
171
|
+
# disputes — the cache was pruned on that claim, so downloading again is
|
|
172
|
+
# the ONLY honest recovery and that decision is not the plan's to make.
|
|
173
|
+
if any(p.name.endswith(SENTINEL_SUFFIX) for p in plan.to_upload):
|
|
174
|
+
raise SafeSyncError.manifest_drift()
|
|
175
|
+
|
|
176
|
+
return plan, failures, videos
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def format_plan_report(plan, failures: list[tuple[str, str]],
|
|
180
|
+
videos_skipped: int | None, *,
|
|
181
|
+
staged=None) -> str:
|
|
182
|
+
"""Render the plan for the CLI (dry-run print and post-apply summary).
|
|
183
|
+
|
|
184
|
+
Every Google id passes through `redact_link` — the report is print-ready
|
|
185
|
+
and log-safe by construction.
|
|
186
|
+
"""
|
|
187
|
+
lines = [
|
|
188
|
+
f"Plan: {len(plan.to_upload)} to upload, {len(plan.to_reshow)} to "
|
|
189
|
+
f"re-show, {plan.unchanged} unchanged, {len(plan.to_delete)} to hide, "
|
|
190
|
+
f"{plan.already_hidden} already hidden",
|
|
191
|
+
]
|
|
192
|
+
if videos_skipped is not None:
|
|
193
|
+
lines.append(
|
|
194
|
+
f"Videos skipped: {videos_skipped} (metadata delta — videos are "
|
|
195
|
+
f"out of sync scope, never silently dropped)"
|
|
196
|
+
)
|
|
197
|
+
if failures:
|
|
198
|
+
lines.append(f"Failed downloads ({len(failures)}):")
|
|
199
|
+
for gid, err in failures:
|
|
200
|
+
lines.append(f" {redact_link(gid)}: {err}")
|
|
201
|
+
|
|
202
|
+
staged_by_id = {s["google_media_id"]: s for s in staged.staged} if staged else {}
|
|
203
|
+
if plan.to_upload:
|
|
204
|
+
lines.append("Upload candidates (index | id shape | bytes):")
|
|
205
|
+
for i, p in enumerate(plan.to_upload, 1):
|
|
206
|
+
gid = p.name.removesuffix(SENTINEL_SUFFIX)
|
|
207
|
+
size = staged_by_id.get(gid, {}).get("size_bytes")
|
|
208
|
+
size_s = f"{size:,}" if size is not None else "?"
|
|
209
|
+
lines.append(f" {i:4d} | {redact_link(gid)} | {size_s}")
|
|
210
|
+
return "\n".join(lines)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def run_google_sync(album_target: str, frame_arg: str, *, apply: bool = False,
|
|
214
|
+
yes: bool = False, debug: bool = False, session=None,
|
|
215
|
+
aura=None, s3_client=None, sqs_client=None, budget=None,
|
|
216
|
+
workers: int = 4, threshold: float | None = None,
|
|
217
|
+
input_fn=None, is_interactive: bool | None = None,
|
|
218
|
+
list_shared=None, cache_dir=None,
|
|
219
|
+
manifest_path=None) -> int:
|
|
220
|
+
"""The mutating half: album → frame mirror end to end (plan 18-03).
|
|
221
|
+
|
|
222
|
+
DI seams mirror run_sync/run_google_album conventions: session/aura/s3/
|
|
223
|
+
sqs/budget injectable for offline tests, `input_fn` replaces input(),
|
|
224
|
+
`is_interactive` replaces the sys.stdin.isatty() check (None = consult
|
|
225
|
+
stdin; tests pass True/False explicitly), `list_shared` overrides album
|
|
226
|
+
resolution, `threshold` wins over the env.
|
|
227
|
+
|
|
228
|
+
Flow: resolve album+frame → SAFE-01 prechecks → concurrent downloads
|
|
229
|
+
(CSE-01, Google side only) → plan (the pure half) → print → apply gate
|
|
230
|
+
(SAFE-02 mass-hide threshold + v2.0's y/N, --yes bypasses both,
|
|
231
|
+
non-interactive without --yes fails closed) → execute_plan with
|
|
232
|
+
removal_mode='hide' UNCONDITIONALLY (CSE-06/SAFE-03: no delete tier on
|
|
233
|
+
this verb) → manifest persisted ONLY for progress-confirmed uploads →
|
|
234
|
+
prune → report. Without --apply it returns before any mutating call.
|
|
235
|
+
|
|
236
|
+
Exit codes: 0 dry-run/aborted confirmation, 1 failure, 2 usage/ambiguity.
|
|
237
|
+
"""
|
|
238
|
+
from pushframe.google.cache import download_to_cache, prune_cache
|
|
239
|
+
from pushframe.google.manifest import GoogleManifest
|
|
240
|
+
from pushframe.google.client import GoogleSession
|
|
241
|
+
from pushframe.google.enumerate import (
|
|
242
|
+
EnumerateError,
|
|
243
|
+
enumerate_album,
|
|
244
|
+
list_shared_albums,
|
|
245
|
+
)
|
|
246
|
+
from pushframe.google.redaction import redact_tokens
|
|
247
|
+
from pushframe.google.vault import CookieVaultError
|
|
248
|
+
from pushframe.cli import _print_album_candidates, resolve_album, resolve_frame
|
|
249
|
+
|
|
250
|
+
if input_fn is None:
|
|
251
|
+
input_fn = input
|
|
252
|
+
if is_interactive is None:
|
|
253
|
+
is_interactive = sys.stdin.isatty()
|
|
254
|
+
|
|
255
|
+
if session is None:
|
|
256
|
+
try:
|
|
257
|
+
session = GoogleSession.from_vault()
|
|
258
|
+
except CookieVaultError as e:
|
|
259
|
+
print(f'google-sync failed: {e}')
|
|
260
|
+
return 1
|
|
261
|
+
|
|
262
|
+
# --- album resolution (D-05 surface, phase 17 conventions) ---
|
|
263
|
+
try:
|
|
264
|
+
albums = list_shared() if list_shared is not None else list_shared_albums(session)
|
|
265
|
+
resolved = resolve_album(album_target, albums)
|
|
266
|
+
except EnumerateError as e:
|
|
267
|
+
print(f'google-sync failed: {redact_tokens(str(e))}')
|
|
268
|
+
return 1
|
|
269
|
+
|
|
270
|
+
if resolved.status == 'ambiguous':
|
|
271
|
+
print(f"'{album_target}' matches more than one album — re-run with a "
|
|
272
|
+
f"link/id or a fuller name:")
|
|
273
|
+
_print_album_candidates(resolved.candidates, numbered=True, show_count=True)
|
|
274
|
+
return 2
|
|
275
|
+
if resolved.status == 'not_found':
|
|
276
|
+
print(f"No album matches '{album_target}'. Available shared albums:")
|
|
277
|
+
_print_album_candidates(albums, numbered=True, show_count=True)
|
|
278
|
+
return 2
|
|
279
|
+
album = resolved.album
|
|
280
|
+
|
|
281
|
+
page_key = None
|
|
282
|
+
if album.share_url and 'key=' in album.share_url:
|
|
283
|
+
page_key = album.share_url.split('key=', 1)[1].split('&', 1)[0] or None
|
|
284
|
+
|
|
285
|
+
try:
|
|
286
|
+
listing = enumerate_album(session, album.album_id, page_key=page_key)
|
|
287
|
+
except EnumerateError as e:
|
|
288
|
+
print(f'google-sync failed: {redact_tokens(str(e))}')
|
|
289
|
+
return 1
|
|
290
|
+
|
|
291
|
+
# SAFE-01 prechecks happen BEFORE any frame contact.
|
|
292
|
+
if not listing.items:
|
|
293
|
+
print(f'google-sync failed: {SafeSyncError.empty_listing()}')
|
|
294
|
+
return 1
|
|
295
|
+
if listing.exhausted_cleanly is not True:
|
|
296
|
+
print(f'google-sync failed: {SafeSyncError.truncated_listing()}')
|
|
297
|
+
return 1
|
|
298
|
+
|
|
299
|
+
# --- frame resolution + asset listing (v2.0 conventions) ---
|
|
300
|
+
if aura is None:
|
|
301
|
+
from pushframe.aura import Aura
|
|
302
|
+
from pushframe.cli import _configure_cli_logging
|
|
303
|
+
aura = Aura()
|
|
304
|
+
_configure_cli_logging(debug)
|
|
305
|
+
try:
|
|
306
|
+
aura.login()
|
|
307
|
+
except Exception as e:
|
|
308
|
+
print(f'Login failed: {e}')
|
|
309
|
+
return 1
|
|
310
|
+
|
|
311
|
+
frames = aura.frame_api.get_frames()
|
|
312
|
+
frame_res = resolve_frame(frame_arg, frames)
|
|
313
|
+
if frame_res.status == 'ambiguous':
|
|
314
|
+
print(f"'{frame_arg}' matches more than one frame:")
|
|
315
|
+
for i, f in enumerate(frame_res.candidates, 1):
|
|
316
|
+
print(f' {i}. {f.name} (id: {f.id})')
|
|
317
|
+
return 2
|
|
318
|
+
if frame_res.status == 'not_found':
|
|
319
|
+
print(f"No frame matches '{frame_arg}'. Available frames:")
|
|
320
|
+
for f in frame_res.candidates:
|
|
321
|
+
print(f' - {f.name} (id: {f.id})')
|
|
322
|
+
return 2
|
|
323
|
+
frame = frame_res.frame
|
|
324
|
+
|
|
325
|
+
try:
|
|
326
|
+
frame_assets = aura.get_all_assets(frame.id)
|
|
327
|
+
except Exception as e:
|
|
328
|
+
print(f'google-sync failed: reading frame assets failed: {e}')
|
|
329
|
+
return 1
|
|
330
|
+
if not frame_assets:
|
|
331
|
+
print(f'google-sync failed: {SafeSyncError.empty_frame_listing()}')
|
|
332
|
+
return 1
|
|
333
|
+
|
|
334
|
+
# --- downloads (CSE-01: Google side only) + plan (pure half) ---
|
|
335
|
+
manifest = GoogleManifest.load(manifest_path)
|
|
336
|
+
cdir = Path(cache_dir) if cache_dir else default_cache_dir(album.album_id)
|
|
337
|
+
with tqdm(total=len(listing.items), desc='Downloading', unit='photo',
|
|
338
|
+
disable=not sys.stderr.isatty()) as bar:
|
|
339
|
+
def _dl_progress(gid: str, ok: bool) -> None:
|
|
340
|
+
bar.update(1)
|
|
341
|
+
bar.set_postfix_str(f'{redact_link(gid)} {"ok" if ok else "FAIL"}')
|
|
342
|
+
|
|
343
|
+
staged = download_to_cache(session, listing, cdir,
|
|
344
|
+
manifest=manifest, workers=workers,
|
|
345
|
+
progress=_dl_progress)
|
|
346
|
+
|
|
347
|
+
try:
|
|
348
|
+
plan, failures, videos = run_google_sync_plan(
|
|
349
|
+
listing, manifest, staged, cdir, frame_assets,
|
|
350
|
+
metadata_item_count=album.item_count,
|
|
351
|
+
)
|
|
352
|
+
except SafeSyncError as e:
|
|
353
|
+
print(f'google-sync failed: {e}')
|
|
354
|
+
return 1
|
|
355
|
+
|
|
356
|
+
print(format_plan_report(plan, failures, videos, staged=staged))
|
|
357
|
+
|
|
358
|
+
if not apply:
|
|
359
|
+
return 0 # structural dry-run default — nothing above mutated anything
|
|
360
|
+
|
|
361
|
+
# ---- every path below is mutating ----
|
|
362
|
+
|
|
363
|
+
if not yes and not is_interactive:
|
|
364
|
+
print('--apply requires --yes when running non-interactively')
|
|
365
|
+
return 1
|
|
366
|
+
|
|
367
|
+
# SAFE-02: mass-hide gate — removals vs the frame's hash-bearing assets.
|
|
368
|
+
hash_bearing = [a for a in frame_assets if a.md5_hash]
|
|
369
|
+
effective_threshold = threshold if threshold is not None else _threshold()
|
|
370
|
+
removal_count = len(plan.to_delete)
|
|
371
|
+
if removal_count > effective_threshold * max(len(hash_bearing), 1):
|
|
372
|
+
print(f'⚠ {removal_count} of {len(hash_bearing)} photos on '
|
|
373
|
+
f'"{frame.name}" (id: {frame.id}) would be hidden — over the '
|
|
374
|
+
f'{effective_threshold:.0%} safety threshold (SAFE-02).')
|
|
375
|
+
if not yes:
|
|
376
|
+
answer = input_fn('Proceed with this plan? [y/N] ')
|
|
377
|
+
if answer.strip().lower() not in ('y', 'yes'):
|
|
378
|
+
print('Aborted.')
|
|
379
|
+
return 0
|
|
380
|
+
|
|
381
|
+
if not yes:
|
|
382
|
+
answer = input_fn(f'About to apply this plan to "{frame.name}" '
|
|
383
|
+
f'(id: {frame.id}). Proceed? [y/N] ')
|
|
384
|
+
if answer.strip().lower() not in ('y', 'yes'):
|
|
385
|
+
print('Aborted.')
|
|
386
|
+
return 0
|
|
387
|
+
|
|
388
|
+
from pushframe.aws.s3client import S3Client
|
|
389
|
+
from pushframe.aws.sqsclient import SQSClient
|
|
390
|
+
from pushframe.sync import execute_plan
|
|
391
|
+
|
|
392
|
+
s3 = s3_client if s3_client is not None else S3Client()
|
|
393
|
+
sqs = sqs_client if sqs_client is not None else SQSClient()
|
|
394
|
+
|
|
395
|
+
# CSE-01 (MOD-05's rule): frame writes are paced by the per-account
|
|
396
|
+
# WriteBudget. Built at this boundary like run_sync does (execute_plan
|
|
397
|
+
# itself never constructs one); omitted only when the caller injected a
|
|
398
|
+
# budget=None explicitly. Budget state lives per account email (T-09-02).
|
|
399
|
+
import os as _os
|
|
400
|
+
from pushframe.cli import _build_write_budget, _configure_cli_logging
|
|
401
|
+
if debug:
|
|
402
|
+
_configure_cli_logging(True)
|
|
403
|
+
if budget is None and s3_client is None:
|
|
404
|
+
budget = _build_write_budget(_os.getenv('PUSHFRAME_EMAIL') or _os.getenv('AURA_EMAIL'), ignore_budget=False)
|
|
405
|
+
|
|
406
|
+
staged_by_id = staged.staged_by_id
|
|
407
|
+
confirmed_paths: list[str] = []
|
|
408
|
+
|
|
409
|
+
with tqdm(total=len(plan.to_upload) + len(plan.to_delete), desc='Applying',
|
|
410
|
+
unit='item', disable=not sys.stderr.isatty()) as bar:
|
|
411
|
+
def _progress(kind, identifier, ok):
|
|
412
|
+
bar.update(1)
|
|
413
|
+
bar.set_postfix_str(f'{kind} {"ok" if ok else "FAIL"}')
|
|
414
|
+
if kind == 'upload' and ok:
|
|
415
|
+
confirmed_paths.append(str(identifier))
|
|
416
|
+
|
|
417
|
+
exec_kwargs: dict = {}
|
|
418
|
+
if budget is not None:
|
|
419
|
+
exec_kwargs['budget'] = budget
|
|
420
|
+
try:
|
|
421
|
+
result = execute_plan(plan, aura, frame.id, s3_client=s3,
|
|
422
|
+
sqs_client=sqs, removal_mode='hide',
|
|
423
|
+
progress=_progress, **exec_kwargs)
|
|
424
|
+
except Exception as e:
|
|
425
|
+
print(f'google-sync failed: apply aborted: {e}')
|
|
426
|
+
return 1
|
|
427
|
+
|
|
428
|
+
# Persist manifest entries for every staged item whose bytes are PROVEN
|
|
429
|
+
# on the frame — either progress-confirmed via this apply's uploads, or
|
|
430
|
+
# already held (the frame's own listing reports that md5_hash, v2.0's
|
|
431
|
+
# content-hash contract). A manifest entry MEANS "the bytes live on the
|
|
432
|
+
# frame", whichever evidence established it — this is what makes the
|
|
433
|
+
# NEXT run download nothing (steady state, criterion 3).
|
|
434
|
+
frame_md5s = {a.md5_hash for a in frame_assets if a.md5_hash}
|
|
435
|
+
confirmed_md5s: set[str] = set()
|
|
436
|
+
for path_str in confirmed_paths:
|
|
437
|
+
s = staged_by_id.get(Path(path_str).name)
|
|
438
|
+
if s:
|
|
439
|
+
confirmed_md5s.add(s['md5_hash'])
|
|
440
|
+
added = 0
|
|
441
|
+
for s in staged.staged:
|
|
442
|
+
gid = s['google_media_id']
|
|
443
|
+
if manifest.entry_for(gid) is None and (
|
|
444
|
+
s['md5_hash'] in confirmed_md5s or s['md5_hash'] in frame_md5s):
|
|
445
|
+
manifest.add(gid, md5_hash=s['md5_hash'],
|
|
446
|
+
size_bytes=s['size_bytes'],
|
|
447
|
+
album_share_token=album.album_id)
|
|
448
|
+
added += 1
|
|
449
|
+
if added or manifest.entries:
|
|
450
|
+
manifest.save(manifest_path)
|
|
451
|
+
|
|
452
|
+
print(f'Applied: {result.upload_succeeded} uploaded, '
|
|
453
|
+
f'{result.delete_succeeded} hidden, {result.reshow_succeeded} re-shown')
|
|
454
|
+
if result.upload_failures:
|
|
455
|
+
print(f'{len(result.upload_failures)} upload(s) FAILED — no manifest '
|
|
456
|
+
f'entry written; they will retry next run')
|
|
457
|
+
if failures:
|
|
458
|
+
print(f'{len(failures)} download(s) failed and were NOT synced '
|
|
459
|
+
f'(they will retry next run)')
|
|
460
|
+
pruned = prune_cache(cdir, set(manifest.entries))
|
|
461
|
+
print(f'Cache pruned: {pruned} file(s) removed; '
|
|
462
|
+
f'{len(staged.failed)} failed download(s) kept for retry')
|
|
463
|
+
return 0
|
pushframe/migration.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""One-time user-config migration `~/.config/auraframes/` → `~/.config/pushframe/`
|
|
2
|
+
(phase 20, IDN-03).
|
|
3
|
+
|
|
4
|
+
The pushframe rename moved the binary, the module AND the config home. This
|
|
5
|
+
module moves the operator's existing state exactly once, on first run:
|
|
6
|
+
|
|
7
|
+
- Google cookie vault (0600) incl. the legacy probes/ location the vault's
|
|
8
|
+
soft migration reads,
|
|
9
|
+
- Google manifest (0600),
|
|
10
|
+
- persisted write-budget token bucket,
|
|
11
|
+
- Chrome profile directory used by google-link's cookie harvest,
|
|
12
|
+
- legacy CLI logs.
|
|
13
|
+
|
|
14
|
+
Discipline (fail-safe, never destructive):
|
|
15
|
+
- Migration runs only when the NEW directory does not exist yet — a fresh
|
|
16
|
+
install that already created ~/.config/pushframe/ is never touched, so the
|
|
17
|
+
operation is idempotent by construction.
|
|
18
|
+
- The OLD directory is left in place untouched (belt-and-braces against a
|
|
19
|
+
crash mid-copy: nothing is ever deleted).
|
|
20
|
+
- Files keep their modes (the 0600 vaults stay 0600 — shutil.copy2, never a
|
|
21
|
+
move).
|
|
22
|
+
- `migration_notice()` returns the one-line notice the CLI prints; a silent
|
|
23
|
+
no-op on fresh machines (nothing printed, nothing copied).
|
|
24
|
+
"""
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import shutil
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
|
|
30
|
+
OLD_CONFIG_DIR = Path("~/.config/auraframes").expanduser()
|
|
31
|
+
NEW_CONFIG_DIR = Path("~/.config/pushframe").expanduser()
|
|
32
|
+
|
|
33
|
+
# Everything an existing operator may have under the old home. Missing paths
|
|
34
|
+
# are skipped silently; unknown extra paths are copied too (shutil.copytree of
|
|
35
|
+
# the whole directory minus the excluded noise) so nothing is lost.
|
|
36
|
+
_EXCLUDED_NAMES = {"__pycache__"}
|
|
37
|
+
|
|
38
|
+
_MIGRATED_SUMMARY = {
|
|
39
|
+
"google-cookies.json": "Google session vault",
|
|
40
|
+
"google-manifest.json": "Google sync manifest",
|
|
41
|
+
"probes": "probe vaults + Chrome profile",
|
|
42
|
+
"chrome-profile": "google-link browser profile",
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def should_migrate() -> bool:
|
|
47
|
+
"""True only when old state exists AND the new home was never created."""
|
|
48
|
+
return OLD_CONFIG_DIR.is_dir() and not NEW_CONFIG_DIR.exists()
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def migrate_config_dir() -> Path | None:
|
|
52
|
+
"""Copy the old config home to the new one. Returns the new dir or None.
|
|
53
|
+
|
|
54
|
+
Idempotent: a second call always returns None (the new dir exists).
|
|
55
|
+
Never deletes or modifies the old directory.
|
|
56
|
+
"""
|
|
57
|
+
if not should_migrate():
|
|
58
|
+
return None
|
|
59
|
+
|
|
60
|
+
NEW_CONFIG_DIR.mkdir(parents=True, exist_ok=True)
|
|
61
|
+
|
|
62
|
+
for entry in OLD_CONFIG_DIR.iterdir():
|
|
63
|
+
if entry.name in _EXCLUDED_NAMES:
|
|
64
|
+
continue
|
|
65
|
+
target = NEW_CONFIG_DIR / entry.name
|
|
66
|
+
if target.exists(): # never overwrite anything
|
|
67
|
+
continue
|
|
68
|
+
if entry.is_dir():
|
|
69
|
+
shutil.copytree(entry, target, copy_function=shutil.copy2, dirs_exist_ok=True)
|
|
70
|
+
else:
|
|
71
|
+
shutil.copy2(entry, target)
|
|
72
|
+
|
|
73
|
+
return NEW_CONFIG_DIR
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def migration_notice() -> str | None:
|
|
77
|
+
"""Run the migration when due; return the one-line CLI notice, or None."""
|
|
78
|
+
if not should_migrate():
|
|
79
|
+
return None
|
|
80
|
+
migrated = migrate_config_dir()
|
|
81
|
+
if migrated is None: # raced with another process — nothing to announce
|
|
82
|
+
return None
|
|
83
|
+
return (
|
|
84
|
+
f"Config migrated: {OLD_CONFIG_DIR} -> {migrated} "
|
|
85
|
+
f"(old directory left in place; nothing was deleted)"
|
|
86
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import typing
|
|
4
|
+
from enum import Enum
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
from pydantic import BaseModel
|
|
8
|
+
|
|
9
|
+
from pushframe.models.asset import Asset
|
|
10
|
+
from pushframe.models.user import User
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class ReactionType(Enum):
|
|
14
|
+
THANKS = 'thanks'
|
|
15
|
+
LOVE = 'love'
|
|
16
|
+
THERE_IN_SPIRIT = 'be_there_in_spirit'
|
|
17
|
+
WELL_WISHES = 'make_a_wish'
|
|
18
|
+
HUGS_KISSES = 'hugs_and_kisses'
|
|
19
|
+
FAMILY_HUGS = 'family_hugs'
|
|
20
|
+
WELCOME = 'welcome'
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class Reaction(BaseModel):
|
|
24
|
+
activity_id: str
|
|
25
|
+
created_at: str
|
|
26
|
+
formatted_text: str
|
|
27
|
+
plain_text: str
|
|
28
|
+
type: ReactionType
|
|
29
|
+
user: User
|
|
30
|
+
user_id: str
|
|
31
|
+
id: str
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class Comment(BaseModel):
|
|
35
|
+
content: str
|
|
36
|
+
created_at: str
|
|
37
|
+
id: str
|
|
38
|
+
user_id: str
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class ActivityType(Enum):
|
|
42
|
+
ALBUM_ADDED = 'album_added'
|
|
43
|
+
ALBUM_PHOTOS_ADDED = 'album_photos_added'
|
|
44
|
+
ENTER_CLAIM_CODE = 'enter_claim_code'
|
|
45
|
+
FORCED_WIFI_CLAIMED = 'forced_wifi_frame_claimed'
|
|
46
|
+
FORCED_WIFI_CREATE = 'forced_wifi_frame_created'
|
|
47
|
+
FRAME_CREATED = 'frame_created'
|
|
48
|
+
FRAME_GIFT_CLAIMED = 'frame_gift_claimed'
|
|
49
|
+
FRAME_GIFT_CREATED = 'frame_gift_created'
|
|
50
|
+
PHOTOS_ADDED = 'photos_added'
|
|
51
|
+
USER_JOINED = 'user_joined'
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class SuggestionManifest(BaseModel):
|
|
55
|
+
local_identifier: str
|
|
56
|
+
location: list[float]
|
|
57
|
+
taken_at: str
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class Activity(BaseModel):
|
|
61
|
+
id: str
|
|
62
|
+
asset_count: int
|
|
63
|
+
comment_count: int
|
|
64
|
+
commenters: list[User]
|
|
65
|
+
created_at: str
|
|
66
|
+
formatted_text: str
|
|
67
|
+
frame_id: str
|
|
68
|
+
plain_text: str
|
|
69
|
+
playlist: typing.Any # unknown
|
|
70
|
+
playlist_id: Optional[str]
|
|
71
|
+
reactions: list[Reaction]
|
|
72
|
+
recent_comments: list[Comment]
|
|
73
|
+
representative_asset_ids: list[str]
|
|
74
|
+
type: ActivityType
|
|
75
|
+
user_id: str
|
|
76
|
+
viewable_asset_count: int
|
|
77
|
+
suggestion_manifest: Optional[list[SuggestionManifest]] = None
|
|
78
|
+
user: Optional[User] = None
|
|
79
|
+
representative_assets: Optional[list[Asset]] = None
|