pushframe 5.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pushframe/__init__.py +4 -0
- pushframe/api/__init__.py +0 -0
- pushframe/api/accountApi.py +72 -0
- pushframe/api/activityApi.py +87 -0
- pushframe/api/assetApi.py +194 -0
- pushframe/api/baseApi.py +6 -0
- pushframe/api/frameApi.py +271 -0
- pushframe/api/notificationApi.py +15 -0
- pushframe/api/peopleApi.py +25 -0
- pushframe/api/playlistApi.py +9 -0
- pushframe/aura.py +182 -0
- pushframe/aws/__init__.py +0 -0
- pushframe/aws/awsclient.py +23 -0
- pushframe/aws/s3client.py +40 -0
- pushframe/aws/sqsclient.py +33 -0
- pushframe/cache.py +50 -0
- pushframe/cli.py +1134 -0
- pushframe/client.py +267 -0
- pushframe/exif.py +147 -0
- pushframe/export.py +53 -0
- pushframe/google/__init__.py +43 -0
- pushframe/google/bootstrap.py +134 -0
- pushframe/google/cache.py +167 -0
- pushframe/google/client.py +140 -0
- pushframe/google/enumerate.py +270 -0
- pushframe/google/manifest.py +111 -0
- pushframe/google/parsers.py +345 -0
- pushframe/google/redaction.py +33 -0
- pushframe/google/vault.py +126 -0
- pushframe/gsync.py +463 -0
- pushframe/migration.py +86 -0
- pushframe/models/__init__.py +0 -0
- pushframe/models/activity.py +79 -0
- pushframe/models/asset.py +159 -0
- pushframe/models/frame.py +105 -0
- pushframe/models/meta.py +11 -0
- pushframe/models/person.py +24 -0
- pushframe/models/user.py +22 -0
- pushframe/ratelimit.py +222 -0
- pushframe/reconcile.py +384 -0
- pushframe/sync.py +1105 -0
- pushframe/utils/dt.py +15 -0
- pushframe/utils/io.py +23 -0
- pushframe/utils/settings.py +59 -0
- pushframe-5.0.0.dist-info/METADATA +53 -0
- pushframe-5.0.0.dist-info/RECORD +49 -0
- pushframe-5.0.0.dist-info/WHEEL +4 -0
- pushframe-5.0.0.dist-info/entry_points.txt +2 -0
- pushframe-5.0.0.dist-info/licenses/LICENSE +31 -0
pushframe/client.py
ADDED
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
import copy
|
|
2
|
+
from collections import deque
|
|
3
|
+
from typing import Optional, Deque
|
|
4
|
+
|
|
5
|
+
import httpx
|
|
6
|
+
from httpx import Response, Timeout
|
|
7
|
+
from loguru import logger
|
|
8
|
+
|
|
9
|
+
AURA_API_BASE_URL = 'https://api.pushd.com'
|
|
10
|
+
AURA_API_VERSION = 'v5'
|
|
11
|
+
USER_AGENT = 'Aura/4.7.790 (Android 30; Client)'
|
|
12
|
+
|
|
13
|
+
# Keys whose values are secrets and must never reach the on-disk logs (D-07).
|
|
14
|
+
_REDACT_KEYS = {'password', 'auth_token', 'x-token-auth'}
|
|
15
|
+
_REDACTED = '***REDACTED***'
|
|
16
|
+
|
|
17
|
+
# Status codes that mean "the server is throttling / has locked out this
|
|
18
|
+
# account" rather than a per-request client error. 429 is the standard
|
|
19
|
+
# Too Many Requests; 475 is Pushd's non-standard code observed during the
|
|
20
|
+
# select-asset-401-unauthorized debug session — returned with valid
|
|
21
|
+
# credentials once the account's write burst tripped the anti-abuse layer,
|
|
22
|
+
# and it escalates to reject login too. Both are treated as a single
|
|
23
|
+
# back-off-and-stop signal (see RateLimitError) so a tripped batch aborts
|
|
24
|
+
# with one clear message instead of N misleading per-item 401s.
|
|
25
|
+
_RATE_LIMIT_STATUS_CODES = {429, 475}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class AuraError(Exception):
|
|
29
|
+
"""Package-wide exception root for pushframe's own control-flow types
|
|
30
|
+
(MOD-03/D-20). This is a MARKER base, not a codebase-wide taxonomy --
|
|
31
|
+
most of this codebase still raises bare ``RuntimeError``/``ValueError``
|
|
32
|
+
deliberately (MOD-03 scopes typed-exception conversion to the write
|
|
33
|
+
path only, "where it pays", not a full rewrite).
|
|
34
|
+
|
|
35
|
+
Catching ``AuraError`` broadly is deliberately discouraged and, inside
|
|
36
|
+
`pushframe/sync.py`, actively prohibited: `execute_plan`'s write-loop
|
|
37
|
+
except-ladder depends on `RateLimitError` and
|
|
38
|
+
`ConsecutiveWriteFailureError` each being matched by their OWN specific
|
|
39
|
+
branch before any broader handler could see them. An `except AuraError`
|
|
40
|
+
introduced there would silently swallow both and break that ordering.
|
|
41
|
+
"""
|
|
42
|
+
pass
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class AuthenticationError(AuraError):
|
|
46
|
+
"""Raised by `execute_plan`'s write-chunk 401 retry (REL-01/REL-04,
|
|
47
|
+
D-01/D-02) when a write returns HTTP 401, a re-login is attempted to
|
|
48
|
+
discriminate a genuine authentication failure from an anti-abuse trip,
|
|
49
|
+
and that re-login itself raises.
|
|
50
|
+
|
|
51
|
+
This is a HARD STOP, never a transient/retryable condition: two write
|
|
52
|
+
401s in a row would be ambiguous on their own, but a re-login that
|
|
53
|
+
*itself* fails means the credentials are bad or revoked. The write is
|
|
54
|
+
never retried and the failure is never re-labelled transient.
|
|
55
|
+
|
|
56
|
+
Carries `underlying` -- the exception the re-login raised (or `None`)
|
|
57
|
+
-- for callers/logs. The message names the failure class and
|
|
58
|
+
`type(underlying).__name__` ONLY; it must never interpolate a response
|
|
59
|
+
body, an auth token, or the password (T-11-02).
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
def __init__(self, underlying: Exception | None = None):
|
|
63
|
+
self.underlying = underlying
|
|
64
|
+
underlying_name = type(underlying).__name__ if underlying is not None else 'unknown error'
|
|
65
|
+
super().__init__(
|
|
66
|
+
f'Re-login failed after a write returned HTTP 401 -- treating this as a '
|
|
67
|
+
f'genuine authentication failure (bad or revoked credentials), not a '
|
|
68
|
+
f'transient/retryable anti-abuse trip. Underlying: {underlying_name}.'
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class WriteEndpointError(AuraError):
|
|
73
|
+
"""The write-path form of an Aura error-envelope response (MOD-03/D-20):
|
|
74
|
+
the call reached the server, the server answered HTTP 200, and the body
|
|
75
|
+
carried an `error` key -- distinct from a network/HTTP-status failure.
|
|
76
|
+
|
|
77
|
+
Converted from exactly two bare `RuntimeError`s in
|
|
78
|
+
`pushframe/api/assetApi.py` (`batch_update`, `delete_asset`). Every
|
|
79
|
+
other write-endpoint error envelope (`select_asset`/`exclude_asset`/
|
|
80
|
+
`remove_asset` in `pushframe/api/frameApi.py`) and every read-path
|
|
81
|
+
raise is deliberately left as `RuntimeError` this phase -- D-20 scopes
|
|
82
|
+
conversion narrowly so a phase about the write path's *trustworthiness*
|
|
83
|
+
does not also become the widest-blast-radius rewrite of its *type
|
|
84
|
+
surface*.
|
|
85
|
+
"""
|
|
86
|
+
pass
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class RateLimitError(AuraError):
|
|
90
|
+
"""Raised when the Aura/Pushd API signals rate-limiting or an account
|
|
91
|
+
lockout (HTTP 429 or the custom 475).
|
|
92
|
+
|
|
93
|
+
Carries the offending ``status_code`` and, when the server provided a
|
|
94
|
+
``Retry-After`` header, ``retry_after`` (an int number of seconds when
|
|
95
|
+
the header was numeric, otherwise the raw header string — e.g. an
|
|
96
|
+
HTTP-date). Distinct from ``httpx.HTTPStatusError`` so callers can
|
|
97
|
+
abort a whole batch and surface a single "back off" message rather than
|
|
98
|
+
treating it as one of many per-item failures.
|
|
99
|
+
"""
|
|
100
|
+
|
|
101
|
+
def __init__(self, status_code: int, retry_after=None, server_message: str | None = None):
|
|
102
|
+
self.status_code = status_code
|
|
103
|
+
self.retry_after = retry_after
|
|
104
|
+
self.server_message = server_message
|
|
105
|
+
|
|
106
|
+
detail = (
|
|
107
|
+
f' Retry after {retry_after}s.'
|
|
108
|
+
if isinstance(retry_after, int)
|
|
109
|
+
else (f' Retry-After: {retry_after}.' if retry_after else '')
|
|
110
|
+
)
|
|
111
|
+
server = f' Server said: {server_message}.' if server_message else ''
|
|
112
|
+
super().__init__(
|
|
113
|
+
f'Aura API is rate-limiting or has locked out this account '
|
|
114
|
+
f'(HTTP {status_code}). Stop and back off before retrying; '
|
|
115
|
+
f'continued calls may extend the lockout.{detail}{server}'
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _parse_retry_after(raw: str | None):
|
|
120
|
+
"""Parse a ``Retry-After`` header value.
|
|
121
|
+
|
|
122
|
+
Returns an ``int`` when the header is a plain number of seconds, the
|
|
123
|
+
stripped raw string when it is an HTTP-date (or otherwise non-numeric),
|
|
124
|
+
or ``None`` when the header is absent. Stdlib only — no date parsing is
|
|
125
|
+
attempted; a non-numeric value is surfaced verbatim for the human.
|
|
126
|
+
"""
|
|
127
|
+
if raw is None:
|
|
128
|
+
return None
|
|
129
|
+
raw = raw.strip()
|
|
130
|
+
if not raw:
|
|
131
|
+
return None
|
|
132
|
+
return int(raw) if raw.isdigit() else raw
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _redact(value):
|
|
136
|
+
"""Return a deep copy of a dict/list with secret-bearing keys masked.
|
|
137
|
+
|
|
138
|
+
Recurses into nested dicts (e.g. the ``user`` sub-dict of the login payload)
|
|
139
|
+
and lists so a secret can never leak from a deeper level. Non-container
|
|
140
|
+
values are returned unchanged. Stdlib only.
|
|
141
|
+
"""
|
|
142
|
+
if isinstance(value, dict):
|
|
143
|
+
return {
|
|
144
|
+
k: (_REDACTED if k in _REDACT_KEYS else _redact(v))
|
|
145
|
+
for k, v in value.items()
|
|
146
|
+
}
|
|
147
|
+
if isinstance(value, list):
|
|
148
|
+
return [_redact(item) for item in value]
|
|
149
|
+
return copy.copy(value)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
# Use something similar to:
|
|
153
|
+
# https://github.com/sudoguy/tiktokpy/blob/master/tiktokpy/client/__init__.py
|
|
154
|
+
# https://github.com/mkb79/Audible/tree/master/src/audible
|
|
155
|
+
# https://github.com/ssut/py-googletrans/blob/master/googletrans/client.py
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
# TODO: This should be reworked to be async, particularly for mass uploads/clones.
|
|
159
|
+
|
|
160
|
+
class Client:
|
|
161
|
+
|
|
162
|
+
def __init__(self, history_len: int = 30, transport: httpx.BaseTransport | None = None,
|
|
163
|
+
base_url: str | None = None):
|
|
164
|
+
# TEST-01 candidate #4 (Phase 19): base_url is injectable for offline
|
|
165
|
+
# tests; None keeps the historical composition byte-identical.
|
|
166
|
+
resolved_base_url = base_url or f'{AURA_API_BASE_URL}/{AURA_API_VERSION}'
|
|
167
|
+
self.http2_client = httpx.Client(http2=True, base_url=resolved_base_url, headers={
|
|
168
|
+
'accept-language': 'en-US',
|
|
169
|
+
'cache-control': 'no-cache',
|
|
170
|
+
'user-agent': USER_AGENT,
|
|
171
|
+
'content-type': 'application/json; charset=utf-8',
|
|
172
|
+
}, timeout=Timeout(timeout=20.0), transport=transport)
|
|
173
|
+
|
|
174
|
+
self.history: Deque[Response] = deque(maxlen=history_len)
|
|
175
|
+
|
|
176
|
+
def get(self, url, query_params: Optional[dict] = None, headers: Optional[dict] = None):
|
|
177
|
+
query_params = {k: v for k, v in query_params.items() if v is not None} if query_params else None
|
|
178
|
+
logger.info(f'GET request to {url}', query_params=query_params, headers=headers)
|
|
179
|
+
response = self.http2_client.get(url=url, params=query_params, headers=headers)
|
|
180
|
+
|
|
181
|
+
self.history.append(response)
|
|
182
|
+
self._raise_if_rate_limited(response)
|
|
183
|
+
response.raise_for_status()
|
|
184
|
+
logger.debug(f'Response ({response.status_code}), body: {_redact(response.json())}')
|
|
185
|
+
|
|
186
|
+
self._set_cookies(response)
|
|
187
|
+
|
|
188
|
+
return response.json()
|
|
189
|
+
|
|
190
|
+
def post(self, url, data: dict = None, query_params: Optional[dict] = None, headers: Optional[dict] = None):
|
|
191
|
+
logger.info(f'POST request to {url}', data=_redact(data), query_params=query_params, headers=headers)
|
|
192
|
+
response = self.http2_client.post(url=url, json=data, headers=headers, params=query_params)
|
|
193
|
+
|
|
194
|
+
self.history.append(response)
|
|
195
|
+
self._raise_if_rate_limited(response)
|
|
196
|
+
response.raise_for_status()
|
|
197
|
+
logger.debug(f'Response ({response.status_code}), body: {_redact(response.json())}')
|
|
198
|
+
|
|
199
|
+
self._set_cookies(response)
|
|
200
|
+
|
|
201
|
+
return response.json()
|
|
202
|
+
|
|
203
|
+
def delete(self, url, query_params: Optional[dict] = None, headers: Optional[dict] = None):
|
|
204
|
+
logger.info(f'DELETE request to {url}', query_params=query_params, headers=headers)
|
|
205
|
+
response = self.http2_client.delete(url=url, headers=headers, params=query_params)
|
|
206
|
+
|
|
207
|
+
self.history.append(response)
|
|
208
|
+
self._raise_if_rate_limited(response)
|
|
209
|
+
response.raise_for_status()
|
|
210
|
+
logger.debug(f'Response ({response.status_code}), body: {_redact(response.json())}')
|
|
211
|
+
|
|
212
|
+
self._set_cookies(response)
|
|
213
|
+
|
|
214
|
+
return response.json()
|
|
215
|
+
|
|
216
|
+
def put(self, url, data: dict = None, query_params: Optional[dict] = None, headers: Optional[dict] = None):
|
|
217
|
+
logger.info(f'PUT request to {url}', data=_redact(data), query_params=query_params, headers=headers)
|
|
218
|
+
response = self.http2_client.put(url=url, json=data, headers=headers, params=query_params)
|
|
219
|
+
|
|
220
|
+
self.history.append(response)
|
|
221
|
+
self._raise_if_rate_limited(response)
|
|
222
|
+
response.raise_for_status()
|
|
223
|
+
logger.debug(f'Response ({response.status_code}), body: {_redact(response.json())}')
|
|
224
|
+
|
|
225
|
+
self._set_cookies(response)
|
|
226
|
+
|
|
227
|
+
return response.json()
|
|
228
|
+
|
|
229
|
+
def _raise_if_rate_limited(self, response: httpx.Response) -> None:
|
|
230
|
+
"""Convert a rate-limit / lockout response (HTTP 429 or 475) into a
|
|
231
|
+
`RateLimitError` before the generic `raise_for_status()` runs.
|
|
232
|
+
|
|
233
|
+
Runs on every request method (read and write) so a throttle that
|
|
234
|
+
first appears on a GET is classified just as clearly as one on a
|
|
235
|
+
write. Reads the server's ``message`` body field (best-effort) and
|
|
236
|
+
the ``Retry-After`` header so the raised error can tell the caller
|
|
237
|
+
how long to wait.
|
|
238
|
+
"""
|
|
239
|
+
if response.status_code not in _RATE_LIMIT_STATUS_CODES:
|
|
240
|
+
return
|
|
241
|
+
|
|
242
|
+
retry_after = _parse_retry_after(response.headers.get('retry-after'))
|
|
243
|
+
server_message = None
|
|
244
|
+
try:
|
|
245
|
+
body = response.json()
|
|
246
|
+
if isinstance(body, dict):
|
|
247
|
+
server_message = body.get('message')
|
|
248
|
+
except Exception:
|
|
249
|
+
# A rate-limit response with a non-JSON body must still raise a
|
|
250
|
+
# clean RateLimitError, never a JSON-decode error.
|
|
251
|
+
server_message = None
|
|
252
|
+
|
|
253
|
+
logger.warning(
|
|
254
|
+
f'Rate-limited/locked-out response (HTTP {response.status_code}) '
|
|
255
|
+
f'from {response.request.url}; aborting.'
|
|
256
|
+
)
|
|
257
|
+
raise RateLimitError(response.status_code, retry_after, server_message)
|
|
258
|
+
|
|
259
|
+
def add_default_headers(self, headers: dict) -> None:
|
|
260
|
+
self.http2_client.headers.update(headers)
|
|
261
|
+
|
|
262
|
+
def _set_cookies(self, response: httpx.Response) -> None:
|
|
263
|
+
if len(response.cookies):
|
|
264
|
+
logger.debug(f'Response Cookies: {response.cookies}')
|
|
265
|
+
|
|
266
|
+
for cookie_name, cookie_data in response.cookies.items():
|
|
267
|
+
self.http2_client.cookies.set(cookie_name, cookie_data)
|
pushframe/exif.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
from datetime import datetime
|
|
2
|
+
from geopy import Nominatim
|
|
3
|
+
import io
|
|
4
|
+
import piexif
|
|
5
|
+
import piexif.helper
|
|
6
|
+
from fractions import Fraction
|
|
7
|
+
import json
|
|
8
|
+
import time
|
|
9
|
+
from loguru import logger
|
|
10
|
+
|
|
11
|
+
from pushframe.models.asset import Asset
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
# Most of the exif writing is from:
|
|
15
|
+
# https://gitlab.com/searchwing/development/payloads/ros-generic/-/blob/master/searchwing_common_py/scripts/ImageSaverNode.py
|
|
16
|
+
|
|
17
|
+
def build_gps_ifd(location_dms: tuple[any, any]):
|
|
18
|
+
if not location_dms:
|
|
19
|
+
return {}
|
|
20
|
+
|
|
21
|
+
return {
|
|
22
|
+
piexif.GPSIFD.GPSVersionID: (2, 3, 0, 0),
|
|
23
|
+
piexif.GPSIFD.GPSLatitudeRef: location_dms[0][3],
|
|
24
|
+
piexif.GPSIFD.GPSLatitude: location_dms[0][:-1],
|
|
25
|
+
piexif.GPSIFD.GPSLongitudeRef: location_dms[1][3],
|
|
26
|
+
piexif.GPSIFD.GPSLongitude: location_dms[1][:-1],
|
|
27
|
+
piexif.GPSIFD.GPSAltitudeRef: 0,
|
|
28
|
+
piexif.GPSIFD.GPSAltitude: (0, 1),
|
|
29
|
+
piexif.GPSIFD.GPSStatus: b'A'
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class ExifWriter:
|
|
34
|
+
geolocator = Nominatim(user_agent="pushframe-python-client/1.0")
|
|
35
|
+
cache = {}
|
|
36
|
+
|
|
37
|
+
# TODO: LRU Cache would be nice but probably over-engineered
|
|
38
|
+
def _lookup_gps(self, location_name: str):
|
|
39
|
+
location_dms = self.cache.get(location_name)
|
|
40
|
+
if location_dms:
|
|
41
|
+
return location_dms
|
|
42
|
+
|
|
43
|
+
try:
|
|
44
|
+
location = self.geolocator.geocode(location_name)
|
|
45
|
+
except Exception:
|
|
46
|
+
# GPS is conditional (D-09): a geocode failure is tolerable — log
|
|
47
|
+
# and skip GPS rather than failing the whole download.
|
|
48
|
+
logger.info(f'Failed to read GPS data for {location_name}')
|
|
49
|
+
return None
|
|
50
|
+
|
|
51
|
+
if not location:
|
|
52
|
+
return None
|
|
53
|
+
|
|
54
|
+
longitude_dms = convert_to_rational_dms(to_deg(location.longitude, is_longitude=True))
|
|
55
|
+
latitude_dms = convert_to_rational_dms(to_deg(location.latitude, is_longitude=False))
|
|
56
|
+
|
|
57
|
+
self.cache[location_name] = (longitude_dms, latitude_dms)
|
|
58
|
+
return longitude_dms, latitude_dms
|
|
59
|
+
|
|
60
|
+
def write_exif(self, image, asset: Asset, thumbnail=None, set_gps_ifd=True):
|
|
61
|
+
taken_datetime = asset.taken_at_dt.strftime('%Y:%m:%d %H:%M:%S').encode()
|
|
62
|
+
|
|
63
|
+
exif_dict = {
|
|
64
|
+
"Exif": {
|
|
65
|
+
piexif.ExifIFD.DateTimeOriginal: taken_datetime,
|
|
66
|
+
piexif.ExifIFD.DateTimeDigitized: taken_datetime,
|
|
67
|
+
piexif.ExifIFD.OffsetTime: b'-05:00',
|
|
68
|
+
piexif.ExifIFD.OffsetTimeOriginal: b'-05:00',
|
|
69
|
+
|
|
70
|
+
},
|
|
71
|
+
'0th': {
|
|
72
|
+
piexif.ImageIFD.DateTime: taken_datetime,
|
|
73
|
+
piexif.ImageIFD.Artist: asset.user.name if asset.user else None
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
if set_gps_ifd:
|
|
78
|
+
location_dms = self._lookup_gps(asset.location_name)
|
|
79
|
+
exif_dict['GPS'] = build_gps_ifd(location_dms)
|
|
80
|
+
|
|
81
|
+
if thumbnail:
|
|
82
|
+
exif_dict['thumbnail'] = thumbnail
|
|
83
|
+
exif_dict['1st'] = {piexif.ImageIFD.Make: u"Canon",
|
|
84
|
+
piexif.ImageIFD.XResolution: (40, 1),
|
|
85
|
+
piexif.ImageIFD.YResolution: (40, 1),
|
|
86
|
+
piexif.ImageIFD.Software: u"piexif"
|
|
87
|
+
}
|
|
88
|
+
new_imag = io.BytesIO()
|
|
89
|
+
exif_bytes = piexif.dump(exif_dict)
|
|
90
|
+
|
|
91
|
+
try:
|
|
92
|
+
piexif.insert(exif_bytes, image, new_imag)
|
|
93
|
+
except Exception:
|
|
94
|
+
# A corrupt/non-JPEG body must fail loudly (D-11): re-raise instead
|
|
95
|
+
# of returning an empty BytesIO that would be saved as a 0-byte file.
|
|
96
|
+
logger.error('Failed to write EXIF to image.')
|
|
97
|
+
raise
|
|
98
|
+
return new_imag
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def change_to_rational(number):
|
|
102
|
+
f = Fraction(str(number))
|
|
103
|
+
return f.numerator, f.denominator
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def convert_to_rational_dms(dms: tuple[int, int, float, str]):
|
|
107
|
+
return change_to_rational(dms[0]), change_to_rational(dms[1]), change_to_rational(dms[2]), dms[3]
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def clone_exif(original_path, clone_path):
|
|
111
|
+
piexif.transplant(original_path, clone_path)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def get_readable_exif(image_path):
|
|
115
|
+
exif_dict = piexif.load(image_path)
|
|
116
|
+
readable_dict = {}
|
|
117
|
+
for ifd in exif_dict:
|
|
118
|
+
readable_dict[ifd] = {}
|
|
119
|
+
if not exif_dict[ifd]:
|
|
120
|
+
continue
|
|
121
|
+
for tag in exif_dict[ifd]:
|
|
122
|
+
if ifd != 'thumbnail':
|
|
123
|
+
readable_dict[ifd][piexif.TAGS[ifd][tag]["name"]] = exif_dict[ifd][tag]
|
|
124
|
+
else:
|
|
125
|
+
readable_dict[ifd][tag] = exif_dict[ifd][tag]
|
|
126
|
+
return readable_dict
|
|
127
|
+
|
|
128
|
+
def to_deg(value, is_longitude):
|
|
129
|
+
# convert decimal coordinates into degrees, minutes and seconds tuple
|
|
130
|
+
# Keyword arguments:
|
|
131
|
+
# value is float gps-value,
|
|
132
|
+
# loc is direction list ["S", "N"] or ["W", "E"]
|
|
133
|
+
# return: tuple like (25, 13, 48.343 ,'N')
|
|
134
|
+
loc = ["W", "E"] if is_longitude else ["S", "N"]
|
|
135
|
+
|
|
136
|
+
if value < 0:
|
|
137
|
+
loc_value = loc[0]
|
|
138
|
+
elif value > 0:
|
|
139
|
+
loc_value = loc[1]
|
|
140
|
+
else:
|
|
141
|
+
loc_value = ""
|
|
142
|
+
abs_value = abs(value)
|
|
143
|
+
deg = int(abs_value)
|
|
144
|
+
t1 = (abs_value - deg) * 60
|
|
145
|
+
min = int(t1)
|
|
146
|
+
sec = round((t1 - min) * 60, 5)
|
|
147
|
+
return deg, min, sec, loc_value
|
pushframe/export.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import shutil
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
from io import BytesIO
|
|
5
|
+
from loguru import logger
|
|
6
|
+
from PIL import Image, UnidentifiedImageError
|
|
7
|
+
import httpx
|
|
8
|
+
|
|
9
|
+
from pushframe.exif import ExifWriter
|
|
10
|
+
from pushframe.models.asset import Asset
|
|
11
|
+
from pushframe.utils import settings
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _get_path_safe_datetime(date_str: datetime):
|
|
15
|
+
return date_str.strftime('%Y%m%dT%H%M%S')
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def get_thumbnail(asset: Asset, original_image: BytesIO = None):
|
|
19
|
+
"""
|
|
20
|
+
Aura thumbnails can be saved incorrectly, we can generate our own if we have the original image.
|
|
21
|
+
TODO: We could also check the other fields on `Asset` to see if there are smaller images instead of using PIL.
|
|
22
|
+
"""
|
|
23
|
+
thumbnail_response = httpx.get(asset.thumbnail_url)
|
|
24
|
+
thumbnail_bytes = BytesIO(thumbnail_response.content)
|
|
25
|
+
try:
|
|
26
|
+
with Image.open(thumbnail_bytes) as http_thumbnail:
|
|
27
|
+
http_thumbnail.verify()
|
|
28
|
+
except UnidentifiedImageError:
|
|
29
|
+
if not original_image:
|
|
30
|
+
return None
|
|
31
|
+
|
|
32
|
+
with Image.open(original_image) as pil_image:
|
|
33
|
+
out_bytes = BytesIO()
|
|
34
|
+
pil_image.thumbnail((100, 100))
|
|
35
|
+
pil_image.save(out_bytes, 'jpeg')
|
|
36
|
+
return out_bytes.getvalue()
|
|
37
|
+
|
|
38
|
+
return thumbnail_bytes.getvalue()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def get_image_from_asset(asset: Asset, path: str, exif_writer: ExifWriter = None, ignore_cache=False):
|
|
42
|
+
new_filename = os.path.join(path, f'{_get_path_safe_datetime(asset.taken_at_dt)}-{asset.file_name}')
|
|
43
|
+
if (os.path.isfile(new_filename) and not ignore_cache):
|
|
44
|
+
with open(new_filename, 'rb') as in_file:
|
|
45
|
+
return in_file.read()
|
|
46
|
+
original_image_bytes = httpx.get(f'{settings.IMAGE_PROXY_BASE_URL}/{asset.user_id}/{asset.file_name}').content
|
|
47
|
+
|
|
48
|
+
thumbnail = get_thumbnail(asset, BytesIO(original_image_bytes)) if exif_writer else None
|
|
49
|
+
image = exif_writer.write_exif(original_image_bytes, asset, thumbnail)
|
|
50
|
+
|
|
51
|
+
with open(new_filename, 'wb') as out:
|
|
52
|
+
shutil.copyfileobj(image, out)
|
|
53
|
+
return original_image_bytes
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""Google Photos integration package (phase 17 — LGS-02..05, TEST-02).
|
|
2
|
+
|
|
3
|
+
The phase-16-proven mechanics, migrated from the `probes/` instruments into
|
|
4
|
+
production shape:
|
|
5
|
+
|
|
6
|
+
- `GoogleSession` (`client`) — full-cookie-jar httpx client over
|
|
7
|
+
photos.google.com, built from the 0600 cookie vault (`vault`, denylist
|
|
8
|
+
boundary carried over verbatim), with the harvesting browser's UA;
|
|
9
|
+
- parsers (`parsers`) — share-page ds:1 and snAcKc RPC payloads (shared §1b
|
|
10
|
+
item shape), batchexecute envelope splitting, fail-loud everywhere;
|
|
11
|
+
- enumeration (`enumerate`) — the snAcKc continuation loop (300/page, AH_
|
|
12
|
+
token swap, clean exhaustion) and the disk-weight measurer (1-byte Range
|
|
13
|
+
GETs → Content-Range totals);
|
|
14
|
+
- redaction (`redaction`) — every printable URL goes through the helpers.
|
|
15
|
+
|
|
16
|
+
Everything Google-facing is offline-tested through injected transports
|
|
17
|
+
(TEST-02): zero live network in the test suite.
|
|
18
|
+
"""
|
|
19
|
+
from pushframe.google.client import GoogleSession, GoogleSessionError
|
|
20
|
+
from pushframe.google.parsers import (
|
|
21
|
+
BatchexecuteEntry,
|
|
22
|
+
ProbeParseError,
|
|
23
|
+
SnackcPage,
|
|
24
|
+
parse_af_initdata,
|
|
25
|
+
parse_batchexecute,
|
|
26
|
+
parse_snackc_payload,
|
|
27
|
+
)
|
|
28
|
+
from pushframe.google.redaction import redact_link, redact_tokens
|
|
29
|
+
from pushframe.google.vault import CookieVaultError
|
|
30
|
+
|
|
31
|
+
__all__ = [
|
|
32
|
+
"GoogleSession",
|
|
33
|
+
"GoogleSessionError",
|
|
34
|
+
"CookieVaultError",
|
|
35
|
+
"ProbeParseError",
|
|
36
|
+
"SnackcPage",
|
|
37
|
+
"BatchexecuteEntry",
|
|
38
|
+
"parse_af_initdata",
|
|
39
|
+
"parse_snackc_payload",
|
|
40
|
+
"parse_batchexecute",
|
|
41
|
+
"redact_link",
|
|
42
|
+
"redact_tokens",
|
|
43
|
+
]
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"""Interactive Google bootstrap: dedicated-profile browser harvest.
|
|
2
|
+
|
|
3
|
+
Migrated from probes/browser_bootstrap.py's `_bootstrap` (phase 16,
|
|
4
|
+
live-proven). The CLI's `google-link` drives it through a `bootstrap_fn`
|
|
5
|
+
seam so offline tests inject a fake and never launch a browser (TEST-02).
|
|
6
|
+
|
|
7
|
+
Posture carried over (T-16-06): the DAILY-DRIVER profile is structurally
|
|
8
|
+
unreachable — the dedicated profile directory must be named by
|
|
9
|
+
`AURA_PROBE_CHROME_PROFILE` before any browser exists; unset/empty fails
|
|
10
|
+
loud with the exact remedy. Playwright is imported lazily INSIDE
|
|
11
|
+
`run_bootstrap` only — importing this module never requires playwright
|
|
12
|
+
(BROWSER-AUTOMATION §4.2 isolation rule).
|
|
13
|
+
|
|
14
|
+
Output discipline (T-16-07): identity signals only — cookie NAMES and
|
|
15
|
+
counts, never values; the vault path is printed, never its contents.
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import os
|
|
20
|
+
import shutil
|
|
21
|
+
import time
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
from pushframe.google import vault
|
|
25
|
+
from pushframe.google.client import PHOTOS_HOME
|
|
26
|
+
|
|
27
|
+
PROFILE_ENV_VAR = "PUSHFRAME_PROBE_CHROME_PROFILE"
|
|
28
|
+
# IDN-04: the AURA_* spelling stays readable for one release of grace.
|
|
29
|
+
_LEGACY_PROFILE_ENV_VAR = "AURA_PROBE_CHROME_PROFILE"
|
|
30
|
+
|
|
31
|
+
_AUTO_WAIT_SECONDS = 30 * 60 # the operator may log in much later
|
|
32
|
+
|
|
33
|
+
# Identity-signal cookie names (names only, never values).
|
|
34
|
+
_AUTH_MARKER_NAMES = ("SID", "SAPISID", "__Secure-1PSID", "__Secure-3PSID",
|
|
35
|
+
"LSID", "__Secure-1PAPISID", "__Secure-3PAPISID")
|
|
36
|
+
|
|
37
|
+
# Login-completion markers for auto-detect.
|
|
38
|
+
_AUTH_COMPLETION_MARKERS = {"SAPISID", "__Secure-1PAPISID", "__Secure-3PAPISID"}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class BootstrapError(RuntimeError):
|
|
42
|
+
"""Bootstrap prerequisites missing or the login never completed."""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _require_profile() -> Path:
|
|
46
|
+
"""Resolve the dedicated profile directory; fail loud when unset (T-16-06)."""
|
|
47
|
+
profile = os.environ.get(PROFILE_ENV_VAR, "").strip()
|
|
48
|
+
if not profile:
|
|
49
|
+
profile = os.environ.get(_LEGACY_PROFILE_ENV_VAR, "").strip()
|
|
50
|
+
if not profile:
|
|
51
|
+
raise BootstrapError(
|
|
52
|
+
f"{PROFILE_ENV_VAR} is unset — the daily-driver profile is "
|
|
53
|
+
f"structurally unreachable; point the env var at a dedicated "
|
|
54
|
+
f"profile directory (e.g. ~/.config/pushframe/chrome-profile) "
|
|
55
|
+
f"and re-run (T-16-06)"
|
|
56
|
+
)
|
|
57
|
+
path = Path(profile).expanduser().resolve()
|
|
58
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
59
|
+
return path
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def run_bootstrap(*, auto: bool = False) -> dict:
|
|
63
|
+
"""Open the dedicated-profile browser, harvest cookies, save the vault.
|
|
64
|
+
|
|
65
|
+
auto=True polls the context for completed Google auth (SAPISID-family
|
|
66
|
+
cookies) instead of waiting for an Enter in the terminal — for runs
|
|
67
|
+
launched from a non-interactive shell. Returns an identity summary dict:
|
|
68
|
+
{vault_path, cookie_count, auth_markers} — never cookie values.
|
|
69
|
+
"""
|
|
70
|
+
profile_dir = _require_profile()
|
|
71
|
+
from playwright.sync_api import sync_playwright # lazy, in-function import ONLY
|
|
72
|
+
|
|
73
|
+
print(f"dedicated profile: {profile_dir}")
|
|
74
|
+
print("opening browser — log into Google in the window.")
|
|
75
|
+
# Anti-bot-detection posture (BROWSER-AUTOMATION §2): channel="chrome"
|
|
76
|
+
# uses the REAL installed Google Chrome when present (Google's risk
|
|
77
|
+
# engine already treats it as an ordinary browser); otherwise bundled
|
|
78
|
+
# Chromium with automation switches masked so navigator.webdriver stays
|
|
79
|
+
# false.
|
|
80
|
+
has_chrome = shutil.which("google-chrome") is not None
|
|
81
|
+
launch_kwargs = dict(user_data_dir=str(profile_dir), headless=False,
|
|
82
|
+
args=["--disable-blink-features=AutomationControlled"])
|
|
83
|
+
if has_chrome:
|
|
84
|
+
launch_kwargs["channel"] = "chrome"
|
|
85
|
+
print("browser: system Google Chrome (channel=chrome)")
|
|
86
|
+
else:
|
|
87
|
+
launch_kwargs["ignore_default_args"] = ["--enable-automation"]
|
|
88
|
+
print("browser: bundled Chromium (automation switches masked)")
|
|
89
|
+
with sync_playwright() as p:
|
|
90
|
+
context = p.chromium.launch_persistent_context(**launch_kwargs)
|
|
91
|
+
page = context.pages[0] if context.pages else context.new_page()
|
|
92
|
+
page.goto(PHOTOS_HOME, wait_until="domcontentloaded")
|
|
93
|
+
if auto:
|
|
94
|
+
print(f"waiting for login to complete (auto-detect, "
|
|
95
|
+
f"{_AUTO_WAIT_SECONDS // 60} min timeout)…", flush=True)
|
|
96
|
+
deadline = time.monotonic() + _AUTO_WAIT_SECONDS
|
|
97
|
+
auth_names: set[str] = set()
|
|
98
|
+
while time.monotonic() < deadline:
|
|
99
|
+
cookies = context.cookies()
|
|
100
|
+
auth_names = {c["name"] for c in cookies
|
|
101
|
+
if c["name"] in _AUTH_COMPLETION_MARKERS}
|
|
102
|
+
if auth_names:
|
|
103
|
+
break
|
|
104
|
+
time.sleep(2)
|
|
105
|
+
if not auth_names:
|
|
106
|
+
context.close()
|
|
107
|
+
raise BootstrapError(
|
|
108
|
+
"no auth cookies detected within the wait window — was the "
|
|
109
|
+
"login completed in the opened window?"
|
|
110
|
+
)
|
|
111
|
+
print(f"login detected via {sorted(auth_names)} — harvesting", flush=True)
|
|
112
|
+
cookies = context.cookies()
|
|
113
|
+
context.close()
|
|
114
|
+
else:
|
|
115
|
+
input("…press Enter here AFTER logging in on the opened window> ")
|
|
116
|
+
cookies = context.cookies()
|
|
117
|
+
context.close()
|
|
118
|
+
|
|
119
|
+
if not any(c["name"] in ("SID", "SAPISID", "__Secure-1PSID") for c in cookies):
|
|
120
|
+
raise BootstrapError(
|
|
121
|
+
"no Google session cookies found after login — was the login "
|
|
122
|
+
"completed in the opened window?"
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
vault_path = vault.save(cookies)
|
|
126
|
+
markers = sorted({c["name"] for c in cookies if c["name"] in _AUTH_MARKER_NAMES})
|
|
127
|
+
return {"vault_path": vault_path, "cookie_count": len(cookies),
|
|
128
|
+
"auth_markers": markers}
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def default_bootstrap(*, auto: bool = True) -> dict:
|
|
132
|
+
"""The CLI's default `bootstrap_fn`: interactive harvest with auto-detect
|
|
133
|
+
(the non-interactive-friendly posture the phase-16 probe settled on)."""
|
|
134
|
+
return run_bootstrap(auto=auto)
|