skapi 2.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- skapi/__init__.py +15 -0
- skapi/_cognito.py +386 -0
- skapi/_database.py +1454 -0
- skapi/_encryption.py +1359 -0
- skapi/_forward.py +484 -0
- skapi/_http.py +373 -0
- skapi/_md5.py +94 -0
- skapi/_project.py +83 -0
- skapi/_realtime.py +549 -0
- skapi/_social.py +323 -0
- skapi/_srp.py +126 -0
- skapi/_user.py +880 -0
- skapi/_ws.py +176 -0
- skapi/client.py +270 -0
- skapi/errors.py +26 -0
- skapi-2.4.0.dist-info/METADATA +46 -0
- skapi-2.4.0.dist-info/RECORD +20 -0
- skapi-2.4.0.dist-info/WHEEL +5 -0
- skapi-2.4.0.dist-info/licenses/LICENSE +201 -0
- skapi-2.4.0.dist-info/top_level.txt +1 -0
skapi/_database.py
ADDED
|
@@ -0,0 +1,1454 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Database methods, ported from skapi-js (src/methods/database.ts): same validation, same request
|
|
3
|
+
bodies, same decoding of the records the server returns.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
import mimetypes
|
|
10
|
+
import os
|
|
11
|
+
import random
|
|
12
|
+
import re
|
|
13
|
+
import string
|
|
14
|
+
import time
|
|
15
|
+
import warnings
|
|
16
|
+
from typing import Any, Dict, List, Optional, Union
|
|
17
|
+
from urllib.parse import quote, urlsplit
|
|
18
|
+
|
|
19
|
+
from .errors import SkapiError
|
|
20
|
+
from ._encryption import parse_file_marker
|
|
21
|
+
|
|
22
|
+
BASE62 = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
|
|
23
|
+
NUMBER_SHIFT = 4503599627370496 # 2^52: number index values are stored shifted by this
|
|
24
|
+
CONDITIONS = ["gt", "gte", "lt", "lte", ">", ">=", "<", "<=", "=", "eq"]
|
|
25
|
+
CONDITIONS_NE = CONDITIONS + ["!=", "ne"]
|
|
26
|
+
SPECIAL_INDEX = ("$updated", "$uploaded", "$referenced_count", "$user_id")
|
|
27
|
+
ORDER_BY = ["average_number", "total_number", "number_count", "average_bool", "total_bool",
|
|
28
|
+
"bool_count", "string_count", "index_name", "number_of_records"]
|
|
29
|
+
_UUID = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.IGNORECASE)
|
|
30
|
+
_CONTROL = re.compile(r"[\u0000-\u001F\u007F]|\U0010FFFF")
|
|
31
|
+
_table_string_warned = False
|
|
32
|
+
# Seconds before a private access key expires at which it is no longer sent.
|
|
33
|
+
_PRIVATE_KEY_REFRESH_MARGIN = 60
|
|
34
|
+
_EXPIRING_KEY = re.compile(r"([0-9]+)\.")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _jd(v):
|
|
38
|
+
"""A value as it appears in an error message; never fails on odd types."""
|
|
39
|
+
return json.dumps(v, default=repr, ensure_ascii=False)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _err(message: str, code: str = "INVALID_PARAMETER") -> SkapiError:
|
|
43
|
+
return SkapiError(message, code)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def js_length(text: str) -> int:
|
|
47
|
+
"""String length as JavaScript counts it (UTF-16 code units: an emoji counts 2)."""
|
|
48
|
+
return len(text.encode("utf-16-le")) // 2
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def from_base62(text: str) -> int:
|
|
52
|
+
n = 0
|
|
53
|
+
for i, ch in enumerate(reversed(text)):
|
|
54
|
+
idx = BASE62.find(ch)
|
|
55
|
+
n += idx * (62 ** i)
|
|
56
|
+
return n
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def to_base62(n: int) -> str:
|
|
60
|
+
if n == 0:
|
|
61
|
+
return "0"
|
|
62
|
+
out = ""
|
|
63
|
+
while n > 0:
|
|
64
|
+
n, r = divmod(n, 62)
|
|
65
|
+
out = BASE62[r] + out
|
|
66
|
+
return out
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def encode_delimiters(value: str) -> str:
|
|
70
|
+
return value.replace("%", "%25").replace("/", "%2F").replace("!", "%21").replace("*", "%2A").replace("#", "%23")
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def decode_delimiters(value: Any) -> Any:
|
|
74
|
+
if not isinstance(value, str):
|
|
75
|
+
return value
|
|
76
|
+
value = re.sub(r"%(2F|21|2A|23)", lambda m: {"2F": "/", "21": "!", "2A": "*", "23": "#"}[m.group(1).upper()], value, flags=re.IGNORECASE)
|
|
77
|
+
return re.sub(r"%25", "%", value, flags=re.IGNORECASE)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def validate_string(value: Any, field: str, *, allow_empty: bool = False, max_length: int = 256,
|
|
81
|
+
escape: bool = False, no_dollar: bool = False, alphanumeric: bool = False) -> str:
|
|
82
|
+
if not isinstance(value, str):
|
|
83
|
+
raise _err(f'"{field}" should be type: <string>.')
|
|
84
|
+
if alphanumeric and re.search(r"[^a-zA-Z0-9]", value):
|
|
85
|
+
raise _err(f'"{field}" should only contain alphanumeric characters.')
|
|
86
|
+
if not allow_empty and len(value) == 0:
|
|
87
|
+
raise _err(f'"{field}" is required.')
|
|
88
|
+
if max_length and js_length(value) > max_length:
|
|
89
|
+
raise _err(f'"{field}" should be <= {max_length} characters.')
|
|
90
|
+
if _CONTROL.search(value):
|
|
91
|
+
raise _err(f'"{field}" cannot include control characters or unsupported sentinel characters.')
|
|
92
|
+
if escape:
|
|
93
|
+
value = encode_delimiters(value)
|
|
94
|
+
if max_length and js_length(value) > max_length:
|
|
95
|
+
raise _err(f'"{field}" should be <= {max_length} characters. "/", "!", "*", "#" and "%" count as 3 characters each.')
|
|
96
|
+
if no_dollar and value.startswith("$"):
|
|
97
|
+
raise _err(f'"{field}" cannot start with "$".')
|
|
98
|
+
return value
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def table_name(v, field="table.name", allow_empty=False):
|
|
102
|
+
return validate_string(v, field, allow_empty=allow_empty, escape=True)
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def tag_name(v, field="tag", allow_empty=False):
|
|
106
|
+
return validate_string(v, field, allow_empty=allow_empty, escape=True)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def index_name(v, field="index.name", allow_empty=False):
|
|
110
|
+
return validate_string(v, field, allow_empty=allow_empty, escape=True, no_dollar=True)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def user_id(v, field="User ID"):
|
|
114
|
+
if not v:
|
|
115
|
+
raise _err(f"{field} is empty.")
|
|
116
|
+
if not isinstance(v, str):
|
|
117
|
+
raise _err(f"{field} should be type: string.")
|
|
118
|
+
if not _UUID.match(v):
|
|
119
|
+
raise _err(f"{field} is invalid.")
|
|
120
|
+
return v
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def access_group_in(v, signed_in: bool):
|
|
124
|
+
if isinstance(v, bool):
|
|
125
|
+
raise _err('"table.access_group" should be type: <number | string>.')
|
|
126
|
+
if isinstance(v, (int, float)):
|
|
127
|
+
if v > 99 or v < 0:
|
|
128
|
+
raise _err('"table.access_group" value should be within a range of 0 ~ 99.')
|
|
129
|
+
result = v
|
|
130
|
+
elif isinstance(v, str):
|
|
131
|
+
mapping = {"*": "private", "private": "private", "public": 0, "authorized": 1, "admin": 99}
|
|
132
|
+
if v not in mapping:
|
|
133
|
+
raise _err('"table.access_group" is invalid.')
|
|
134
|
+
result = mapping[v]
|
|
135
|
+
else:
|
|
136
|
+
raise _err('"table.access_group" should be type: <number | string>.')
|
|
137
|
+
if not signed_in and result:
|
|
138
|
+
raise _err("Unsigned users have no access to records with access group.")
|
|
139
|
+
return result
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def access_group_out(v) -> Union[str, int]:
|
|
143
|
+
if v == "**":
|
|
144
|
+
return "private"
|
|
145
|
+
try:
|
|
146
|
+
n = int(str(v))
|
|
147
|
+
except ValueError:
|
|
148
|
+
return v
|
|
149
|
+
return {0: "public", 1: "authorized", 99: "admin"}.get(n, n)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def index_value(v, field="index.value"):
|
|
153
|
+
if isinstance(v, bool):
|
|
154
|
+
return v
|
|
155
|
+
if isinstance(v, (int, float)):
|
|
156
|
+
if abs(v) > NUMBER_SHIFT:
|
|
157
|
+
raise _err("Number value should be within range -4503599627370496 ~ +4503599627370496")
|
|
158
|
+
return v
|
|
159
|
+
if isinstance(v, str):
|
|
160
|
+
return validate_string(v, field, allow_empty=True)
|
|
161
|
+
raise _err(f'"{field}" should be type: <number | boolean | string>.')
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _typeof(v) -> str:
|
|
165
|
+
if isinstance(v, bool):
|
|
166
|
+
return "boolean"
|
|
167
|
+
if isinstance(v, (int, float)):
|
|
168
|
+
return "number"
|
|
169
|
+
if isinstance(v, str):
|
|
170
|
+
return "string"
|
|
171
|
+
return "object"
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _condition(v, allowed=CONDITIONS, field="condition"):
|
|
175
|
+
if v not in allowed:
|
|
176
|
+
raise _err(f'{_jd(v)} is invalid in "{field}". allowed types or values are: ' + ", ".join(_jd(c) for c in allowed) + ".")
|
|
177
|
+
return v
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _record_or_unique_id(query: dict, is_delete: bool) -> Optional[dict]:
|
|
181
|
+
"""ID mode: only service, owner and record_id (or else unique_id) survive."""
|
|
182
|
+
if is_delete:
|
|
183
|
+
for key in ("record_id", "unique_id"):
|
|
184
|
+
v = query.get(key)
|
|
185
|
+
if v is not None and (v == "" or v == []):
|
|
186
|
+
raise _err(f'"{key}" is empty.')
|
|
187
|
+
if not query.get("record_id") and not query.get("unique_id"):
|
|
188
|
+
return None
|
|
189
|
+
out = {}
|
|
190
|
+
for key in ("service", "owner"):
|
|
191
|
+
if query.get(key):
|
|
192
|
+
out[key] = query[key]
|
|
193
|
+
key = "record_id" if query.get("record_id") else "unique_id"
|
|
194
|
+
value = query[key]
|
|
195
|
+
|
|
196
|
+
def one(v, field):
|
|
197
|
+
if key == "record_id":
|
|
198
|
+
return validate_string(v, field, alphanumeric=True, max_length=0)
|
|
199
|
+
if not isinstance(v, str):
|
|
200
|
+
raise _err(f'"{field}" should be type: <string>.')
|
|
201
|
+
if not v:
|
|
202
|
+
raise _err(f'"{field}" is required.')
|
|
203
|
+
return v
|
|
204
|
+
|
|
205
|
+
if isinstance(value, list):
|
|
206
|
+
if not is_delete:
|
|
207
|
+
raise _err(f'"{key}" should be type: <string>.')
|
|
208
|
+
if not value:
|
|
209
|
+
raise _err(f'"{key}" should not be an empty list.')
|
|
210
|
+
if len(value) > 1000:
|
|
211
|
+
raise _err(f'"{key}" can hold at most 1000 ids.')
|
|
212
|
+
out[key] = [one(v, f"{key}[{i}]") for i, v in enumerate(value)]
|
|
213
|
+
else:
|
|
214
|
+
out[key] = one(value, key)
|
|
215
|
+
return out
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _typed_string(v: str) -> Any:
|
|
219
|
+
"""Index values as the server stores them ("!S%text", "!N%<number + 2^52>", "!B%1")."""
|
|
220
|
+
prefix, body = v[:3], v[3:]
|
|
221
|
+
if prefix == "!S%":
|
|
222
|
+
return body
|
|
223
|
+
if prefix == "!B%":
|
|
224
|
+
return body == "1"
|
|
225
|
+
if prefix in ("!L%", "!D%"):
|
|
226
|
+
try:
|
|
227
|
+
return json.loads(body)
|
|
228
|
+
except ValueError:
|
|
229
|
+
raise SkapiError("Value parse error.", "PARSE_ERROR") from None
|
|
230
|
+
if prefix == "!N%":
|
|
231
|
+
whole, _, frac = body.partition(".")
|
|
232
|
+
n = int(whole) - NUMBER_SHIFT
|
|
233
|
+
if not frac:
|
|
234
|
+
return n
|
|
235
|
+
if n >= 0 or not re.search(r"[1-9]", frac):
|
|
236
|
+
return float(f"{n}.{frac}")
|
|
237
|
+
digits = frac.rstrip("0")
|
|
238
|
+
complement = "".join(str(9 - int(d)) for d in digits[:-1]) + str(10 - int(digits[-1]))
|
|
239
|
+
return float(f"-{-n - 1}.{complement}")
|
|
240
|
+
return v
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def parse_stored_data(r: Any) -> Any:
|
|
244
|
+
"""The stored record payload ('!J%<json>' and the legacy forms) as the value that was posted."""
|
|
245
|
+
if isinstance(r, str):
|
|
246
|
+
if r.startswith("!J%"):
|
|
247
|
+
try:
|
|
248
|
+
return json.loads(r[3:])
|
|
249
|
+
except ValueError:
|
|
250
|
+
return r
|
|
251
|
+
if r == "!D%{}":
|
|
252
|
+
return {}
|
|
253
|
+
if r == "!L%[]":
|
|
254
|
+
return []
|
|
255
|
+
return r
|
|
256
|
+
if isinstance(r, dict) and list(r.keys()) == ["__json__"] and isinstance(r["__json__"], str):
|
|
257
|
+
try:
|
|
258
|
+
return json.loads(r["__json__"])
|
|
259
|
+
except ValueError:
|
|
260
|
+
return r
|
|
261
|
+
return r
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _is_offload_marker(r: Any) -> bool:
|
|
265
|
+
return (isinstance(r, dict) and list(r.keys()) == ["__data__"] and isinstance(r["__data__"], str)
|
|
266
|
+
and r["__data__"].endswith("/__data__/__json__.json"))
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _random_key(n: int = 6) -> str:
|
|
270
|
+
return "".join(random.choice(string.ascii_letters + string.digits) for _ in range(n))
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
# The ASCII bytes a URL path encoder percent-encodes (as the WHATWG URL parser skapi-js uses):
|
|
274
|
+
# C0 controls except tab/LF/CR, space " < > ` { } DEL, and ^. get_file itself only encodes the
|
|
275
|
+
# space, which is in this set, so urls from either SDK are undone the same way.
|
|
276
|
+
_URL_PATH_ENCODED_ASCII = frozenset([
|
|
277
|
+
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0B, 0x0C, 0x0E, 0x0F,
|
|
278
|
+
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F,
|
|
279
|
+
0x20, 0x22, 0x3C, 0x3E, 0x5E, 0x60, 0x7B, 0x7D, 0x7F])
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def undo_url_path_encoding(url: str) -> str:
|
|
283
|
+
"""
|
|
284
|
+
Undoes only what a url path encoder produces: uppercase %XX of the bytes above and of valid
|
|
285
|
+
UTF-8 sequences. Any other "%" is part of the file's own name ("50%25 off.pdf", "a%2Fb.pdf")
|
|
286
|
+
and is kept. Same as skapi-js database.ts undoUrlPathEncoding.
|
|
287
|
+
"""
|
|
288
|
+
def run(m):
|
|
289
|
+
tokens = re.findall(r"%[0-9A-F]{2}", m.group(0))
|
|
290
|
+
out, i = "", 0
|
|
291
|
+
while i < len(tokens):
|
|
292
|
+
b = int(tokens[i][1:], 16)
|
|
293
|
+
if b < 0x80:
|
|
294
|
+
out += chr(b) if b in _URL_PATH_ENCODED_ASCII else tokens[i]
|
|
295
|
+
i += 1
|
|
296
|
+
continue
|
|
297
|
+
n = 4 if b >= 0xF0 else 3 if b >= 0xE0 else 2 if b >= 0xC0 else 1
|
|
298
|
+
seq = tokens[i:i + n]
|
|
299
|
+
try:
|
|
300
|
+
if len(seq) < n or n == 1:
|
|
301
|
+
raise ValueError
|
|
302
|
+
out += bytes(int(t[1:], 16) for t in seq).decode("utf-8")
|
|
303
|
+
i += n
|
|
304
|
+
except ValueError:
|
|
305
|
+
out += tokens[i]
|
|
306
|
+
i += 1
|
|
307
|
+
return out
|
|
308
|
+
return re.sub(r"(?:%[0-9A-F]{2})+", run, url)
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
class DatabaseMixin:
|
|
312
|
+
"""Mixed into Skapi. Needs: _call(path, data, auth=, method=, fetch_options=, ...), _signed_in(),
|
|
313
|
+
_fresh_id_token(), user, service, owner, _ensure_connection(), _requester."""
|
|
314
|
+
|
|
315
|
+
# caches, created by Skapi.__init__
|
|
316
|
+
_private_access_key: Dict[str, str]
|
|
317
|
+
_my_unique_ids: Dict[str, Dict[str, str]]
|
|
318
|
+
|
|
319
|
+
def _scope(self, service=None, owner=None) -> str:
|
|
320
|
+
return f"{service or self.service}/{owner or self.owner}"
|
|
321
|
+
|
|
322
|
+
def _forget_unique_ids_of(self, scope: str, record_ids) -> None:
|
|
323
|
+
"""Drops every remembered unique_id that points at one of these records (renamed or deleted)."""
|
|
324
|
+
known = self._my_unique_ids.get(scope)
|
|
325
|
+
if known:
|
|
326
|
+
for uid in [u for u, rid in known.items() if rid in record_ids]:
|
|
327
|
+
del known[uid]
|
|
328
|
+
|
|
329
|
+
def _remember_unique_id(self, scope: str, record: dict) -> None:
|
|
330
|
+
"""After a write: the record's current unique_id, and none of its earlier ones."""
|
|
331
|
+
rid = record.get("record_id")
|
|
332
|
+
if not rid:
|
|
333
|
+
return
|
|
334
|
+
self._forget_unique_ids_of(scope, (rid,))
|
|
335
|
+
if record.get("unique_id"):
|
|
336
|
+
self._my_unique_ids.setdefault(scope, {})[record["unique_id"]] = rid
|
|
337
|
+
|
|
338
|
+
def _remember_handed_out_url(self, handed_out: str, requested: str) -> None:
|
|
339
|
+
"""
|
|
340
|
+
A record file url get_file gave out (as "endpoint" or "info") may come back to delete_files
|
|
341
|
+
or remove_bin. The server removes the stored file url by exact string, so the url given out
|
|
342
|
+
is mapped back to the exact url it was made from. Kept only when the two differ, or when the
|
|
343
|
+
url looks encoded ("a%20b.pdf" is its own stored form), so the map stays small.
|
|
344
|
+
"""
|
|
345
|
+
base = handed_out.split("?")[0]
|
|
346
|
+
raw = requested.split("?")[0]
|
|
347
|
+
if base.startswith("https://") and (base != raw or undo_url_path_encoding(base) != base):
|
|
348
|
+
self._handed_out_file_urls[base] = raw
|
|
349
|
+
|
|
350
|
+
def _stored_record_file_url(self, url: str) -> str:
|
|
351
|
+
"""
|
|
352
|
+
The form the server stores a record file url in. A non-public file's url from get_file
|
|
353
|
+
ends in "?t=<token>" and has its spaces percent-encoded; sent back like that it matched no
|
|
354
|
+
stored file, and the server answers success when nothing matched, so the file stayed.
|
|
355
|
+
1. A url get_file handed out on this client maps back to the exact url it was made from.
|
|
356
|
+
2. Otherwise a url with a query had its path encoded, and only that encoding is undone.
|
|
357
|
+
3. A url without a query is already the stored form and is sent as written.
|
|
358
|
+
"""
|
|
359
|
+
base = url.split("?")[0]
|
|
360
|
+
known = getattr(self, "_handed_out_file_urls", {}).get(base)
|
|
361
|
+
if known is not None:
|
|
362
|
+
return known
|
|
363
|
+
return undo_url_path_encoding(base) if "?" in url else base
|
|
364
|
+
|
|
365
|
+
def _is_admin(self) -> bool:
|
|
366
|
+
u = self.user
|
|
367
|
+
return bool(u and u.get("service") == self.service and u.get("owner") == "skapi")
|
|
368
|
+
|
|
369
|
+
def _require_login_gate(self, method: str):
|
|
370
|
+
conn = self._ensure_connection()
|
|
371
|
+
if not self._signed_in() and (conn or {}).get("opt", {}).get("require_login") is True:
|
|
372
|
+
raise SkapiError(f'This project requires users to sign in before reading its database, so "{method}" is not available to a signed-out visitor.', "REQUIRE_LOGIN")
|
|
373
|
+
|
|
374
|
+
# ---- query builder ----------------------------------------------------------------------
|
|
375
|
+
|
|
376
|
+
def _build_query(self, query: Optional[dict], is_delete: bool):
|
|
377
|
+
global _table_string_warned
|
|
378
|
+
query = dict(query or {})
|
|
379
|
+
scope = self._scope(query.get("service"), query.get("owner"))
|
|
380
|
+
is_reference_fetch: Any = None
|
|
381
|
+
signed_in = self._signed_in()
|
|
382
|
+
|
|
383
|
+
id_query = _record_or_unique_id(query, is_delete)
|
|
384
|
+
if id_query is not None:
|
|
385
|
+
query = id_query
|
|
386
|
+
single = query.get("record_id") or query.get("unique_id")
|
|
387
|
+
if isinstance(single, list):
|
|
388
|
+
if "unique_id" in query:
|
|
389
|
+
for uid in single:
|
|
390
|
+
self._my_unique_ids.get(scope, {}).pop(uid, None)
|
|
391
|
+
elif is_delete:
|
|
392
|
+
self._forget_unique_ids_of(scope, set(single))
|
|
393
|
+
return query, ""
|
|
394
|
+
is_reference_fetch = single
|
|
395
|
+
cached_key = self._cached_private_key(single)
|
|
396
|
+
if cached_key:
|
|
397
|
+
query["private_key"] = cached_key
|
|
398
|
+
cached = self._my_unique_ids.get(scope, {}).get(single)
|
|
399
|
+
if is_delete:
|
|
400
|
+
self._my_unique_ids.get(scope, {}).pop(single, None)
|
|
401
|
+
if "record_id" in query:
|
|
402
|
+
self._forget_unique_ids_of(scope, (single,))
|
|
403
|
+
elif cached:
|
|
404
|
+
query["record_id"] = cached
|
|
405
|
+
return query, is_reference_fetch
|
|
406
|
+
|
|
407
|
+
is_admin = self._is_admin()
|
|
408
|
+
ref_user_is_me = False
|
|
409
|
+
ref = query.get("reference")
|
|
410
|
+
if isinstance(ref, dict):
|
|
411
|
+
rid = ref.get("record_id") or ref.get("unique_id")
|
|
412
|
+
if rid:
|
|
413
|
+
is_reference_fetch = rid
|
|
414
|
+
cached_key = self._cached_private_key(rid)
|
|
415
|
+
if cached_key:
|
|
416
|
+
query["private_key"] = cached_key
|
|
417
|
+
query["reference"] = rid
|
|
418
|
+
elif ref.get("user_id"):
|
|
419
|
+
ref_user_is_me = bool(self.user) and ref["user_id"] == self.user.get("user_id")
|
|
420
|
+
query["reference"] = ref["user_id"]
|
|
421
|
+
|
|
422
|
+
table = query.get("table")
|
|
423
|
+
if isinstance(table, str):
|
|
424
|
+
if not _table_string_warned:
|
|
425
|
+
_table_string_warned = True
|
|
426
|
+
warnings.warn(f'skapi: table: "{table}" in get_records() and delete_records() is deprecated. Write the access group with the name: '
|
|
427
|
+
f'table: {{"name": "{table}", "access_group": "public"}}. See https://docs.skapi.com/deprecated/deprecated.html',
|
|
428
|
+
DeprecationWarning, stacklevel=3)
|
|
429
|
+
query["table"] = table = {"name": table, "access_group": 0}
|
|
430
|
+
if not is_admin and isinstance(table, dict) and table.get("access_group") is None:
|
|
431
|
+
raise _err('"table.access_group" is required: a table is always { name, access_group }. '
|
|
432
|
+
'To read what references a record in every table, send "reference" without "table".')
|
|
433
|
+
index = query.get("index")
|
|
434
|
+
if isinstance(index, dict) and "range" in index and "condition" in index:
|
|
435
|
+
index = dict(index)
|
|
436
|
+
del index["range"]
|
|
437
|
+
query["index"] = index
|
|
438
|
+
|
|
439
|
+
reads_by_reference = not is_delete and signed_in and bool(is_reference_fetch or (isinstance(query.get("reference"), str) and query.get("reference")))
|
|
440
|
+
required_table = not (ref_user_is_me or is_admin or reads_by_reference)
|
|
441
|
+
|
|
442
|
+
out: Dict[str, Any] = {}
|
|
443
|
+
if "table" in query:
|
|
444
|
+
t = query["table"]
|
|
445
|
+
if not isinstance(t, dict):
|
|
446
|
+
raise _err(f'{_jd(t)} is invalid in "table". allowed type or value is: Type<object>.')
|
|
447
|
+
nt = dict(t)
|
|
448
|
+
if "name" in t:
|
|
449
|
+
nt["name"] = table_name(t["name"])
|
|
450
|
+
if "access_group" in t:
|
|
451
|
+
nt["access_group"] = access_group_in(t["access_group"], signed_in)
|
|
452
|
+
if "subscription" in t:
|
|
453
|
+
nt["subscription"] = user_id(t["subscription"], 'User ID in "subscription"')
|
|
454
|
+
out["table"] = nt
|
|
455
|
+
elif required_table:
|
|
456
|
+
raise _err('Key "table" is required.')
|
|
457
|
+
if "reference" in query:
|
|
458
|
+
if not isinstance(query["reference"], str):
|
|
459
|
+
raise _err(f'{_jd(query["reference"])} is invalid in "reference". allowed type or value is: Type<string>.')
|
|
460
|
+
out["reference"] = query["reference"]
|
|
461
|
+
if "index" in query:
|
|
462
|
+
i = query["index"]
|
|
463
|
+
if not isinstance(i, dict):
|
|
464
|
+
raise _err(f'{_jd(i)} is invalid in "index". allowed type or value is: Type<object>.')
|
|
465
|
+
ni = dict(i)
|
|
466
|
+
raw_name = i.get("name")
|
|
467
|
+
if "name" in i:
|
|
468
|
+
ni["name"] = raw_name if raw_name in SPECIAL_INDEX else index_name(raw_name)
|
|
469
|
+
if "value" in i:
|
|
470
|
+
v = i["value"]
|
|
471
|
+
if raw_name in ("$updated", "$uploaded", "$referenced_count"):
|
|
472
|
+
if isinstance(v, bool) or not isinstance(v, (int, float)):
|
|
473
|
+
raise _err('"index.value" should be type: number.')
|
|
474
|
+
elif raw_name == "$user_id":
|
|
475
|
+
v = user_id(v)
|
|
476
|
+
elif v == "":
|
|
477
|
+
v = ""
|
|
478
|
+
elif isinstance(v, str) and isinstance(raw_name, str) and raw_name.endswith("."):
|
|
479
|
+
v = validate_string(v, "index.value", allow_empty=True, escape=True)
|
|
480
|
+
else:
|
|
481
|
+
v = index_value(v)
|
|
482
|
+
ni["value"] = v
|
|
483
|
+
if "condition" in i:
|
|
484
|
+
ni["condition"] = _condition(i["condition"], CONDITIONS, "condition")
|
|
485
|
+
if "range" in i:
|
|
486
|
+
if "value" not in i:
|
|
487
|
+
raise _err('"index.value" is required.')
|
|
488
|
+
if raw_name == "$record_id":
|
|
489
|
+
raise _err('Cannot do "index.range" on $record_id')
|
|
490
|
+
r = i["range"]
|
|
491
|
+
if _typeof(i["value"]) != _typeof(r):
|
|
492
|
+
raise _err('"index.range" type should match the type of "index.value".')
|
|
493
|
+
if isinstance(r, str):
|
|
494
|
+
r = validate_string(r, "index.range", allow_empty=True, escape=isinstance(raw_name, str) and raw_name.endswith("."))
|
|
495
|
+
ni["range"] = r
|
|
496
|
+
out["index"] = ni
|
|
497
|
+
if "tag" in query and query["tag"] is not None:
|
|
498
|
+
if not isinstance(query["tag"], str):
|
|
499
|
+
raise _err('"tag" should be type: string.')
|
|
500
|
+
out["tag"] = tag_name(query["tag"])
|
|
501
|
+
for key in ("private_key", "service", "owner"):
|
|
502
|
+
if key in query:
|
|
503
|
+
if not isinstance(query[key], str):
|
|
504
|
+
raise _err(f'{_jd(query[key])} is invalid in "{key}". allowed type or value is: Type<string>.')
|
|
505
|
+
out[key] = query[key]
|
|
506
|
+
return out, is_reference_fetch
|
|
507
|
+
|
|
508
|
+
# ---- reads ------------------------------------------------------------------------------
|
|
509
|
+
|
|
510
|
+
def get_records(self, query: Optional[dict] = None, fetch_options: Optional[dict] = None) -> dict:
|
|
511
|
+
"""Records by id, unique id, table, index, tag or reference. See the docs for the query keys."""
|
|
512
|
+
self._require_login_gate("get_records")
|
|
513
|
+
q, ref_id = self._build_query(query, False)
|
|
514
|
+
signed_in = self._signed_in()
|
|
515
|
+
result = self._call("get-records", q, auth=signed_in, method="POST" if signed_in else "GET", fetch_options=fetch_options)
|
|
516
|
+
if ref_id and isinstance(result, dict) and isinstance(result.get("reference_private_key"), str):
|
|
517
|
+
self._private_access_key[ref_id] = result["reference_private_key"]
|
|
518
|
+
if isinstance(result, dict) and isinstance(result.get("list"), list):
|
|
519
|
+
result["list"] = [self._normalize_record(r) for r in result["list"]]
|
|
520
|
+
return result
|
|
521
|
+
|
|
522
|
+
def delete_records(self, query: dict, fetch_options: Optional[dict] = None):
|
|
523
|
+
"""Deletes your records (or any, as an admin) by id, unique id, or query."""
|
|
524
|
+
self._ensure_connection()
|
|
525
|
+
q, ref_id = self._build_query(query, True)
|
|
526
|
+
result = self._call("del-records", q, auth=True, fetch_options=fetch_options)
|
|
527
|
+
if ref_id and isinstance(result, dict) and isinstance(result.get("reference_private_key"), str):
|
|
528
|
+
self._private_access_key[ref_id] = result["reference_private_key"]
|
|
529
|
+
if isinstance(result, dict) and isinstance(result.get("list"), list):
|
|
530
|
+
out = []
|
|
531
|
+
for r in result["list"]:
|
|
532
|
+
try:
|
|
533
|
+
out.append(self._normalize_record(r, skip_data_fetch=True, skip_bin_resolve=True) if isinstance(r, dict) else r)
|
|
534
|
+
except Exception:
|
|
535
|
+
out.append(r)
|
|
536
|
+
result["list"] = out
|
|
537
|
+
# A delete by query names no ids up front: the unique_ids remembered for the records it
|
|
538
|
+
# deleted are dropped here, from the records it returns (as skapi-js deleteRecords).
|
|
539
|
+
deleted = {r.get("record_id") or r.get("rec") for r in out if isinstance(r, dict)}
|
|
540
|
+
deleted = {rid for rid in deleted if isinstance(rid, str) and rid}
|
|
541
|
+
if deleted:
|
|
542
|
+
self._forget_unique_ids_of(self._scope((query or {}).get("service"), (query or {}).get("owner")), deleted)
|
|
543
|
+
if isinstance(result, dict) and result.get("message"):
|
|
544
|
+
return result["message"]
|
|
545
|
+
return result
|
|
546
|
+
|
|
547
|
+
def get_tables(self, query: Optional[dict] = None, fetch_options: Optional[dict] = None) -> dict:
|
|
548
|
+
query = query or {}
|
|
549
|
+
p: Dict[str, Any] = {}
|
|
550
|
+
if "table" in query:
|
|
551
|
+
p["table"] = table_name(query["table"], "table", allow_empty=True)
|
|
552
|
+
if "condition" in query:
|
|
553
|
+
p["condition"] = _condition(query["condition"], CONDITIONS_NE)
|
|
554
|
+
p.update({k: query[k] for k in ("service", "owner") if k in query})
|
|
555
|
+
res = self._call("get-table", p, auth=self._signed_in(), fetch_options=fetch_options)
|
|
556
|
+
names = {"pv": "private", "00": "public", "01": "authorized", "99": "admin"}
|
|
557
|
+
for item in res.get("list", []) if isinstance(res, dict) else []:
|
|
558
|
+
for k in list(item.keys()):
|
|
559
|
+
if k.startswith("grp_"):
|
|
560
|
+
x = k[4:]
|
|
561
|
+
item[f"number_of_records_in_access_group_{names.get(x, x)}"] = item.pop(k)
|
|
562
|
+
for old, new in (("cnt_rec", "number_of_records"), ("tbl", "table"), ("srvc", "service")):
|
|
563
|
+
if old in item:
|
|
564
|
+
item[new] = item.pop(old)
|
|
565
|
+
if "table" in item:
|
|
566
|
+
item["table"] = decode_delimiters(item["table"])
|
|
567
|
+
return res
|
|
568
|
+
|
|
569
|
+
def get_tags(self, query: Optional[dict] = None, fetch_options: Optional[dict] = None) -> dict:
|
|
570
|
+
query = query or {}
|
|
571
|
+
p: Dict[str, Any] = {}
|
|
572
|
+
if "table" in query:
|
|
573
|
+
p["table"] = table_name(query["table"], "table", allow_empty=True)
|
|
574
|
+
if "tag" in query:
|
|
575
|
+
p["tag"] = tag_name(query["tag"], "tag", allow_empty=True)
|
|
576
|
+
if "condition" in query:
|
|
577
|
+
p["condition"] = _condition(query["condition"], CONDITIONS_NE)
|
|
578
|
+
p.update({k: query[k] for k in ("service", "owner") if k in query})
|
|
579
|
+
res = self._call("get-tag", p, auth=self._signed_in(), fetch_options=fetch_options)
|
|
580
|
+
if isinstance(res, dict) and isinstance(res.get("list"), list):
|
|
581
|
+
out = []
|
|
582
|
+
for item in res["list"]:
|
|
583
|
+
parts = item.get("tag", "").split("/")
|
|
584
|
+
out.append({"table": decode_delimiters(parts[1] if len(parts) > 1 else ""), "tag": decode_delimiters(parts[0]),
|
|
585
|
+
"number_of_records": item.get("cnt_rec")})
|
|
586
|
+
res["list"] = out
|
|
587
|
+
return res
|
|
588
|
+
|
|
589
|
+
def get_indexes(self, query: dict, fetch_options: Optional[dict] = None) -> dict:
|
|
590
|
+
if not (query or {}).get("table"):
|
|
591
|
+
raise _err('"table" is required.')
|
|
592
|
+
p: Dict[str, Any] = {"table": table_name(query["table"], "table", allow_empty=True)}
|
|
593
|
+
if "index" in query:
|
|
594
|
+
p["index"] = index_name(query["index"], "index.name")
|
|
595
|
+
if "order" in query:
|
|
596
|
+
o = query["order"]
|
|
597
|
+
if not isinstance(o, dict):
|
|
598
|
+
raise _err(f'{_jd(o)} is invalid in "order". allowed type or value is: Type<object>.')
|
|
599
|
+
no = dict(o)
|
|
600
|
+
if "by" in o:
|
|
601
|
+
no["by"] = _condition(o["by"], ORDER_BY, "by")
|
|
602
|
+
if "value" in o:
|
|
603
|
+
v = o["value"]
|
|
604
|
+
if o.get("by") == "index_name" and isinstance(v, str):
|
|
605
|
+
v = validate_string(v, "order.value", allow_empty=True, escape=True)
|
|
606
|
+
elif not isinstance(v, (str, int, float, bool)):
|
|
607
|
+
raise _err('"order.value" should be type: <string | number | boolean>.')
|
|
608
|
+
no["value"] = v
|
|
609
|
+
if "condition" in o:
|
|
610
|
+
no["condition"] = _condition(o["condition"], CONDITIONS_NE)
|
|
611
|
+
if "by" not in no:
|
|
612
|
+
raise _err('"order.by" is required.')
|
|
613
|
+
if "condition" in no and "value" not in no:
|
|
614
|
+
raise _err('"value" is required for "condition".')
|
|
615
|
+
if "index" in p and not p["index"].endswith("."):
|
|
616
|
+
raise _err('"index" should be a parent index name of the compound index when using "order.by"')
|
|
617
|
+
p["order"] = no
|
|
618
|
+
p.update({k: query[k] for k in ("service", "owner") if k in query})
|
|
619
|
+
res = self._call("get-index", p, auth=self._signed_in(), fetch_options=fetch_options)
|
|
620
|
+
if isinstance(res, dict) and isinstance(res.get("list"), list):
|
|
621
|
+
out = []
|
|
622
|
+
rename = {"cnt_bool": "boolean_count", "cnt_numb": "number_count", "totl_numb": "total_number", "totl_bool": "total_bool",
|
|
623
|
+
"avrg_numb": "average_number", "avrg_bool": "average_bool", "cnt_str": "string_count"}
|
|
624
|
+
for i in res["list"]:
|
|
625
|
+
parts = i.get("idx", "").split("/")
|
|
626
|
+
item = {"table": decode_delimiters(parts[1] if len(parts) > 1 else ""), "index": decode_delimiters(parts[2] if len(parts) > 2 else ""),
|
|
627
|
+
"number_of_records": i.get("cnt_rec")}
|
|
628
|
+
for k, v in rename.items():
|
|
629
|
+
if k in i:
|
|
630
|
+
item[v] = i[k]
|
|
631
|
+
out.append(item)
|
|
632
|
+
res["list"] = out
|
|
633
|
+
return res
|
|
634
|
+
|
|
635
|
+
def get_unique_id(self, query: Optional[dict] = None, fetch_options: Optional[dict] = None) -> dict:
|
|
636
|
+
query = query or {}
|
|
637
|
+
p: Dict[str, Any] = {}
|
|
638
|
+
if "unique_id" in query:
|
|
639
|
+
if not isinstance(query["unique_id"], str):
|
|
640
|
+
raise _err(f'{_jd(query["unique_id"])} is invalid in "unique_id". allowed type or value is: Type<string>.')
|
|
641
|
+
p["unique_id"] = query["unique_id"]
|
|
642
|
+
if "condition" in query:
|
|
643
|
+
p["condition"] = _condition(query["condition"], CONDITIONS_NE)
|
|
644
|
+
p.update({k: query[k] for k in ("service", "owner") if k in query})
|
|
645
|
+
res = self._call("get-uniqueid", p, auth=self._signed_in(), fetch_options=fetch_options)
|
|
646
|
+
if isinstance(res, dict) and isinstance(res.get("list"), list):
|
|
647
|
+
res["list"] = [{"unique_id": i.get("unq"), "record_id": i.get("rec")} for i in res["list"]]
|
|
648
|
+
return res
|
|
649
|
+
|
|
650
|
+
# ---- writes -----------------------------------------------------------------------------
|
|
651
|
+
|
|
652
|
+
def _setup_post_config(self, config: dict) -> tuple:
|
|
653
|
+
"""Validated config (what goes on the wire) and the reference id to remember a private key under."""
|
|
654
|
+
if not isinstance(config, dict):
|
|
655
|
+
raise _err('"config" argument is required.')
|
|
656
|
+
signed_in = self._signed_in()
|
|
657
|
+
c = dict(config)
|
|
658
|
+
if not c.get("record_id") and not c.get("table"):
|
|
659
|
+
raise _err('"table.name" is required.')
|
|
660
|
+
if isinstance(c.get("table"), str):
|
|
661
|
+
c["table"] = {"name": c["table"]} if c.get("record_id") else {"name": c["table"], "access_group": 0}
|
|
662
|
+
scope = self._scope(c.get("service"), c.get("owner"))
|
|
663
|
+
out: Dict[str, Any] = {}
|
|
664
|
+
is_reference_post = None
|
|
665
|
+
|
|
666
|
+
if "record_id" in c and c["record_id"] is not None:
|
|
667
|
+
rid = c["record_id"]
|
|
668
|
+
cached = self._my_unique_ids.get(scope, {}).get(rid) if isinstance(rid, str) else None
|
|
669
|
+
out["record_id"] = cached or validate_string(rid, "record_id", max_length=0)
|
|
670
|
+
if "unique_id" in c:
|
|
671
|
+
if c["unique_id"] is not None and not isinstance(c["unique_id"], str):
|
|
672
|
+
raise _err(f'{_jd(c["unique_id"])} is invalid in "unique_id". allowed types or values are: Type<string>, null.')
|
|
673
|
+
out["unique_id"] = c["unique_id"]
|
|
674
|
+
if "readonly" in c:
|
|
675
|
+
if not isinstance(c["readonly"], bool):
|
|
676
|
+
raise _err(f'{_jd(c["readonly"])} is invalid in "readonly". allowed type or value is: Type<boolean>.')
|
|
677
|
+
out["readonly"] = c["readonly"]
|
|
678
|
+
if "table" in c and c["table"] is not None:
|
|
679
|
+
t = c["table"]
|
|
680
|
+
if not isinstance(t, dict):
|
|
681
|
+
raise _err(f'{_jd(t)} is invalid in "table". allowed types or values are: Type<string>, Type<object>.')
|
|
682
|
+
nt = dict(t)
|
|
683
|
+
if "name" in t:
|
|
684
|
+
nt["name"] = table_name(t["name"])
|
|
685
|
+
if "access_group" in t:
|
|
686
|
+
nt["access_group"] = access_group_in(t["access_group"], signed_in)
|
|
687
|
+
if "subscription" in t and t["subscription"] is not None:
|
|
688
|
+
if not isinstance(t["subscription"], dict):
|
|
689
|
+
raise _err('"table.subscription" should be type: <object | null>.')
|
|
690
|
+
for k, v in t["subscription"].items():
|
|
691
|
+
if k in ("is_subscription_record", "upload_to_feed", "notify_subscribers", "feed_referencing_records", "notify_referencing_records") and not isinstance(v, bool):
|
|
692
|
+
raise _err(f'"table.subscription.{k}" should be type: <boolean>.')
|
|
693
|
+
out["table"] = nt
|
|
694
|
+
if "notification" in c and c["notification"] is not None:
|
|
695
|
+
n = c["notification"]
|
|
696
|
+
if not isinstance(n, dict):
|
|
697
|
+
raise _err('"notification" should be an object with "title" and "body".')
|
|
698
|
+
for k in ("title", "body"):
|
|
699
|
+
if not isinstance(n.get(k), str) or not n[k].strip():
|
|
700
|
+
raise _err(f'"notification.{k}" is required.')
|
|
701
|
+
out["notification"] = {"title": n["title"], "body": n["body"]}
|
|
702
|
+
if "source" in c and c["source"] is not None:
|
|
703
|
+
s = dict(c["source"])
|
|
704
|
+
if "referencing_limit" in s and s["referencing_limit"] is not None:
|
|
705
|
+
v = s["referencing_limit"]
|
|
706
|
+
if isinstance(v, bool) or not isinstance(v, (int, float)):
|
|
707
|
+
raise _err('"reference_limit" should be type: <number | null>')
|
|
708
|
+
if v < 0:
|
|
709
|
+
raise _err('"reference_limit" should be >= 0')
|
|
710
|
+
if v > 4503599627370546:
|
|
711
|
+
raise _err('"reference_limit" should be <= 4503599627370546')
|
|
712
|
+
for k in ("prevent_multiple_referencing", "can_remove_referencing_records", "only_granted_can_reference", "allow_granted_to_grant_others"):
|
|
713
|
+
if k in s and not isinstance(s[k], bool):
|
|
714
|
+
raise _err(f'"source.{k}" should be type: <boolean>.')
|
|
715
|
+
if "referencing_index_restrictions" in s:
|
|
716
|
+
r = s["referencing_index_restrictions"]
|
|
717
|
+
if not r:
|
|
718
|
+
s["referencing_index_restrictions"] = None
|
|
719
|
+
else:
|
|
720
|
+
items = r if isinstance(r, list) else [r]
|
|
721
|
+
fixed = []
|
|
722
|
+
for item in items:
|
|
723
|
+
if "name" not in item:
|
|
724
|
+
raise _err('Key "name" is required.')
|
|
725
|
+
if "range" in item:
|
|
726
|
+
raise _err('Index restriction "range" type should match the type of "value".')
|
|
727
|
+
ni = dict(item)
|
|
728
|
+
ni["name"] = index_name(item["name"], '"name" in "index_restrictions"')
|
|
729
|
+
if "value" in item:
|
|
730
|
+
ni["value"] = index_value(item["value"])
|
|
731
|
+
ni["condition"] = item.get("condition") if item.get("condition") in CONDITIONS_NE else None
|
|
732
|
+
fixed.append(ni)
|
|
733
|
+
s["referencing_index_restrictions"] = fixed
|
|
734
|
+
out["source"] = s
|
|
735
|
+
if "reference" in c:
|
|
736
|
+
ref = c["reference"]
|
|
737
|
+
if ref is None:
|
|
738
|
+
out["reference"] = {"record_id": None}
|
|
739
|
+
elif not ref:
|
|
740
|
+
pass
|
|
741
|
+
elif isinstance(ref, str):
|
|
742
|
+
out["reference"] = self._my_unique_ids.get(scope, {}).get(ref) or ref
|
|
743
|
+
is_reference_post = out["reference"]
|
|
744
|
+
elif isinstance(ref, dict):
|
|
745
|
+
nr = dict(ref)
|
|
746
|
+
if ref.get("record_id") is not None:
|
|
747
|
+
nr["record_id"] = validate_string(ref["record_id"], "reference.record_id", alphanumeric=True, max_length=0)
|
|
748
|
+
if "unique_id" in ref and not isinstance(ref["unique_id"], str):
|
|
749
|
+
raise _err('"reference.unique_id" should be type: <string>.')
|
|
750
|
+
out["reference"] = nr
|
|
751
|
+
else:
|
|
752
|
+
raise _err('"reference" should be type: <string | object>.')
|
|
753
|
+
if "index" in c:
|
|
754
|
+
i = c["index"]
|
|
755
|
+
if i is None:
|
|
756
|
+
out["index"] = None
|
|
757
|
+
else:
|
|
758
|
+
if not isinstance(i, dict):
|
|
759
|
+
raise _err('"index" should be type: <object | null>.')
|
|
760
|
+
ni = dict(i)
|
|
761
|
+
if "name" in i:
|
|
762
|
+
ni["name"] = index_name(i["name"])
|
|
763
|
+
if "value" in i:
|
|
764
|
+
ni["value"] = index_value(i["value"])
|
|
765
|
+
out["index"] = ni
|
|
766
|
+
if "tags" in c and c["tags"] is None:
|
|
767
|
+
out["tags"] = None # removes every tag, as in skapi-js
|
|
768
|
+
elif "tags" in c:
|
|
769
|
+
tags = c["tags"]
|
|
770
|
+
if isinstance(tags, str):
|
|
771
|
+
tags = [t.strip() for t in tags.split(",")]
|
|
772
|
+
if not isinstance(tags, list):
|
|
773
|
+
raise _err('"tag" should be type: <string | string[]>.')
|
|
774
|
+
out["tags"] = [tag_name(t) for t in tags]
|
|
775
|
+
if "remove_bin" in c:
|
|
776
|
+
rb = c["remove_bin"]
|
|
777
|
+
# JavaScript falsiness: None, False, 0 and "" mean null; an empty list removes nothing
|
|
778
|
+
if rb is None or rb is False or rb == 0 or rb == "":
|
|
779
|
+
out["remove_bin"] = None
|
|
780
|
+
else:
|
|
781
|
+
if not isinstance(rb, list):
|
|
782
|
+
raise _err('"remove_bin" should be type: <string[] | BinaryFile[] | null>')
|
|
783
|
+
fixed = []
|
|
784
|
+
for item in rb:
|
|
785
|
+
if isinstance(item, str):
|
|
786
|
+
# the stored file url, as delete_files sends it: no "?t=<token>", and
|
|
787
|
+
# get_file's encoding undone without touching a "%" of the name itself
|
|
788
|
+
fixed.append(self._stored_record_file_url(item))
|
|
789
|
+
elif isinstance(item, dict) and item.get("url") and item.get("size") and item.get("filename"):
|
|
790
|
+
u = urlsplit(item["url"])
|
|
791
|
+
fixed.append(f"{u.scheme}://{u.netloc}/{item['path']}")
|
|
792
|
+
else:
|
|
793
|
+
raise _err('"remove_bin" should be type: <string[] | BinaryFile[] | null>')
|
|
794
|
+
out["remove_bin"] = fixed
|
|
795
|
+
for key in ("service", "owner"):
|
|
796
|
+
if key in c:
|
|
797
|
+
out[key] = c[key]
|
|
798
|
+
|
|
799
|
+
if not signed_in:
|
|
800
|
+
t = out.get("table") or {}
|
|
801
|
+
if out.get("record_id"):
|
|
802
|
+
raise _err("Public users cannot update existing records.", "INVALID_REQUEST")
|
|
803
|
+
if t.get("access_group") not in ("public", 0):
|
|
804
|
+
raise _err("Public users can only post records to public tables.", "INVALID_REQUEST")
|
|
805
|
+
if any((t.get("subscription") or {}).values()):
|
|
806
|
+
raise _err("Public users cannot post subscription records.", "INVALID_REQUEST")
|
|
807
|
+
if out.get("remove_bin"):
|
|
808
|
+
raise _err("Public users cannot remove files from records.", "INVALID_REQUEST")
|
|
809
|
+
if out.get("unique_id"):
|
|
810
|
+
raise _err("Public users cannot set unique_id for records.", "INVALID_REQUEST")
|
|
811
|
+
return out, is_reference_post
|
|
812
|
+
|
|
813
|
+
_NO_DATA = object()
|
|
814
|
+
|
|
815
|
+
def post_record(self, data: Any = _NO_DATA, config: Optional[dict] = None, files: Optional[list] = None) -> dict:
|
|
816
|
+
"""
|
|
817
|
+
Creates a record, or updates one when config has record_id.
|
|
818
|
+
data: any JSON value. Leave it out on an update to keep the stored data; None stores null.
|
|
819
|
+
files: [{"name": "<field>", "file": <path, bytes or file object>, "filename"?: str, "content_type"?: str}]
|
|
820
|
+
"""
|
|
821
|
+
self._ensure_connection()
|
|
822
|
+
if config is None:
|
|
823
|
+
raise _err('"config" argument is required.')
|
|
824
|
+
conf, is_reference_post = self._setup_post_config(config)
|
|
825
|
+
upload_files = _prepare_files(files) # read and checked before the record is created
|
|
826
|
+
if "data" in config:
|
|
827
|
+
data = config["data"]
|
|
828
|
+
sent_data = data is not self._NO_DATA
|
|
829
|
+
view = dict(config)
|
|
830
|
+
if "table" in conf:
|
|
831
|
+
view["table"] = conf["table"]
|
|
832
|
+
enc = self._maybe_encrypt(data if sent_data else None, view, sent_data)
|
|
833
|
+
conf.pop("data", None)
|
|
834
|
+
if not enc.get("omit"):
|
|
835
|
+
if not enc.get("encrypted") and len(_json_bytes(enc["send"])) > 2 * 1024 * 1024:
|
|
836
|
+
raise _err("Data should not exceed 2MB", "INVALID_REQUEST")
|
|
837
|
+
conf["data"] = enc["send"]
|
|
838
|
+
rid_in = conf.get("record_id")
|
|
839
|
+
if enc.get("files") == "decrypt" and rid_in:
|
|
840
|
+
# private -> not private: attachments back to plaintext while the key is still here
|
|
841
|
+
conf["remove_bin"] = (conf.get("remove_bin") or []) + self._convert_record_files(rid_in, "decrypt")
|
|
842
|
+
if enc.get("spill") is not None and rid_in:
|
|
843
|
+
# ciphertext over 256 KB travels as a file, uploaded before the record points at it
|
|
844
|
+
up = self._upload_prepared([{"name": "__skenc_ct__", "filename": "payload.bin", "content": enc["spill"],
|
|
845
|
+
"content_type": "application/octet-stream"}], rid_in)
|
|
846
|
+
if up["failed"]:
|
|
847
|
+
raise up["errors"][0]["error"]
|
|
848
|
+
if rid_in in self._enc.deks and up["bin_endpoints"][0]:
|
|
849
|
+
self._enc.deks[rid_in]["ct_url"] = up["bin_endpoints"][0]
|
|
850
|
+
self._enc.deks[rid_in]["ct_stale"] = []
|
|
851
|
+
if enc.get("retire") and rid_in:
|
|
852
|
+
# old payload files: retired whether this write spilled again or now fits inline
|
|
853
|
+
conf["remove_bin"] = (conf.get("remove_bin") or []) + list(enc["retire"])
|
|
854
|
+
rec = self._call("post-record", conf, auth=self._signed_in(), method="POST")
|
|
855
|
+
if enc.get("encrypted") and rec.get("rec"):
|
|
856
|
+
self._bind_record_dek(rec["rec"], enc.get("dek"))
|
|
857
|
+
if enc.get("files") == "encrypt" and rec.get("rec"):
|
|
858
|
+
# now private with its key cached: seal the existing attachments, then retire the plain ones
|
|
859
|
+
try:
|
|
860
|
+
retire = self._convert_record_files(rec["rec"], "encrypt")
|
|
861
|
+
if retire:
|
|
862
|
+
self.post_record(config={"record_id": rec["rec"], "remove_bin": retire})
|
|
863
|
+
except Exception as err:
|
|
864
|
+
warnings.warn(f"Failed to encrypt existing attachments after making the record private: {err}", RuntimeWarning, stacklevel=2)
|
|
865
|
+
|
|
866
|
+
if upload_files:
|
|
867
|
+
if not enc.get("encrypted"):
|
|
868
|
+
# this write did not seal the record, so its cached key was not confirmed by it
|
|
869
|
+
self._confirm_file_dek(rec.get("rec"))
|
|
870
|
+
result = self._upload_prepared(upload_files, rec.get("rec"), service=conf.get("service"))
|
|
871
|
+
rec["bin"] = (rec.get("bin") or []) + result["bin_endpoints"]
|
|
872
|
+
if is_reference_post and isinstance(rec.get("reference_private_key"), str):
|
|
873
|
+
self._private_access_key[is_reference_post] = rec["reference_private_key"]
|
|
874
|
+
record = self._normalize_record(rec, skip_data_fetch=sent_data, skip_decrypt=bool(enc.get("encrypted")))
|
|
875
|
+
if sent_data and _is_offload_marker(rec.get("data")):
|
|
876
|
+
record["data"] = data
|
|
877
|
+
if enc.get("encrypted") and "plain" in enc:
|
|
878
|
+
record["data"] = enc["plain"]
|
|
879
|
+
record["encrypted"] = {"status": "encrypted", "recipients": enc["recipients"]}
|
|
880
|
+
self._remember_unique_id(self._scope(conf.get("service"), conf.get("owner")), record)
|
|
881
|
+
return record
|
|
882
|
+
|
|
883
|
+
def post_records(self, items: List[dict]) -> dict:
|
|
884
|
+
"""Many records in few requests: items = [{"data": ..., "config": {...}}]. Returns {list, failed}."""
|
|
885
|
+
self._ensure_connection()
|
|
886
|
+
if not isinstance(items, list) or not items:
|
|
887
|
+
raise _err('"items" should be a non-empty array.')
|
|
888
|
+
queue, failed, out = [], [], []
|
|
889
|
+
for i, item in enumerate(items):
|
|
890
|
+
try:
|
|
891
|
+
if not isinstance(item, dict):
|
|
892
|
+
raise _err(f'"items[{i}]" should be an object with "config".')
|
|
893
|
+
if not isinstance(item.get("config"), dict):
|
|
894
|
+
raise _err(f'"items[{i}].config" is required.')
|
|
895
|
+
conf, _ = self._setup_post_config(item["config"])
|
|
896
|
+
payload = item["config"]["data"] if "data" in item["config"] else item.get("data", self._NO_DATA)
|
|
897
|
+
has = payload is not self._NO_DATA
|
|
898
|
+
view = dict(item["config"])
|
|
899
|
+
if "table" in conf:
|
|
900
|
+
view["table"] = conf["table"]
|
|
901
|
+
enc = self._maybe_encrypt(payload if has else None, view, has)
|
|
902
|
+
conf.pop("data", None)
|
|
903
|
+
if enc.get("encrypted"):
|
|
904
|
+
if enc.get("spill") is not None:
|
|
905
|
+
# skapi-js stores an empty ciphertext here (no payload file is uploaded in bulk); refuse instead
|
|
906
|
+
raise _err("This encrypted update is over 256 KB once sealed; save it with post_record().", "INVALID_REQUEST")
|
|
907
|
+
conf["data"] = enc["send"]
|
|
908
|
+
conf["_skapi_plain"] = enc
|
|
909
|
+
elif has:
|
|
910
|
+
if len(_json_bytes(payload)) > 2 * 1024 * 1024:
|
|
911
|
+
raise _err("Data should not exceed 2MB", "INVALID_REQUEST")
|
|
912
|
+
conf["data"] = payload
|
|
913
|
+
queue.append((i, conf))
|
|
914
|
+
except SkapiError as err:
|
|
915
|
+
failed.append({"index": i, "code": str(err.code or "INVALID_PARAMETER"), "message": err.message})
|
|
916
|
+
position = 0
|
|
917
|
+
while position < len(queue):
|
|
918
|
+
chunk = queue[position:position + 100]
|
|
919
|
+
answered = self._bulk_post([c for _, c in chunk])
|
|
920
|
+
if not answered:
|
|
921
|
+
for idx, _ in chunk:
|
|
922
|
+
failed.append({"index": idx, "code": "ERROR", "message": "The server answered nothing for this record."})
|
|
923
|
+
position += len(chunk)
|
|
924
|
+
continue
|
|
925
|
+
taken = min(len(answered), len(chunk))
|
|
926
|
+
for k in range(taken):
|
|
927
|
+
rec = answered[k]
|
|
928
|
+
if rec.get("record_id"):
|
|
929
|
+
out.append(rec)
|
|
930
|
+
else:
|
|
931
|
+
e = rec.get("error") or {}
|
|
932
|
+
failed.append({"index": chunk[k][0], "code": e.get("code") or "ERROR", "message": e.get("message") or "The record was not saved."})
|
|
933
|
+
position += taken
|
|
934
|
+
failed.sort(key=lambda f: f["index"])
|
|
935
|
+
return {"list": out, "failed": failed}
|
|
936
|
+
|
|
937
|
+
def _bulk_post(self, configs: List[dict]) -> List[dict]:
|
|
938
|
+
services = {c["service"] for c in configs if "service" in c}
|
|
939
|
+
owners = {c["owner"] for c in configs if "owner" in c}
|
|
940
|
+
if len(services) > 1:
|
|
941
|
+
raise _err('All bulk params should share the same "service" value.')
|
|
942
|
+
if len(owners) > 1:
|
|
943
|
+
raise _err('All bulk params should share the same "owner" value.')
|
|
944
|
+
common_service = services.pop() if services else ""
|
|
945
|
+
common_owner = owners.pop() if owners else ""
|
|
946
|
+
sent = [{k: v for k, v in c.items() if k not in ("service", "owner", "_skapi_plain")} for c in configs]
|
|
947
|
+
res = self._call("post-record", {"_is_bulk_": sent, "service": common_service, "owner": common_owner}, auth=self._signed_in(), method="POST")
|
|
948
|
+
if not isinstance(res, list):
|
|
949
|
+
return []
|
|
950
|
+
records = []
|
|
951
|
+
for conf, raw in zip(configs, res):
|
|
952
|
+
sealed = conf.get("_skapi_plain")
|
|
953
|
+
rec = self._normalize_record(raw, skip_data_fetch="data" in conf, skip_decrypt=bool(sealed)) if isinstance(raw, dict) else {}
|
|
954
|
+
if "data" in conf and isinstance(raw, dict) and _is_offload_marker(raw.get("data")) and not raw.get("error"):
|
|
955
|
+
rec["data"] = conf["data"]
|
|
956
|
+
if sealed and rec.get("record_id"):
|
|
957
|
+
self._bind_record_dek(rec["record_id"], sealed.get("dek"))
|
|
958
|
+
rec["data"] = sealed["plain"]
|
|
959
|
+
rec["encrypted"] = {"status": "encrypted"}
|
|
960
|
+
if isinstance(raw, dict) and raw.get("error") and not rec.get("record_id"):
|
|
961
|
+
rec["error"] = raw["error"]
|
|
962
|
+
self._remember_unique_id(self._scope(common_service or None, common_owner or None), rec)
|
|
963
|
+
records.append(rec)
|
|
964
|
+
return records
|
|
965
|
+
|
|
966
|
+
# ---- files ------------------------------------------------------------------------------
|
|
967
|
+
|
|
968
|
+
def upload_files(self, files: list, record_id: str, service: Optional[str] = None) -> dict:
|
|
969
|
+
"""Attaches files to a record you can write: [{"name", "file", "filename"?, "content_type"?}]."""
|
|
970
|
+
if not record_id:
|
|
971
|
+
raise _err('"record_id" is required.')
|
|
972
|
+
entries = _prepare_files(files)
|
|
973
|
+
self._confirm_file_dek(record_id) # an encrypted record's files are sealed only with its current key
|
|
974
|
+
return self._upload_prepared(entries, record_id, service)
|
|
975
|
+
|
|
976
|
+
def _upload_prepared(self, entries: List[dict], record_id: str, service: Optional[str] = None) -> dict:
|
|
977
|
+
entries = [self._maybe_encrypt_file(record_id, e) for e in entries] # sealed when the record's key is cached
|
|
978
|
+
reserved_key = _random_key(6)
|
|
979
|
+
base = {"reserved_key": reserved_key, "service": service or self.service, "request": "post", "id": record_id}
|
|
980
|
+
params = [{"key": f'{e["name"]}/{e["filename"]}', "sizeKey": to_base62(len(e["content"])), "contentType": e["content_type"]} for e in entries]
|
|
981
|
+
permits: List[Any] = [None] * len(entries)
|
|
982
|
+
if len(entries) >= 5 and not getattr(self, "_bulk_permits_unavailable", False):
|
|
983
|
+
try:
|
|
984
|
+
for start in range(0, len(entries), 100):
|
|
985
|
+
chunk = params[start:start + 100]
|
|
986
|
+
res = self._call("get-signed-urls", {"files": chunk, "reserved_key": reserved_key, "service": base["service"], "request": "post", "id": record_id},
|
|
987
|
+
auth=self._signed_in())
|
|
988
|
+
answered = res.get("list") or []
|
|
989
|
+
errors = {f.get("index"): f for f in res.get("failed") or []}
|
|
990
|
+
for k in range(len(chunk)):
|
|
991
|
+
if k in errors:
|
|
992
|
+
permits[start + k] = SkapiError(errors[k].get("message") or "Upload permit refused.", errors[k].get("code") or "ERROR")
|
|
993
|
+
elif k < len(answered) and answered[k]:
|
|
994
|
+
permits[start + k] = answered[k]
|
|
995
|
+
else:
|
|
996
|
+
permits[start + k] = SkapiError("No upload permit was answered for this file.", "ERROR")
|
|
997
|
+
except SkapiError as err:
|
|
998
|
+
if str(err.code) in ("404", "501") or re.search(r"missing authentication token|not implemented", err.message, re.I) \
|
|
999
|
+
or (str(err.code) == "403" and re.search(r"not found", err.message, re.I)):
|
|
1000
|
+
self._bulk_permits_unavailable = True
|
|
1001
|
+
permits = [None] * len(entries)
|
|
1002
|
+
else:
|
|
1003
|
+
raise
|
|
1004
|
+
completed, failed, errors, endpoints, landed = [], [], [], [], []
|
|
1005
|
+
for i, e in enumerate(entries):
|
|
1006
|
+
permit = permits[i]
|
|
1007
|
+
if permit is None:
|
|
1008
|
+
try:
|
|
1009
|
+
permit = self._call("get-signed-url", dict(params[i], **base), auth=self._signed_in())
|
|
1010
|
+
except SkapiError as err:
|
|
1011
|
+
permit = err
|
|
1012
|
+
if isinstance(permit, SkapiError):
|
|
1013
|
+
endpoints.append("")
|
|
1014
|
+
failed.append(e["filename"])
|
|
1015
|
+
errors.append({"file": e["filename"], "name": e["name"], "error": permit})
|
|
1016
|
+
continue
|
|
1017
|
+
endpoints.append(permit.get("cdn"))
|
|
1018
|
+
try:
|
|
1019
|
+
self._requester.upload(permit["url"], permit.get("fields") or {}, e["filename"], e["content"], e["content_type"])
|
|
1020
|
+
completed.append(e["filename"])
|
|
1021
|
+
landed.append(permit.get("cdn"))
|
|
1022
|
+
except SkapiError as err:
|
|
1023
|
+
failed.append(e["filename"])
|
|
1024
|
+
errors.append({"file": e["filename"], "name": e["name"], "error": err})
|
|
1025
|
+
return {"completed": completed, "failed": failed, "errors": errors, "bin_endpoints": endpoints, "landed": landed}
|
|
1026
|
+
|
|
1027
|
+
def delete_files(self, endpoints: Union[str, List[str]]) -> list:
|
|
1028
|
+
if isinstance(endpoints, str):
|
|
1029
|
+
endpoints = [endpoints]
|
|
1030
|
+
if not isinstance(endpoints, list):
|
|
1031
|
+
raise _err('"endpoints" should be type: array | string.')
|
|
1032
|
+
# A non-public file's url ends in "?t=<token>" and is percent-encoded (get_file "endpoint").
|
|
1033
|
+
# The server removes the stored, decoded url by exact string and answers success when
|
|
1034
|
+
# nothing matched, so the stored form is sent.
|
|
1035
|
+
endpoints = [self._stored_record_file_url(e) if isinstance(e, str) else e for e in endpoints]
|
|
1036
|
+
res = self._call("del-files", {"endpoints": endpoints, "storage": "records"}, auth=True, method="POST")
|
|
1037
|
+
return [self._normalize_record(r) for r in res] if isinstance(res, list) else res
|
|
1038
|
+
|
|
1039
|
+
def get_file(self, url: str, config: Optional[dict] = None):
|
|
1040
|
+
"""
|
|
1041
|
+
A file of a record. config["dataType"]: "bytes" (default), "text", "endpoint" (a URL you can
|
|
1042
|
+
open), "info" (what the URL says, no download). config["expires"]: seconds, for a signed URL
|
|
1043
|
+
that works without signing in. config["save_to"]: a path to write the file to (returns the path).
|
|
1044
|
+
"""
|
|
1045
|
+
config = dict(config or {})
|
|
1046
|
+
if not isinstance(url, str):
|
|
1047
|
+
raise _err('"url" should be type: string.')
|
|
1048
|
+
requested_url = url
|
|
1049
|
+
base, sep, query = url.partition("?")
|
|
1050
|
+
try:
|
|
1051
|
+
parts = urlsplit(base)
|
|
1052
|
+
if parts.scheme not in ("http", "https") or not parts.netloc:
|
|
1053
|
+
raise ValueError
|
|
1054
|
+
base = parts.geturl().replace(" ", "%20")
|
|
1055
|
+
except ValueError:
|
|
1056
|
+
raise _err(f'"{url}" is an invalid url.') from None
|
|
1057
|
+
url = base + (sep + query if sep else "")
|
|
1058
|
+
data_type = config.get("dataType", config.get("data_type", "bytes"))
|
|
1059
|
+
if data_type == "blob":
|
|
1060
|
+
data_type = "bytes"
|
|
1061
|
+
if data_type not in ("bytes", "text", "endpoint", "info"):
|
|
1062
|
+
data_type = "bytes"
|
|
1063
|
+
expires = config.get("expires") or 0
|
|
1064
|
+
if isinstance(expires, bool) or not isinstance(expires, (int, float)):
|
|
1065
|
+
expires = 0
|
|
1066
|
+
|
|
1067
|
+
host = urlsplit(base).netloc.split(".")
|
|
1068
|
+
subdomain = host[0] if len(host) == 3 and host[1] == "skapi" else None
|
|
1069
|
+
target_key = base.split("/")[3:]
|
|
1070
|
+
need_auth = False
|
|
1071
|
+
is_skapi_file = bool(subdomain)
|
|
1072
|
+
presigned = bool(re.search(r"(^|&)(Signature|X-Amz-Signature)=", query))
|
|
1073
|
+
if presigned:
|
|
1074
|
+
# a link from get_file(..., {"expires": ...}): already signed, opened as it is
|
|
1075
|
+
if data_type == "endpoint":
|
|
1076
|
+
return url
|
|
1077
|
+
target_key = []
|
|
1078
|
+
if not subdomain and target_key and target_key[0] in ("auth", "publ"):
|
|
1079
|
+
if len(target_key) < 4 or not _UUID.match(target_key[2]) or not _UUID.match(target_key[3]):
|
|
1080
|
+
raise _err("Invalid file url.")
|
|
1081
|
+
need_auth = target_key[0] == "auth"
|
|
1082
|
+
is_skapi_file = True
|
|
1083
|
+
service = None if subdomain else (target_key[1] if len(target_key) > 1 else None)
|
|
1084
|
+
|
|
1085
|
+
if data_type == "info":
|
|
1086
|
+
self._remember_handed_out_url(url, requested_url)
|
|
1087
|
+
g = target_key[6] if len(target_key) > 6 else None
|
|
1088
|
+
return {
|
|
1089
|
+
"url": url, "filename": target_key[-1] if target_key else "", "fileKey": target_key[-2] if len(target_key) > 1 else "",
|
|
1090
|
+
"access_group": "private" if g == "**" else "authorized" if g == "01" else "public" if g == "00" else (int(g) if g and g.isdigit() else g),
|
|
1091
|
+
"uploader": target_key[3] if len(target_key) > 3 else None,
|
|
1092
|
+
"record_id": target_key[5] if len(target_key) > 5 and target_key[4] == "records" else "N/A",
|
|
1093
|
+
"filesize": from_base62(target_key[9]) if len(target_key) > 9 else None,
|
|
1094
|
+
"uploaded": from_base62(target_key[8]) if len(target_key) > 8 else None,
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
if expires:
|
|
1098
|
+
if not is_skapi_file:
|
|
1099
|
+
raise _err("Expires option can only be used on skapi cdn endpoints.")
|
|
1100
|
+
if expires < 0:
|
|
1101
|
+
raise _err('"config.expires" should be > 0. (seconds)')
|
|
1102
|
+
params = {"request": "get-host" if subdomain else "get", "id": subdomain or target_key[5], "key": requested_url.split("?")[0], "expires": expires}
|
|
1103
|
+
if service:
|
|
1104
|
+
params["service"] = service
|
|
1105
|
+
url = self._call("get-signed-url", params, auth=True)["url"]
|
|
1106
|
+
elif need_auth:
|
|
1107
|
+
token = self._fresh_id_token()
|
|
1108
|
+
url += ("&" if "?" in url else "?") + "t=" + quote(token, safe="-_.!~*'()")
|
|
1109
|
+
group = target_key[6]
|
|
1110
|
+
u = self.user or {}
|
|
1111
|
+
if u.get("user_id") != target_key[3] and (group == "**" or (group.isdigit() and int(u.get("access_group") or 0) < int(group))):
|
|
1112
|
+
rid = target_key[5]
|
|
1113
|
+
key = self._cached_private_key(rid)
|
|
1114
|
+
if not key and self.owner != "skapi":
|
|
1115
|
+
try:
|
|
1116
|
+
key = self.request_private_record_access_key({"record_id": rid, "reference_id": config.get("_ref")})
|
|
1117
|
+
except SkapiError:
|
|
1118
|
+
key = None
|
|
1119
|
+
if key:
|
|
1120
|
+
url += "&p=" + quote(key, safe="-_.!~*'()")
|
|
1121
|
+
|
|
1122
|
+
if data_type == "endpoint":
|
|
1123
|
+
# a url handed out here may be fed back to delete_files or remove_bin
|
|
1124
|
+
self._remember_handed_out_url(url, requested_url)
|
|
1125
|
+
return url
|
|
1126
|
+
if presigned or expires:
|
|
1127
|
+
content = self._requester.fetch_plain(url)
|
|
1128
|
+
else:
|
|
1129
|
+
content = self._requester.request(url, None, method="GET", content_type="application/x-www-form-urlencoded", ignore_service=True, raw=True)
|
|
1130
|
+
if not config.get("_raw"):
|
|
1131
|
+
content = self._maybe_decrypt_file(requested_url, content)
|
|
1132
|
+
if config.get("save_to"):
|
|
1133
|
+
path = os.path.abspath(os.path.expanduser(config["save_to"]))
|
|
1134
|
+
with open(path, "wb") as f:
|
|
1135
|
+
f.write(content)
|
|
1136
|
+
return path
|
|
1137
|
+
if data_type == "text":
|
|
1138
|
+
return content.decode("utf-8", errors="replace")
|
|
1139
|
+
return content
|
|
1140
|
+
|
|
1141
|
+
# ---- private record access --------------------------------------------------------------
|
|
1142
|
+
|
|
1143
|
+
def _record_access(self, record_id, uid, execute):
|
|
1144
|
+
ids = self._record_access_ids(record_id, uid)
|
|
1145
|
+
return self._call("grant-private-access", {"record_id": record_id, "user_id": ids, "execute": execute}, auth=True, ignore_service=True)
|
|
1146
|
+
|
|
1147
|
+
@staticmethod
|
|
1148
|
+
def _record_access_ids(record_id, uid) -> List[str]:
|
|
1149
|
+
if not isinstance(record_id, str):
|
|
1150
|
+
raise _err(f'{_jd(record_id)} is invalid in "record_id". allowed type or value is: Type<string>.')
|
|
1151
|
+
if not uid:
|
|
1152
|
+
raise _err("User ID is required.")
|
|
1153
|
+
ids = [uid] if isinstance(uid, str) else uid
|
|
1154
|
+
if len(ids) > 100:
|
|
1155
|
+
raise _err("Cannot process more than 100 users at once.")
|
|
1156
|
+
return [user_id(i, 'User ID in "user_id"') for i in ids]
|
|
1157
|
+
|
|
1158
|
+
def grant_private_record_access(self, params: dict):
|
|
1159
|
+
uid = params.get("user_id")
|
|
1160
|
+
if self._enc is not None and uid and isinstance(params.get("record_id"), str):
|
|
1161
|
+
# an encrypted record gets a key wrap for each new user first
|
|
1162
|
+
self._enc_add_recipients(params["record_id"], [uid] if isinstance(uid, str) else list(uid))
|
|
1163
|
+
return self._record_access(params.get("record_id"), uid, "add")
|
|
1164
|
+
|
|
1165
|
+
def remove_private_record_access(self, params: dict):
|
|
1166
|
+
uid = params.get("user_id") or None
|
|
1167
|
+
if self._enc is not None:
|
|
1168
|
+
# The data key is rolled BEFORE the access change: a fresh key, wrapped only for the users
|
|
1169
|
+
# who keep access. The roll is the step that can fail (attachments are sealed again and
|
|
1170
|
+
# uploaded), and failing first leaves access exactly as it was. As skapi-js.
|
|
1171
|
+
ids = self._record_access_ids(params.get("record_id"), uid)
|
|
1172
|
+
self._enc_drop_and_roll(params["record_id"], ids)
|
|
1173
|
+
return self._record_access(params.get("record_id"), uid, "remove")
|
|
1174
|
+
|
|
1175
|
+
def list_private_record_access(self, params: dict, fetch_options: Optional[dict] = None) -> dict:
|
|
1176
|
+
rid, uid = params.get("record_id") or None, params.get("user_id") or None
|
|
1177
|
+
if not rid and not uid:
|
|
1178
|
+
raise _err("Either record_id or user_id must be provided.")
|
|
1179
|
+
p: Dict[str, Any] = {}
|
|
1180
|
+
if rid:
|
|
1181
|
+
p["record_id"] = rid
|
|
1182
|
+
if uid:
|
|
1183
|
+
if isinstance(uid, str):
|
|
1184
|
+
p["user_id"] = [user_id(uid)]
|
|
1185
|
+
elif isinstance(uid, list):
|
|
1186
|
+
p["user_id"] = [user_id(u) for u in uid]
|
|
1187
|
+
else:
|
|
1188
|
+
raise _err("user_id should be type: <string | string[]>")
|
|
1189
|
+
p["execute"] = "list"
|
|
1190
|
+
res = self._call("grant-private-access", p, auth=True, fetch_options=fetch_options, ignore_service=True)
|
|
1191
|
+
for item in res.get("list", []) if isinstance(res, dict) else []:
|
|
1192
|
+
if item.get("rec_usr"):
|
|
1193
|
+
item["record_id"], item["user_id"] = item["rec_usr"].split("/")[0], item["rec_usr"].split("/")[1]
|
|
1194
|
+
elif item.get("usr_rec"):
|
|
1195
|
+
item["user_id"], item["record_id"] = item["usr_rec"].split("/")[0], item["usr_rec"].split("/")[1]
|
|
1196
|
+
return res
|
|
1197
|
+
|
|
1198
|
+
def _cached_private_key(self, record_id) -> Optional[str]:
|
|
1199
|
+
"""The cached private access key for `record_id`, or None when there is none worth sending.
|
|
1200
|
+
|
|
1201
|
+
The server mints "<exp>.<hash>", exp the unix second the key stops working (an hour after
|
|
1202
|
+
it was minted). A key that has expired or expires within _PRIVATE_KEY_REFRESH_MARGIN
|
|
1203
|
+
seconds is dropped, so the caller fetches a fresh one the usual way. A legacy key (64 hex
|
|
1204
|
+
characters, no expiry) is dropped too: the server accepts it only until a follow-up
|
|
1205
|
+
deploy turns that off, and a fresh key costs one request."""
|
|
1206
|
+
key = self._private_access_key.get(record_id) if isinstance(record_id, str) else None
|
|
1207
|
+
if not isinstance(key, str) or not key:
|
|
1208
|
+
return None
|
|
1209
|
+
m = _EXPIRING_KEY.match(key)
|
|
1210
|
+
if m and int(m.group(1)) - _PRIVATE_KEY_REFRESH_MARGIN > time.time():
|
|
1211
|
+
return key
|
|
1212
|
+
self._private_access_key.pop(record_id, None)
|
|
1213
|
+
return None
|
|
1214
|
+
|
|
1215
|
+
def request_private_record_access_key(self, params: dict) -> str:
|
|
1216
|
+
rid, ref = params.get("record_id"), params.get("reference_id")
|
|
1217
|
+
if not rid:
|
|
1218
|
+
raise _err("Record ID is required.")
|
|
1219
|
+
if not isinstance(rid, str):
|
|
1220
|
+
raise _err("Record ID should be type: <string | string[]>")
|
|
1221
|
+
if ref and not isinstance(ref, str):
|
|
1222
|
+
raise _err("Reference ID should be type: <string>")
|
|
1223
|
+
cached_key = self._cached_private_key(rid)
|
|
1224
|
+
if cached_key:
|
|
1225
|
+
return cached_key
|
|
1226
|
+
body = {"record_id": rid}
|
|
1227
|
+
if ref:
|
|
1228
|
+
body["reference_id"] = ref
|
|
1229
|
+
res = self._call("request-private-access-key", body, auth=True)
|
|
1230
|
+
key = res if isinstance(res, str) else (res.get("private_key") if isinstance(res, dict) else None)
|
|
1231
|
+
if not isinstance(key, str):
|
|
1232
|
+
raise SkapiError("Invalid private access key response.", "ERROR")
|
|
1233
|
+
self._private_access_key[rid] = key
|
|
1234
|
+
return key
|
|
1235
|
+
|
|
1236
|
+
# ---- decoding -----------------------------------------------------------------------------
|
|
1237
|
+
|
|
1238
|
+
def _normalize_record(self, record: Any, skip_data_fetch: bool = False, skip_bin_resolve: bool = False, skip_decrypt: bool = False) -> dict:
|
|
1239
|
+
if not isinstance(record, dict):
|
|
1240
|
+
return record
|
|
1241
|
+
if record.get("record_id"):
|
|
1242
|
+
return record
|
|
1243
|
+
out: Dict[str, Any] = {
|
|
1244
|
+
"user_id": "", "record_id": "", "updated": 0, "uploaded": 0, "readonly": False,
|
|
1245
|
+
"table": {"name": "", "access_group": 0, "subscription": {"is_subscription_record": False, "upload_to_feed": False,
|
|
1246
|
+
"notify_subscribers": False, "feed_referencing_records": False, "notify_referencing_records": False}},
|
|
1247
|
+
"referenced_count": 0,
|
|
1248
|
+
"source": {"referencing_limit": None, "prevent_multiple_referencing": False, "can_remove_referencing_records": False,
|
|
1249
|
+
"only_granted_can_reference": False},
|
|
1250
|
+
"ip": "", "bin": {},
|
|
1251
|
+
}
|
|
1252
|
+
anonymous = False
|
|
1253
|
+
bins = record.get("bin") if isinstance(record.get("bin"), list) else []
|
|
1254
|
+
data_url = None
|
|
1255
|
+
ct_urls = []
|
|
1256
|
+
for u in bins:
|
|
1257
|
+
if isinstance(u, str):
|
|
1258
|
+
sp = u.split("/")[3:]
|
|
1259
|
+
if len(sp) >= 12 and sp[10] == "__data__" and sp[-1] == "__json__.json":
|
|
1260
|
+
data_url = u
|
|
1261
|
+
if len(sp) >= 12 and sp[10] == "__skenc_ct__" and sp[-1] == "payload.bin":
|
|
1262
|
+
ct_urls.append(u) # an encrypted record's large ciphertext
|
|
1263
|
+
|
|
1264
|
+
# Newest upload first: every record file url carries its upload time right after "bin"
|
|
1265
|
+
# (".../bin/<base62 ms>/<size>/<field>/<name>"), and the record's file list is a set with
|
|
1266
|
+
# no order. Older SDKs could leave several payload files; the newest is the current one.
|
|
1267
|
+
def _uploaded(u):
|
|
1268
|
+
sp = u.split("/")[3:]
|
|
1269
|
+
try:
|
|
1270
|
+
return from_base62(sp[8]) if sp[7] == "bin" else 0
|
|
1271
|
+
except Exception:
|
|
1272
|
+
return 0
|
|
1273
|
+
ct_urls.sort(key=_uploaded, reverse=True)
|
|
1274
|
+
ct_url = ct_urls[0] if ct_urls else None
|
|
1275
|
+
|
|
1276
|
+
if "ip" in record:
|
|
1277
|
+
parts = str(record["ip"]).split("#")
|
|
1278
|
+
ip = parts[0]
|
|
1279
|
+
if len(parts) > 1:
|
|
1280
|
+
out["unique_id"] = "#".join(parts[1:])
|
|
1281
|
+
if ip.endswith("R"):
|
|
1282
|
+
out["readonly"] = True
|
|
1283
|
+
ip = ip[:-1]
|
|
1284
|
+
if re.match(r"^\d{1,3}-\d{1,3}-\d{1,3}-\d{1,3}$", ip):
|
|
1285
|
+
ip = ip.replace("-", ".")
|
|
1286
|
+
anonymous = True
|
|
1287
|
+
out["ip"] = ip
|
|
1288
|
+
if record.get("rec"):
|
|
1289
|
+
out["record_id"] = record["rec"]
|
|
1290
|
+
out["uploaded"] = from_base62(record["rec"][:-9])
|
|
1291
|
+
if "usr" in record:
|
|
1292
|
+
out["user_id"] = record["usr"]
|
|
1293
|
+
if "tbl" in record and not out["table"]["name"]:
|
|
1294
|
+
p = str(record["tbl"]).split("/")
|
|
1295
|
+
out["table"]["name"] = decode_delimiters(p[0])
|
|
1296
|
+
out["table"]["access_group"] = access_group_out(p[2]) if len(p) > 2 else 0
|
|
1297
|
+
out["table"]["subscription"]["is_subscription_record"] = len(p) > 3 and bool(p[3])
|
|
1298
|
+
if "usr_tbl" in record:
|
|
1299
|
+
p = str(record["usr_tbl"]).split("/")
|
|
1300
|
+
if not out["user_id"]:
|
|
1301
|
+
out["user_id"] = p[0]
|
|
1302
|
+
if not out["table"]["name"]:
|
|
1303
|
+
out["table"]["name"] = decode_delimiters(p[1] if len(p) > 1 else "")
|
|
1304
|
+
out["table"]["access_group"] = access_group_out(p[3]) if len(p) > 3 else 0
|
|
1305
|
+
out["table"]["subscription"]["is_subscription_record"] = len(p) > 4 and bool(p[4])
|
|
1306
|
+
if "idx" in record and isinstance(record["idx"], str):
|
|
1307
|
+
parts = record["idx"].split("!")
|
|
1308
|
+
out["index"] = {"name": decode_delimiters(parts[0]), "value": _typed_string("!" + "!".join(parts[1:]))}
|
|
1309
|
+
if record.get("ref"):
|
|
1310
|
+
out["reference"] = str(record["ref"]).split("/")[0]
|
|
1311
|
+
if "tags" in record:
|
|
1312
|
+
out["tags"] = [decode_delimiters(t) for t in record["tags"]] if isinstance(record["tags"], list) else record["tags"]
|
|
1313
|
+
if "upd" in record:
|
|
1314
|
+
out["updated"] = record["upd"]
|
|
1315
|
+
if "acpt_mrf" in record:
|
|
1316
|
+
out["source"]["prevent_multiple_referencing"] = not record["acpt_mrf"]
|
|
1317
|
+
if "ref_limt" in record:
|
|
1318
|
+
out["source"]["referencing_limit"] = record["ref_limt"]
|
|
1319
|
+
if "rfd" in record:
|
|
1320
|
+
out["referenced_count"] = record["rfd"]
|
|
1321
|
+
if bins:
|
|
1322
|
+
for u in bins:
|
|
1323
|
+
if not isinstance(u, str) or not u:
|
|
1324
|
+
continue
|
|
1325
|
+
path = "/".join(u.split("/")[3:])
|
|
1326
|
+
sp = path.split("/")
|
|
1327
|
+
if len(sp) < 12 or sp[10] in ("__data__", "__skenc_ct__"):
|
|
1328
|
+
continue
|
|
1329
|
+
path_key = sp[10]
|
|
1330
|
+
group = access_group_out(sp[6])
|
|
1331
|
+
endpoint = u
|
|
1332
|
+
if group != "public" and not skip_bin_resolve:
|
|
1333
|
+
try:
|
|
1334
|
+
endpoint = self.get_file(u, {"dataType": "endpoint", "_ref": out.get("reference")})
|
|
1335
|
+
except SkapiError:
|
|
1336
|
+
endpoint = u
|
|
1337
|
+
entry = {"access_group": group, "filename": sp[-1], "url": endpoint, "path": path,
|
|
1338
|
+
"size": from_base62(sp[9]), "uploaded": from_base62(sp[8])}
|
|
1339
|
+
sealed = parse_file_marker(path_key)
|
|
1340
|
+
if sealed:
|
|
1341
|
+
# an encrypted attachment: listed under its own field name, with the plaintext size
|
|
1342
|
+
path_key, entry["size"] = sealed
|
|
1343
|
+
entry["encrypted"], entry["stored_size"] = True, from_base62(sp[9])
|
|
1344
|
+
out["bin"].setdefault(path_key, []).append(entry)
|
|
1345
|
+
if isinstance(record.get("prv_acs"), dict):
|
|
1346
|
+
for k, v in record["prv_acs"].items():
|
|
1347
|
+
if k in ("notify_subscribers", "upload_to_feed", "feed_referencing_records", "notify_referencing_records"):
|
|
1348
|
+
out["table"]["subscription"][k] = v
|
|
1349
|
+
elif k == "referencing_index_restrictions" and isinstance(v, list):
|
|
1350
|
+
out["source"][k] = [dict(i, name=decode_delimiters(i["name"])) if isinstance(i, dict) and isinstance(i.get("name"), str) else i for i in v]
|
|
1351
|
+
else:
|
|
1352
|
+
out["source"][k] = v
|
|
1353
|
+
if "data" in record:
|
|
1354
|
+
r = record["data"]
|
|
1355
|
+
decoded = parse_stored_data(r)
|
|
1356
|
+
if decoded is not r or isinstance(r, str):
|
|
1357
|
+
out["data"] = decoded
|
|
1358
|
+
elif _is_offload_marker(r):
|
|
1359
|
+
if data_url and skip_data_fetch:
|
|
1360
|
+
out["data"] = r
|
|
1361
|
+
elif data_url or not skip_data_fetch:
|
|
1362
|
+
url = data_url or self._derived_data_url(record, r["__data__"], bins)
|
|
1363
|
+
out["data"] = r
|
|
1364
|
+
if url:
|
|
1365
|
+
try:
|
|
1366
|
+
out["data"] = json.loads(self.get_file(url, {"dataType": "text", "_ref": out.get("reference")}))
|
|
1367
|
+
except Exception:
|
|
1368
|
+
out["data"] = None if data_url else r
|
|
1369
|
+
if self._enc is not None:
|
|
1370
|
+
out["encrypted"] = {"status": "failed", "reason": "DATA_UNAVAILABLE"}
|
|
1371
|
+
else:
|
|
1372
|
+
out["data"] = r
|
|
1373
|
+
else:
|
|
1374
|
+
out["data"] = r
|
|
1375
|
+
if not skip_decrypt and "data" in out:
|
|
1376
|
+
value, flag = self._maybe_decrypt(out["data"], {
|
|
1377
|
+
"access_group": out["table"]["access_group"], "table_name": out["table"]["name"], "user_id": out["user_id"],
|
|
1378
|
+
"record_id": out["record_id"], "unique_id": out.get("unique_id"), "ct_url": ct_url, "ct_urls": ct_urls})
|
|
1379
|
+
if flag:
|
|
1380
|
+
out["data"], out["encrypted"] = value, flag
|
|
1381
|
+
if anonymous:
|
|
1382
|
+
out["user_id"] = "anonymous:" + out["user_id"]
|
|
1383
|
+
return out
|
|
1384
|
+
|
|
1385
|
+
def _derived_data_url(self, record: dict, marker_path: str, bins: list) -> Optional[str]:
|
|
1386
|
+
"""The offloaded data file's URL when the record's file list does not carry it."""
|
|
1387
|
+
if not re.match(r"^[0-9A-Za-z]+/[0-9A-Za-z]+/__data__/__json__\.json$", marker_path):
|
|
1388
|
+
return None
|
|
1389
|
+
service = owner = uploader = group = None
|
|
1390
|
+
if isinstance(record.get("srvc"), str):
|
|
1391
|
+
service, _, owner = record["srvc"].split("!")[0].partition("/")
|
|
1392
|
+
if isinstance(record.get("usr_tbl"), str):
|
|
1393
|
+
p = record["usr_tbl"].split("/")
|
|
1394
|
+
uploader = p[0]
|
|
1395
|
+
service = service or (p[2] if len(p) > 2 else None)
|
|
1396
|
+
group = p[3] if len(p) > 3 else None
|
|
1397
|
+
elif isinstance(record.get("tbl"), str):
|
|
1398
|
+
p = record["tbl"].split("/")
|
|
1399
|
+
service = service or (p[1] if len(p) > 1 else None)
|
|
1400
|
+
group = p[2] if len(p) > 2 else None
|
|
1401
|
+
uploader = record.get("usr") or uploader
|
|
1402
|
+
if not owner and service == self.service:
|
|
1403
|
+
owner = self.owner
|
|
1404
|
+
origin = None
|
|
1405
|
+
for u in bins:
|
|
1406
|
+
if isinstance(u, str) and service and f"/{service}/" in u:
|
|
1407
|
+
s = urlsplit(u)
|
|
1408
|
+
origin = f"{s.scheme}://{s.netloc}"
|
|
1409
|
+
break
|
|
1410
|
+
origin = origin or getattr(self, "_cdn_origins", {}).get(service)
|
|
1411
|
+
rec = record.get("rec")
|
|
1412
|
+
ids_ok = all(isinstance(x, str) and re.match(r"^[0-9A-Za-z_-]+$", x) for x in (service, owner, uploader, rec))
|
|
1413
|
+
if not (origin and ids_ok and isinstance(group, str) and re.match(r"^(\d{2}|\*\*)$", group)):
|
|
1414
|
+
return None
|
|
1415
|
+
return f"{origin}/{'publ' if group == '00' else 'auth'}/{service}/{owner}/{uploader}/records/{rec}/{group}/bin/{marker_path}"
|
|
1416
|
+
|
|
1417
|
+
|
|
1418
|
+
def _json_bytes(v: Any) -> bytes:
|
|
1419
|
+
try:
|
|
1420
|
+
return json.dumps(v, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
|
1421
|
+
except (TypeError, ValueError):
|
|
1422
|
+
raise _err("Invalid data type.", "INVALID_REQUEST") from None
|
|
1423
|
+
|
|
1424
|
+
|
|
1425
|
+
def _prepare_files(files: Optional[list]) -> List[dict]:
|
|
1426
|
+
"""[{"name", "file": path | bytes | file object, "filename"?, "content_type"?}] -> name, filename, content, content_type."""
|
|
1427
|
+
out = []
|
|
1428
|
+
for i, f in enumerate(files or []):
|
|
1429
|
+
if not isinstance(f, dict) or not f.get("name") or "file" not in f:
|
|
1430
|
+
raise _err(f'"files[{i}]" should be {{"name": <field name>, "file": <path, bytes or file object>}}.')
|
|
1431
|
+
src = f["file"]
|
|
1432
|
+
filename = f.get("filename")
|
|
1433
|
+
if isinstance(src, (str, os.PathLike)):
|
|
1434
|
+
path = os.fspath(src)
|
|
1435
|
+
try:
|
|
1436
|
+
with open(path, "rb") as fh:
|
|
1437
|
+
content = fh.read()
|
|
1438
|
+
except OSError as err:
|
|
1439
|
+
raise SkapiError(f'"files[{i}].file" cannot be read: {err.strerror or err}.', "INVALID_PARAMETER", err) from None
|
|
1440
|
+
filename = filename or os.path.basename(path)
|
|
1441
|
+
elif isinstance(src, (bytes, bytearray)):
|
|
1442
|
+
content = bytes(src)
|
|
1443
|
+
elif hasattr(src, "read"):
|
|
1444
|
+
content = src.read()
|
|
1445
|
+
if isinstance(content, str):
|
|
1446
|
+
content = content.encode("utf-8")
|
|
1447
|
+
filename = filename or os.path.basename(getattr(src, "name", "") or "")
|
|
1448
|
+
else:
|
|
1449
|
+
raise _err(f'"files[{i}].file" should be a path, bytes or a file object.')
|
|
1450
|
+
if not filename:
|
|
1451
|
+
raise _err(f'"files[{i}].filename" is required for bytes.')
|
|
1452
|
+
content_type = f.get("content_type") or mimetypes.guess_type(filename)[0]
|
|
1453
|
+
out.append({"name": f["name"], "filename": filename, "content": content, "content_type": content_type})
|
|
1454
|
+
return out
|