sprawdzai-cli 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
sai/commands/sync.py ADDED
@@ -0,0 +1,687 @@
1
+ import typer
2
+ from ..state import resolve_backend
3
+ from typing import Optional
4
+ from ..utils.error_handler import handle_response_error, warn, write_error
5
+ from ..utils.client import APIClient
6
+ from pathlib import Path
7
+ import json
8
+ import hashlib
9
+ import aiofiles
10
+ import asyncio
11
+ from tqdm import tqdm
12
+ import os
13
+
14
+ def parse_time(t: str | int) -> int:
15
+ if type(t) == int:
16
+ return t
17
+ if type(t) == str:
18
+ try:
19
+ spt = list(reversed(t.split(':')))
20
+ mults = [ 1, 60, 60 * 60, 24 * 60 * 60 ]
21
+ if len(spt) > len(mults):
22
+ write_error(f"Got incorrect time limit value: {t}")
23
+ return 0
24
+ res = 0
25
+ for i in range(len(spt)):
26
+ res += mults[i] * int(spt[i])
27
+ return res
28
+ except typer.Exit:
29
+ raise
30
+ except:
31
+ pass
32
+
33
+ write_error(f"Got incorrect time limit value: {t}")
34
+ return 0
35
+
36
+ def parse_memory(m: str | int) -> int:
37
+ if type(m) == int:
38
+ return m
39
+ if type(m) == str and len(m) >= 2:
40
+ try:
41
+ if m[-1] == 'b':
42
+ return round(float(m[:-1]))
43
+ if m[-1] == 'k':
44
+ return round(float(m[:-1]) * 1024)
45
+ if m[-1] == 'm':
46
+ return round(float(m[:-1]) * 1024 * 1024)
47
+ if m[-1] == 'g':
48
+ return round(float(m[:-1]) * 1024 * 1024 * 1024)
49
+ except:
50
+ pass
51
+
52
+ write_error(f"Got incorrect memory limit value: {m}")
53
+ return 0
54
+
55
+ def read_dotsai(path) -> dict:
56
+ try:
57
+ with open(path / '.sai', 'r', encoding='UTF-8') as f:
58
+ txt = f.read()
59
+ try:
60
+ obj = json.loads(txt.split('\n')[-1])
61
+ if 'id' not in obj.keys():
62
+ write_error('.sai file is broken (does not contain "id" field)')
63
+ return obj
64
+ except typer.Exit:
65
+ raise
66
+ except:
67
+ write_error('.sai file is not in valid format')
68
+ except typer.Exit:
69
+ raise
70
+ except Exception:
71
+ write_error('Could not open .sai file (are you in a correct task folder?)')
72
+
73
+ def write_dotsai(path, obj):
74
+ with open(path / '.sai', 'w', encoding='UTF-8') as f:
75
+ res = json.dumps(obj, separators=(',', ':'), ensure_ascii=False)
76
+ f.write(f'SprawdzAI CLI internal file. Do not edit manually.\n' + res)
77
+
78
+ def read_metadata(path) -> dict:
79
+ try:
80
+ with open(path / 'metadata.json', 'r', encoding='UTF-8') as f:
81
+ txt = f.read()
82
+ try:
83
+ return json.loads(txt)
84
+ except typer.Exit:
85
+ raise
86
+ except:
87
+ write_error('metadata.json is not a json')
88
+ return {}
89
+ except typer.Exit:
90
+ raise
91
+ except:
92
+ return {}
93
+
94
+ def read_field_file(path: Path, task_id: int) -> bytes:
95
+ if not path.name.endswith(".ipynb"):
96
+ with open(path, "rb") as f:
97
+ return f.read()
98
+
99
+ try:
100
+ with open(path, "r", encoding="utf-8") as f:
101
+ notebook = json.load(f)
102
+
103
+ assert "metadata" in notebook, 'Notatnik nie zawiera pola "metadata"'
104
+
105
+ if "kernelspec" in notebook["metadata"]:
106
+ del notebook["metadata"]["kernelspec"]
107
+
108
+ if "language_info" in notebook["metadata"]:
109
+ notebook["metadata"]["language_info"]["version"] = "3.12.3"
110
+
111
+ notebook["metadata"].setdefault("sprawdzai", {})
112
+ notebook["metadata"]["sprawdzai"]["source"] = "SprawdzAI"
113
+ notebook["metadata"]["sprawdzai"]["task_id"] = task_id
114
+
115
+ modified = json.dumps(notebook, indent=1, ensure_ascii=False)
116
+ return modified.encode("utf-8")
117
+
118
+ except Exception as e:
119
+ write_error(f"[Task {task_id}] Failed to process file {path}: {e}")
120
+
121
+ async def calc_md5(path: Path, task_id: int) -> str | None:
122
+ if not path.exists():
123
+ return None
124
+ data = read_field_file(path, task_id)
125
+ hash_md5 = hashlib.md5()
126
+ hash_md5.update(data)
127
+ return hash_md5.hexdigest()
128
+
129
+ def validate(n: str, v):
130
+ # TODO: handle everything?
131
+
132
+ if n == 'score_mode':
133
+ return v in ["raw", "normalized", "place-based"], 'Expected one of: "raw", "normalized", "place-based"'
134
+
135
+ if n.endswith('cpus'):
136
+ if type(v) != int:
137
+ return False, "Expected int"
138
+ return 0 <= v, "Expected to be a non-negative int"
139
+
140
+ if n.endswith('vram') or n.endswith('ram') or n.endswith('time'):
141
+ return type(v) == int and 0 <= v, "Expected to be a non-negative integer"
142
+
143
+ if n == 'difficulty':
144
+ return v is None or (type(v) in [int, float] and 1.0 <= v and v <= 7.0), "Expected to be null or in range [1.0, 7.0]"
145
+
146
+ if n == 'tags_ids':
147
+ if type(v) != list:
148
+ return False, "Expected to be a list of integers"
149
+ for el in v:
150
+ if type(el) != int:
151
+ return False, "Expected all elements to be integers"
152
+
153
+ return True, ""
154
+
155
+ def compare(v1, v2):
156
+ if v1 is None and v2 is None:
157
+ return True
158
+ if v1 is None or v2 is None:
159
+ return False
160
+
161
+ if type(v1) in [int, float]:
162
+ if type(v2) not in [int, float]:
163
+ return False
164
+ return v1 == v2
165
+
166
+ if type(v1) in [str, bool]:
167
+ if type(v1) != type(v2):
168
+ return False
169
+ return v1 == v2
170
+
171
+ if type(v1) == list:
172
+ if type(v2) != list:
173
+ return False
174
+ if len(v1) != len(v2):
175
+ return False
176
+ for i in range(len(v1)):
177
+ if type(v1[i]) != type(v2[i]) or type(v1[i]) != type(v1[0]):
178
+ return False
179
+ if v1[i] != v2[i]:
180
+ return False
181
+ return True
182
+
183
+ raise Exception(f'wtf: {v1}, {v2}')
184
+
185
+ def find_filename(path: Path, filename: str):
186
+ res = filename
187
+ fnd = False
188
+ for nm in os.listdir(path):
189
+ if nm.startswith(filename):
190
+ if fnd:
191
+ write_error(f"Error: Found two files for field '{filename}'")
192
+ fnd = True
193
+ res = nm
194
+ return res
195
+
196
+ # Prepare backend metadata for same format
197
+ def prepare_backend_metadata(task):
198
+ return {
199
+ 'title': task['title'],
200
+ 'short_description': task['short_description'],
201
+ 'slug': task['slug'],
202
+ 'tags_ids': [tag['id'] for tag in task['tags']],
203
+ 'difficulty': task['difficulty'],
204
+
205
+ 'notebook_time': task['notebook_config']['time_limit'],
206
+ 'notebook_ram': task['notebook_config']['ram_limit'],
207
+ 'notebook_vram': task['notebook_config']['vram_limit'],
208
+ 'notebook_cpus': int(task['notebook_config']['cpus'] * 100),
209
+ 'notebook_cpu_only': task['notebook_config']['cpu_only'],
210
+ 'notebook_allow_internet': task['notebook_config']['allow_internet'],
211
+
212
+ 'checker_time': task['checker_config']['time_limit'],
213
+ 'checker_ram': task['checker_config']['ram_limit'],
214
+ 'checker_vram': task['checker_config']['vram_limit'],
215
+ 'checker_cpus': int(task['checker_config']['cpus'] * 100),
216
+ 'checker_cpu_only': task['checker_config']['cpu_only'],
217
+ 'checker_allow_internet': task['checker_config']['allow_internet'],
218
+
219
+ 'compute': task['compute'],
220
+ 'submission_files': json.dumps(task['submission_files']),
221
+
222
+ 'valid_daily_submissions': task['nonfinal_submissions_per_day'],
223
+ 'valid_notes_char_limit': task['nonfinal_notes_char_limit'],
224
+ 'valid_stdout_char_limit': task['nonfinal_stdout_char_limit'],
225
+ 'valid_stderr_char_limit': task['nonfinal_stderr_char_limit'],
226
+
227
+ 'test_daily_submissions': task['final_submissions_per_day'],
228
+ 'test_notes_char_limit': task['final_notes_char_limit'],
229
+ 'test_stdout_char_limit': task['final_stdout_char_limit'],
230
+ 'test_stderr_char_limit': task['final_stderr_char_limit'],
231
+
232
+ 'display_ranking': task['display_ranking'],
233
+ 'score_mode': task['score_mode'],
234
+ 'baseline_score': task['baseline_score'],
235
+
236
+ 'statement': task['statement']['name'] + '$' + task['statement']['hash_md5'],
237
+ 'baseline': task['baseline']['name'] + '$' + task['baseline']['hash_md5'],
238
+ 'checker': task['checker']['name'] + '$' + task['checker']['hash_md5'],
239
+ 'subprocess': task['subprocess']['name'] + '$' + task['subprocess']['hash_md5'],
240
+ 'initiator': task['initiator']['name'] + '$' + task['initiator']['hash_md5'],
241
+ 'requirement': task['requirement']['name'] + '$' + task['requirement']['hash_md5'],
242
+ 'image': 'none' if task['image'] is None else task['image']['name'] + '$' + task['image']['hash_md5'],
243
+ }
244
+
245
+ # Prepare local metadata.json for same format
246
+ def prepare_local_metadata(metadata):
247
+ res = {}
248
+
249
+ if 'title' in metadata:
250
+ res['title'] = metadata['title']
251
+ if 'short_description' in metadata:
252
+ res['short_description'] = metadata['short_description']
253
+ if 'slug' in metadata:
254
+ res['slug'] = metadata['slug']
255
+ if 'tags_ids' in metadata:
256
+ res['tags_ids'] = metadata['tags_ids']
257
+ if 'difficulty' in metadata:
258
+ res['difficulty'] = metadata['difficulty']
259
+
260
+ for s, g in [('notebook_limits', 'notebook'), ('checker_limits', 'checker')]:
261
+ if s in metadata:
262
+ el = metadata[s]
263
+ if 'time' in el:
264
+ res[f'{g}_time'] = parse_time(el['time'])
265
+ if 'ram' in el:
266
+ res[f'{g}_ram'] = parse_memory(el['ram'])
267
+ if 'vram' in el:
268
+ res[f'{g}_vram'] = parse_memory(el['vram'])
269
+ if 'cpus' in el:
270
+ res[f'{g}_cpus'] = el['cpus']
271
+ if 'cpu_only' in el:
272
+ res[f'{g}_cpu_only'] = el['cpu_only']
273
+ if 'allow_internet' in el:
274
+ res[f'{g}_allow_internet'] = el['allow_internet']
275
+
276
+ if 'compute' in metadata:
277
+ res['compute'] = metadata['compute']
278
+
279
+ if 'submission_files' in metadata:
280
+ res['submission_files'] = json.dumps([{
281
+ 'name': f['name'],
282
+ 'max_file_size': parse_memory(f['max_file_size'])
283
+ } for f in metadata['submission_files']])
284
+
285
+ for t in ['valid', 'test']:
286
+ if t in metadata:
287
+ el = metadata[t]
288
+ if 'daily_submissions' in el:
289
+ res[f'{t}_daily_submissions'] = el['daily_submissions']
290
+ if 'notes_char_limit' in el:
291
+ res[f'{t}_notes_char_limit'] = el['notes_char_limit']
292
+ if 'stdout_char_limit' in el:
293
+ res[f'{t}_stdout_char_limit'] = el['stdout_char_limit']
294
+ if 'stderr_char_limit' in el:
295
+ res[f'{t}_stderr_char_limit'] = el['stderr_char_limit']
296
+
297
+ if 'display_ranking' in metadata:
298
+ res['display_ranking'] = metadata['display_ranking']
299
+ if 'score_mode' in metadata:
300
+ res['score_mode'] = metadata['score_mode']
301
+ if 'baseline_score' in metadata:
302
+ res['baseline_score'] = metadata['baseline_score']
303
+
304
+ return res
305
+
306
+ # Write final metadata back to metadata.json file
307
+ def write_metadata(path, obj):
308
+ with open(path / 'metadata.json', 'w', encoding='UTF-8') as f:
309
+ json.dump({
310
+ 'title': obj['title'],
311
+ 'short_description': obj['short_description'],
312
+ 'tags_ids': obj['tags_ids'],
313
+ 'slug': obj['slug'],
314
+ 'difficulty': obj['difficulty'],
315
+
316
+ 'notebook_limits': {
317
+ 'time': obj['notebook_time'],
318
+ 'ram': obj['notebook_ram'],
319
+ 'vram': obj['notebook_vram'],
320
+ 'cpus': obj['notebook_cpus'],
321
+ 'cpu_only': obj['notebook_cpu_only'],
322
+ 'allow_internet': obj['notebook_allow_internet'],
323
+ },
324
+ 'checker_limits': {
325
+ 'time': obj['checker_time'],
326
+ 'ram': obj['checker_ram'],
327
+ 'vram': obj['checker_vram'],
328
+ 'cpus': obj['checker_cpus'],
329
+ 'cpu_only': obj['checker_cpu_only'],
330
+ 'allow_internet': obj['checker_allow_internet'],
331
+ },
332
+ 'compute': obj['compute'],
333
+ 'submission_files': json.loads(obj['submission_files']),
334
+ 'valid': {
335
+ 'daily_submissions': obj['valid_daily_submissions'],
336
+ 'notes_char_limit': obj['valid_notes_char_limit'],
337
+ 'stdout_char_limit': obj['valid_stdout_char_limit'],
338
+ 'stderr_char_limit': obj['valid_stderr_char_limit'],
339
+ },
340
+ 'test': {
341
+ 'daily_submissions': obj['test_daily_submissions'],
342
+ 'notes_char_limit': obj['test_notes_char_limit'],
343
+ 'stdout_char_limit': obj['test_stdout_char_limit'],
344
+ 'stderr_char_limit': obj['test_stderr_char_limit'],
345
+ },
346
+
347
+ 'display_ranking': obj['display_ranking'],
348
+ 'score_mode': obj['score_mode'],
349
+ 'baseline_score': obj['baseline_score'],
350
+ }, f, indent=4, ensure_ascii=False)
351
+
352
+ # Backend request
353
+ def get_backend_request(metadata, datasets):
354
+ return {
355
+ 'title': metadata['title'],
356
+ 'short_description': metadata['short_description'],
357
+ 'slug': metadata['slug'],
358
+ 'tags_ids': metadata['tags_ids'],
359
+ 'difficulty': metadata['difficulty'],
360
+
361
+ 'notebook_config': json.dumps({
362
+ 'time_limit': metadata['notebook_time'],
363
+ 'ram_limit': metadata['notebook_ram'],
364
+ 'vram_limit': metadata['notebook_vram'],
365
+ 'cpus': metadata['notebook_cpus'] / 100,
366
+ 'cpu_only': metadata['notebook_cpu_only'],
367
+ 'allow_internet': metadata['notebook_allow_internet'],
368
+ }),
369
+ 'checker_config': json.dumps({
370
+ 'time_limit': metadata['checker_time'],
371
+ 'ram_limit': metadata['checker_ram'],
372
+ 'vram_limit': metadata['checker_vram'],
373
+ 'cpus': metadata['checker_cpus'] / 100,
374
+ 'cpu_only': metadata['checker_cpu_only'],
375
+ 'allow_internet': metadata['checker_allow_internet'],
376
+ }),
377
+
378
+ 'compute': metadata['compute'],
379
+ 'submission_files': metadata['submission_files'],
380
+
381
+ 'nonfinal_submissions_per_day': metadata['valid_daily_submissions'],
382
+ 'nonfinal_notes_char_limit': metadata['valid_notes_char_limit'],
383
+ 'nonfinal_stdout_char_limit': metadata['valid_stdout_char_limit'],
384
+ 'nonfinal_stderr_char_limit': metadata['valid_stderr_char_limit'],
385
+
386
+ 'final_submissions_per_day': metadata['test_daily_submissions'],
387
+ 'final_notes_char_limit': metadata['test_notes_char_limit'],
388
+ 'final_stdout_char_limit': metadata['test_stdout_char_limit'],
389
+ 'final_stderr_char_limit': metadata['test_stderr_char_limit'],
390
+
391
+ 'notes_char_limit': metadata['valid_notes_char_limit'],
392
+ 'stdout_char_limit': metadata['valid_stdout_char_limit'],
393
+ 'stderr_char_limit': metadata['valid_stderr_char_limit'],
394
+
395
+ 'display_ranking': metadata['display_ranking'],
396
+ 'score_mode': metadata['score_mode'],
397
+ 'baseline_score': metadata['baseline_score'],
398
+
399
+ 'datasets': json.dumps([{'type': d['type_'], 'file_id': d['id']} for d in datasets])
400
+ }
401
+
402
+ def sync_task(path: Path, client: APIClient, force_remote: bool, simulate: bool):
403
+ dotsai = read_dotsai(path)
404
+ metadata = prepare_local_metadata(read_metadata(path))
405
+
406
+ # Load task from backend
407
+ task_resp = client.get(f'/tasks/{dotsai["id"]}')
408
+ if task_resp.status_code != 200:
409
+ handle_response_error(task_resp)
410
+
411
+ orig_task = task_resp.json()
412
+
413
+ if not orig_task['has_edit_permission']:
414
+ write_error('You do not have permission to edit this task.')
415
+
416
+ # checklib
417
+ checklib_path = path / "checklib.py"
418
+ with open(Path(__file__).resolve().parent / "checklib.py", 'r') as f:
419
+ checklib = f.read()
420
+ if checklib_path.exists():
421
+ with open(checklib_path, 'r') as f:
422
+ if f.read() != checklib:
423
+ warn('The checklib.py file has been modified. This is just a reference file kept in task folder mostly for autocompletion support. Its contents will now be overwritten with the standard file provided by SprawdzAI creators.')
424
+ res = typer.prompt(f'Do you wish to continue? [y]es/[n]o')
425
+ if type(res) != str or (res.lower() not in ['y', 'yes']):
426
+ typer.secho("Operation aborted", fg='red')
427
+ return
428
+ with open(checklib_path, 'w') as f:
429
+ f.write(checklib)
430
+
431
+ # Fill to same format as metadata
432
+ task = prepare_backend_metadata(orig_task)
433
+ new_metadata = prepare_backend_metadata(orig_task)
434
+
435
+ # Load previous response
436
+ prev_task = dotsai.get('response', task)
437
+
438
+ # Update local metadata with file hashes
439
+ target_file_map = {
440
+ 'statement': 'statement.ipynb',
441
+ 'checker': 'checker.py',
442
+ 'subprocess': 'subprocess_script.py',
443
+ 'initiator': 'initiator.py',
444
+ 'requirement': 'requirements.txt',
445
+ 'image': find_filename(path, 'image'),
446
+ 'baseline': find_filename(path, 'baseline'),
447
+ }
448
+ for field, filename in target_file_map.items():
449
+ hash_md5 = asyncio.run(calc_md5(path / filename, dotsai["id"]))
450
+ if hash_md5 is not None:
451
+ metadata[field] = filename + '$' + hash_md5
452
+
453
+ no_baseline = not (path / target_file_map['baseline']).exists()
454
+ no_image = not (path / target_file_map['image']).exists()
455
+ if no_baseline and 'statement' in metadata:
456
+ hash_md5 = asyncio.run(calc_md5(path / 'statement.ipynb', dotsai["id"]))
457
+ metadata['baseline'] = 'baseline.ipynb' + '$' + hash_md5
458
+ target_file_map['baseline'] = 'baseline.ipynb'
459
+ if no_image:
460
+ metadata['image'] = 'none'
461
+
462
+ # Validate local metadata
463
+ for n in metadata.keys():
464
+ if n not in new_metadata.keys():
465
+ write_error(f'metadata.json has unexpected key {n}')
466
+ is_valid, msg = validate(n, metadata[n])
467
+ if not is_valid:
468
+ write_error(f'Error in metadata.json field {n}: {msg}')
469
+
470
+ # Resolve all changes
471
+ sent_fileds = []
472
+ pulled_fileds = []
473
+ conflicted_fileds = []
474
+
475
+ for n in new_metadata.keys():
476
+ if n not in metadata.keys():
477
+ pulled_fileds.append(n)
478
+ continue
479
+
480
+ if compare(metadata[n], task[n]):
481
+ if not compare(task[n], prev_task[n]):
482
+ warn(f"Field {n} has changed on backend, but it has also changed locally in the same way, so no changes are made for this field.")
483
+ continue
484
+
485
+ if force_remote:
486
+ pulled_fileds.append(n)
487
+ continue
488
+
489
+ if compare(task[n], prev_task[n]):
490
+ sent_fileds.append(n)
491
+ new_metadata[n] = metadata[n]
492
+ continue
493
+
494
+ if compare(metadata[n], prev_task[n]):
495
+ pulled_fileds.append(n)
496
+ else:
497
+ conflicted_fileds.append(n)
498
+
499
+ # Show info about changes
500
+ send_to_backend = False
501
+ if len(sent_fileds) == 0 and len(pulled_fileds) == 0 and len(conflicted_fileds) == 0:
502
+ typer.echo('No changes were made')
503
+ if len(sent_fileds) != 0:
504
+ send_to_backend = True
505
+ typer.echo(f'Sending {len(sent_fileds)} fields local -> backend ({", ".join(sent_fileds)})')
506
+ if len(pulled_fileds) != 0:
507
+ typer.echo(f'Pulling {len(pulled_fileds)} fields backend -> local ({", ".join(pulled_fileds)})')
508
+
509
+ # Resolve conflicts
510
+ save_metadata = {k: v for k, v in new_metadata.items()}
511
+ dotsai_metadata = {k: v for k, v in new_metadata.items()}
512
+ if len(conflicted_fileds) != 0:
513
+ if len(conflicted_fileds) == 1:
514
+ typer.secho(f'There is 1 conflicted filed (both local and backend changed):', fg=typer.colors.RED)
515
+ else:
516
+ typer.secho(f'There are {len(conflicted_fileds)} conflicted fileds (both local and backend changed):', fg=typer.colors.RED)
517
+ if not simulate:
518
+ typer.echo('[i]gnore: Ignore conflict for now - keep local changes but don\'t send them to backend')
519
+ typer.echo('[p]ull: Pull changes backend -> local, will overwrite local changes')
520
+ typer.echo('[s]end: Send changes local -> backend, will keep local changes and overwrite on backend')
521
+ for n in conflicted_fileds:
522
+ typer.echo(f' > Field ', nl=False)
523
+ typer.secho(n, nl=False, fg='blue')
524
+ typer.echo(':', nl=True)
525
+ typer.echo(f' > Original value: {prev_task[n]}')
526
+ typer.echo(f' > Modified value on backend: {task[n]}')
527
+ typer.echo(f' > Locally modified value: {metadata[n]}')
528
+ if simulate:
529
+ typer.echo('')
530
+ continue
531
+ valid_choices = {"i": "ignore", "p": "pull", "s": "send"}
532
+ while True:
533
+ choice = input(" > [i]gnore, [p]ull, [s]end: ").strip().lower()
534
+ if choice in valid_choices.keys() or choice in valid_choices.values():
535
+ if choice in valid_choices.keys():
536
+ choice = valid_choices[choice]
537
+ if choice == 'ignore':
538
+ save_metadata[n] = metadata[n]
539
+ dotsai_metadata[n] = prev_task[n]
540
+ if choice == 'send':
541
+ send_to_backend = True
542
+ new_metadata[n] = metadata[n]
543
+ dotsai_metadata[n] = metadata[n]
544
+ save_metadata[n] = metadata[n]
545
+ sent_fileds.append(n)
546
+ if choice == 'pull':
547
+ save_metadata[n] = task[n]
548
+ pulled_fileds.append(n)
549
+ break
550
+ else:
551
+ typer.echo("Invalid choice.")
552
+
553
+ if simulate:
554
+ return
555
+
556
+ # Send to backend
557
+ if send_to_backend:
558
+ files = {}
559
+ for field in sent_fileds:
560
+ if field in target_file_map.keys():
561
+ if field == 'baseline' and no_baseline:
562
+ filepath = path / 'statement.ipynb'
563
+ else:
564
+ filepath = path / target_file_map[field]
565
+ if field == 'image' and no_image:
566
+ files[field] = (target_file_map[field], b'')
567
+ else:
568
+ files[field] = (target_file_map[field], read_field_file(filepath, dotsai["id"]))
569
+ resp = client.put(
570
+ f'/tasks/{dotsai["id"]}',
571
+ data=get_backend_request(new_metadata, orig_task['datasets']),
572
+ files=files
573
+ )
574
+ if resp.status_code != 200:
575
+ handle_response_error(resp)
576
+ if len(files) > 0:
577
+ typer.secho(f'Updated backend (sent {len(files)} files)!', fg=typer.colors.GREEN)
578
+ else:
579
+ typer.secho(f'Updated backend!', fg=typer.colors.GREEN)
580
+
581
+ # Download files
582
+ files_to_download = []
583
+
584
+ for field in pulled_fileds:
585
+ if field in target_file_map.keys():
586
+
587
+ if field == 'baseline' and no_baseline:
588
+ if orig_task['baseline']['hash_md5'] == orig_task['statement']['hash_md5']:
589
+ continue
590
+ else:
591
+ spt = orig_task['baseline']['name'].split('.')
592
+ if len(spt) <= 1:
593
+ target_file_map['baseline'] = 'baseline'
594
+ else:
595
+ target_file_map['baseline'] = 'baseline.' + spt[-1]
596
+
597
+ if field == 'image':
598
+ if orig_task['image'] == None:
599
+ if not no_image:
600
+ (path / target_file_map['image']).unlink()
601
+ continue
602
+ elif no_image:
603
+ spt = orig_task['image']['name'].split('.')
604
+ if len(spt) <= 1:
605
+ target_file_map['image'] = 'image'
606
+ else:
607
+ target_file_map['image'] = 'image.' + spt[-1]
608
+
609
+ files_to_download.append(field)
610
+
611
+ if len(files_to_download) > 0:
612
+ for field in tqdm(files_to_download, desc='Downloading files'):
613
+ with client.get(f'/files/{orig_task[field]["id"]}/download', stream=True) as resp:
614
+ if resp.status_code != 200:
615
+ handle_response_error(resp)
616
+ with (path / target_file_map[field]).open("wb") as f:
617
+ for chunk in resp.iter_content(chunk_size=8192):
618
+ if chunk:
619
+ f.write(chunk)
620
+
621
+ # Final save
622
+ write_metadata(path, save_metadata)
623
+ write_dotsai(path, {
624
+ 'id': dotsai['id'],
625
+ 'response': dotsai_metadata
626
+ })
627
+
628
+ def sync(
629
+ simulate: bool = typer.Option(
630
+ False,
631
+ "--simulate",
632
+ "-s",
633
+ help=(
634
+ "With this flag, you are guaranteed no changes will be made on backend or locally. "
635
+ "Use this to see all logs sai would normally produce, but without affecting anything. "
636
+ "Useful for example for seeing what changes have been made."
637
+ )
638
+ ),
639
+ force_local: bool = typer.Option(
640
+ False,
641
+ "--force-local",
642
+ "--fl",
643
+ help=(
644
+ "This will untrack any changes on backend and will overwrite everything with local changes. "
645
+ "In other words, it will make exact copy of local version on backend ignoring any changes "
646
+ "that might have came from other sources (someone else editing task or you editing it on the website). "
647
+ ),
648
+ ),
649
+ force_remote: bool = typer.Option(
650
+ False,
651
+ "--force-remote",
652
+ "--fr",
653
+ help=(
654
+ "Similar to --force-local, but will overwrite any local changes with what is on backend. "
655
+ "In other words, it will make exact copy of remote version locally."
656
+ ),
657
+ ),
658
+ path: Path = typer.Option(
659
+ Path.cwd(),
660
+ "--location",
661
+ "-l",
662
+ help="Path where the task is located (defaults to current directory)."
663
+ ),
664
+ backend: Optional[str] = typer.Option(
665
+ None,
666
+ "--backend",
667
+ "-b",
668
+ help=(
669
+ "Use a different backend: "
670
+ "local (l, l1, l2, ...), "
671
+ "remote (r, r1, r2, ...) "
672
+ "or full URL (https://example.com)"
673
+ )
674
+ )
675
+ ):
676
+ """
677
+ Sync a task with server.
678
+ """
679
+
680
+ if force_local:
681
+ dotsai = read_dotsai(path)
682
+ write_dotsai(path, {'id': dotsai['id']})
683
+
684
+ backend = resolve_backend(backend)
685
+ client = APIClient(backend)
686
+
687
+ sync_task(path, client, force_remote, simulate)