QuLab 2.7.19__cp311-cp311-macosx_10_9_universal2.whl → 2.9.0__cp311-cp311-macosx_10_9_universal2.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qulab/__init__.py +1 -0
- qulab/executor/cli.py +5 -0
- qulab/executor/storage.py +380 -143
- qulab/executor/template.py +76 -13
- qulab/fun.cpython-311-darwin.so +0 -0
- qulab/utils.py +7 -7
- qulab/version.py +1 -1
- {qulab-2.7.19.dist-info → qulab-2.9.0.dist-info}/METADATA +3 -2
- {qulab-2.7.19.dist-info → qulab-2.9.0.dist-info}/RECORD +13 -13
- {qulab-2.7.19.dist-info → qulab-2.9.0.dist-info}/WHEEL +1 -1
- {qulab-2.7.19.dist-info → qulab-2.9.0.dist-info}/entry_points.txt +0 -0
- {qulab-2.7.19.dist-info → qulab-2.9.0.dist-info/licenses}/LICENSE +0 -0
- {qulab-2.7.19.dist-info → qulab-2.9.0.dist-info}/top_level.txt +0 -0
qulab/__init__.py
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
from .executor.storage import find_report
|
2
2
|
from .executor.storage import get_report_by_index as get_report
|
3
|
+
from .executor.template import VAR
|
3
4
|
from .executor.utils import debug_analyze
|
4
5
|
from .scan import Scan, get_record, load_record, lookup, lookup_list
|
5
6
|
from .version import __version__
|
qulab/executor/cli.py
CHANGED
@@ -2,6 +2,7 @@ import functools
|
|
2
2
|
import graphlib
|
3
3
|
import importlib
|
4
4
|
import os
|
5
|
+
import sys
|
5
6
|
from pathlib import Path
|
6
7
|
|
7
8
|
import click
|
@@ -68,6 +69,10 @@ def command_option(command_name):
|
|
68
69
|
help='The path of the bootstrap.')
|
69
70
|
@functools.wraps(func)
|
70
71
|
def wrapper(*args, **kwargs):
|
72
|
+
if 'code' in kwargs and kwargs['code'] is not None:
|
73
|
+
code = os.path.expanduser(kwargs['code'])
|
74
|
+
if code not in sys.path:
|
75
|
+
sys.path.insert(0, code)
|
71
76
|
bootstrap = kwargs.pop('bootstrap')
|
72
77
|
if bootstrap is not None:
|
73
78
|
boot(bootstrap)
|
qulab/executor/storage.py
CHANGED
@@ -1,15 +1,42 @@
|
|
1
1
|
import hashlib
|
2
2
|
import lzma
|
3
3
|
import pickle
|
4
|
+
import re
|
4
5
|
import uuid
|
6
|
+
import zipfile
|
5
7
|
from dataclasses import dataclass, field
|
6
8
|
from datetime import datetime, timedelta
|
7
9
|
from functools import lru_cache
|
8
10
|
from pathlib import Path
|
9
11
|
from typing import Any, Literal
|
12
|
+
from urllib.parse import parse_qs
|
10
13
|
|
11
14
|
from loguru import logger
|
12
15
|
|
16
|
+
try:
|
17
|
+
from paramiko import SSHClient
|
18
|
+
from paramiko.ssh_exception import SSHException
|
19
|
+
except:
|
20
|
+
import warnings
|
21
|
+
|
22
|
+
warnings.warn("Can't import paramiko, ssh support will be disabled.")
|
23
|
+
|
24
|
+
class SSHClient:
|
25
|
+
|
26
|
+
def __init__(self):
|
27
|
+
raise ImportError(
|
28
|
+
"Can't import paramiko, ssh support will be disabled.")
|
29
|
+
|
30
|
+
def __enter__(self):
|
31
|
+
return self
|
32
|
+
|
33
|
+
def __exit__(self, exc_type, exc_value, traceback):
|
34
|
+
pass
|
35
|
+
|
36
|
+
class SSHException(Exception):
|
37
|
+
pass
|
38
|
+
|
39
|
+
|
13
40
|
from ..cli.config import get_config_value
|
14
41
|
|
15
42
|
__current_config_cache = None
|
@@ -31,7 +58,7 @@ class Report():
|
|
31
58
|
index: int = -1
|
32
59
|
previous_path: Path | None = field(default=None, repr=False)
|
33
60
|
heads: dict[str, Path] = field(default_factory=dict, repr=False)
|
34
|
-
base_path: Path | None = field(default=None, repr=False)
|
61
|
+
base_path: str | Path | None = field(default=None, repr=False)
|
35
62
|
path: Path | None = field(default=None, repr=False)
|
36
63
|
config_path: Path | None = field(default=None, repr=False)
|
37
64
|
script_path: Path | None = field(default=None, repr=False)
|
@@ -43,7 +70,7 @@ class Report():
|
|
43
70
|
if state[k] is not None:
|
44
71
|
state[k] = str(state[k])
|
45
72
|
return state
|
46
|
-
|
73
|
+
|
47
74
|
def __setstate__(self, state):
|
48
75
|
for k in ['path', 'previous_path', 'config_path', 'script_path']:
|
49
76
|
if state[k] is not None:
|
@@ -101,6 +128,17 @@ class Report():
|
|
101
128
|
source = load_item(self.script_path, self.base_path)
|
102
129
|
if isinstance(source, str):
|
103
130
|
return source
|
131
|
+
else:
|
132
|
+
from .template import inject_mapping
|
133
|
+
return inject_mapping(*source)[0]
|
134
|
+
else:
|
135
|
+
return None
|
136
|
+
|
137
|
+
@property
|
138
|
+
def template_source(self):
|
139
|
+
if self.script_path is not None and self.base_path is not None:
|
140
|
+
source = load_item(self.script_path, self.base_path)
|
141
|
+
return source
|
104
142
|
else:
|
105
143
|
return None
|
106
144
|
|
@@ -113,106 +151,6 @@ def random_path(base: Path) -> Path:
|
|
113
151
|
return path
|
114
152
|
|
115
153
|
|
116
|
-
def save_config_key_history(key: str, report: Report,
|
117
|
-
base_path: str | Path) -> int:
|
118
|
-
global __current_config_cache
|
119
|
-
base_path = Path(base_path) / 'state'
|
120
|
-
base_path.mkdir(parents=True, exist_ok=True)
|
121
|
-
|
122
|
-
if __current_config_cache is None:
|
123
|
-
if (base_path / 'parameters.pkl').exists():
|
124
|
-
with open(base_path / 'parameters.pkl', 'rb') as f:
|
125
|
-
__current_config_cache = pickle.load(f)
|
126
|
-
else:
|
127
|
-
__current_config_cache = {}
|
128
|
-
|
129
|
-
__current_config_cache[
|
130
|
-
key] = report.data, report.calibrated_time, report.checked_time
|
131
|
-
|
132
|
-
with open(base_path / 'parameters.pkl', 'wb') as f:
|
133
|
-
pickle.dump(__current_config_cache, f)
|
134
|
-
return 0
|
135
|
-
|
136
|
-
|
137
|
-
def find_config_key_history(key: str, base_path: str | Path) -> Report | None:
|
138
|
-
global __current_config_cache
|
139
|
-
base_path = Path(base_path) / 'state'
|
140
|
-
if __current_config_cache is None:
|
141
|
-
if (base_path / 'parameters.pkl').exists():
|
142
|
-
with open(base_path / 'parameters.pkl', 'rb') as f:
|
143
|
-
__current_config_cache = pickle.load(f)
|
144
|
-
else:
|
145
|
-
__current_config_cache = {}
|
146
|
-
|
147
|
-
if key in __current_config_cache:
|
148
|
-
value, calibrated_time, checked_time = __current_config_cache.get(
|
149
|
-
key, None)
|
150
|
-
report = Report(
|
151
|
-
workflow=f'cfg:{key}',
|
152
|
-
bad_data=False,
|
153
|
-
in_spec=True,
|
154
|
-
fully_calibrated=True,
|
155
|
-
parameters={key: value},
|
156
|
-
data=value,
|
157
|
-
calibrated_time=calibrated_time,
|
158
|
-
checked_time=checked_time,
|
159
|
-
)
|
160
|
-
return report
|
161
|
-
return None
|
162
|
-
|
163
|
-
|
164
|
-
def save_report(workflow: str,
|
165
|
-
report: Report,
|
166
|
-
base_path: str | Path,
|
167
|
-
overwrite: bool = False,
|
168
|
-
refresh_heads: bool = True) -> int:
|
169
|
-
if workflow.startswith("cfg:"):
|
170
|
-
return save_config_key_history(workflow[4:], report, base_path)
|
171
|
-
|
172
|
-
logger.debug(
|
173
|
-
f'Saving report for "{workflow}", {report.in_spec=}, {report.bad_data=}, {report.fully_calibrated=}'
|
174
|
-
)
|
175
|
-
base_path = Path(base_path)
|
176
|
-
try:
|
177
|
-
buf = lzma.compress(pickle.dumps(report))
|
178
|
-
except:
|
179
|
-
raise ValueError(f"Can't pickle report for {workflow}")
|
180
|
-
if overwrite:
|
181
|
-
path = report.path
|
182
|
-
if path is None:
|
183
|
-
raise ValueError("Report path is None, can't overwrite.")
|
184
|
-
with open(base_path / 'reports' / path, "rb") as f:
|
185
|
-
index = int.from_bytes(f.read(8), 'big')
|
186
|
-
report.index = index
|
187
|
-
else:
|
188
|
-
path = random_path(base_path / 'reports')
|
189
|
-
(base_path / 'reports' / path).parent.mkdir(parents=True,
|
190
|
-
exist_ok=True)
|
191
|
-
report.path = path
|
192
|
-
report.index = create_index("report",
|
193
|
-
base_path,
|
194
|
-
context=str(path),
|
195
|
-
width=35)
|
196
|
-
with open(base_path / 'reports' / path, "wb") as f:
|
197
|
-
f.write(report.index.to_bytes(8, 'big'))
|
198
|
-
f.write(buf)
|
199
|
-
if refresh_heads:
|
200
|
-
set_head(workflow, path, base_path)
|
201
|
-
return report.index
|
202
|
-
|
203
|
-
|
204
|
-
def load_report(path: str | Path, base_path: str | Path) -> Report | None:
|
205
|
-
base_path = Path(base_path)
|
206
|
-
path = base_path / 'reports' / path
|
207
|
-
|
208
|
-
with open(base_path / 'reports' / path, "rb") as f:
|
209
|
-
index = int.from_bytes(f.read(8), 'big')
|
210
|
-
report = pickle.loads(lzma.decompress(f.read()))
|
211
|
-
report.base_path = base_path
|
212
|
-
report.index = index
|
213
|
-
return report
|
214
|
-
|
215
|
-
|
216
154
|
def find_report(
|
217
155
|
workflow: str, base_path: str | Path = get_config_value("data", Path)
|
218
156
|
) -> Report | None:
|
@@ -252,6 +190,26 @@ def revoke_report(workflow: str, report: Report | None, base_path: str | Path):
|
|
252
190
|
refresh_heads=True)
|
253
191
|
|
254
192
|
|
193
|
+
def get_report_by_index(
|
194
|
+
index: int, base_path: str | Path = get_config_value("data", Path)
|
195
|
+
) -> Report | None:
|
196
|
+
try:
|
197
|
+
path = query_index("report", base_path, index)
|
198
|
+
return load_report(path, base_path)
|
199
|
+
except:
|
200
|
+
raise
|
201
|
+
return None
|
202
|
+
|
203
|
+
|
204
|
+
def get_head(workflow: str, base_path: str | Path) -> Path | None:
|
205
|
+
return get_heads(base_path).get(workflow, None)
|
206
|
+
|
207
|
+
|
208
|
+
#########################################################################
|
209
|
+
## Basic Write API ##
|
210
|
+
#########################################################################
|
211
|
+
|
212
|
+
|
255
213
|
def set_head(workflow: str, path: Path, base_path: str | Path):
|
256
214
|
base_path = Path(base_path)
|
257
215
|
base_path.mkdir(parents=True, exist_ok=True)
|
@@ -265,24 +223,44 @@ def set_head(workflow: str, path: Path, base_path: str | Path):
|
|
265
223
|
pickle.dump(heads, f)
|
266
224
|
|
267
225
|
|
268
|
-
def
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
return None
|
276
|
-
|
226
|
+
def save_report(workflow: str,
|
227
|
+
report: Report,
|
228
|
+
base_path: str | Path,
|
229
|
+
overwrite: bool = False,
|
230
|
+
refresh_heads: bool = True) -> int:
|
231
|
+
if workflow.startswith("cfg:"):
|
232
|
+
return save_config_key_history(workflow[4:], report, base_path)
|
277
233
|
|
278
|
-
|
234
|
+
logger.debug(
|
235
|
+
f'Saving report for "{workflow}", {report.in_spec=}, {report.bad_data=}, {report.fully_calibrated=}'
|
236
|
+
)
|
279
237
|
base_path = Path(base_path)
|
280
238
|
try:
|
281
|
-
|
282
|
-
heads = pickle.load(f)
|
283
|
-
return heads
|
239
|
+
buf = lzma.compress(pickle.dumps(report))
|
284
240
|
except:
|
285
|
-
|
241
|
+
raise ValueError(f"Can't pickle report for {workflow}")
|
242
|
+
if overwrite:
|
243
|
+
path = report.path
|
244
|
+
if path is None:
|
245
|
+
raise ValueError("Report path is None, can't overwrite.")
|
246
|
+
with open(base_path / 'reports' / path, "rb") as f:
|
247
|
+
index = int.from_bytes(f.read(8), 'big')
|
248
|
+
report.index = index
|
249
|
+
else:
|
250
|
+
path = random_path(base_path / 'reports')
|
251
|
+
(base_path / 'reports' / path).parent.mkdir(parents=True,
|
252
|
+
exist_ok=True)
|
253
|
+
report.path = path
|
254
|
+
report.index = create_index("report",
|
255
|
+
base_path,
|
256
|
+
context=str(path),
|
257
|
+
width=35)
|
258
|
+
with open(base_path / 'reports' / path, "wb") as f:
|
259
|
+
f.write(report.index.to_bytes(8, 'big'))
|
260
|
+
f.write(buf)
|
261
|
+
if refresh_heads:
|
262
|
+
set_head(workflow, path, base_path)
|
263
|
+
return report.index
|
286
264
|
|
287
265
|
|
288
266
|
def create_index(name: str,
|
@@ -318,27 +296,6 @@ def create_index(name: str,
|
|
318
296
|
return index
|
319
297
|
|
320
298
|
|
321
|
-
@lru_cache(maxsize=4096)
|
322
|
-
def query_index(name: str, base_path: str | Path, index: int):
|
323
|
-
path = Path(base_path) / "index" / name
|
324
|
-
width = int(path.with_suffix('.width').read_text())
|
325
|
-
|
326
|
-
with path.with_suffix('.idx').open("r") as f:
|
327
|
-
f.seek(index * (width + 1))
|
328
|
-
context = f.read(width)
|
329
|
-
return context.rstrip()
|
330
|
-
|
331
|
-
|
332
|
-
def get_report_by_index(
|
333
|
-
index: int, base_path: str | Path = get_config_value("data", Path)
|
334
|
-
) -> Report | None:
|
335
|
-
try:
|
336
|
-
path = query_index("report", base_path, index)
|
337
|
-
return load_report(path, base_path)
|
338
|
-
except:
|
339
|
-
return None
|
340
|
-
|
341
|
-
|
342
299
|
def save_item(item, data_path):
|
343
300
|
salt = 0
|
344
301
|
buf = pickle.dumps(item)
|
@@ -361,10 +318,290 @@ def save_item(item, data_path):
|
|
361
318
|
return str(item_id)
|
362
319
|
|
363
320
|
|
321
|
+
def save_config_key_history(key: str, report: Report,
|
322
|
+
base_path: str | Path) -> int:
|
323
|
+
global __current_config_cache
|
324
|
+
base_path = Path(base_path) / 'state'
|
325
|
+
base_path.mkdir(parents=True, exist_ok=True)
|
326
|
+
|
327
|
+
if __current_config_cache is None:
|
328
|
+
if (base_path / 'parameters.pkl').exists():
|
329
|
+
with open(base_path / 'parameters.pkl', 'rb') as f:
|
330
|
+
__current_config_cache = pickle.load(f)
|
331
|
+
else:
|
332
|
+
__current_config_cache = {}
|
333
|
+
|
334
|
+
__current_config_cache[
|
335
|
+
key] = report.data, report.calibrated_time, report.checked_time
|
336
|
+
|
337
|
+
with open(base_path / 'parameters.pkl', 'wb') as f:
|
338
|
+
pickle.dump(__current_config_cache, f)
|
339
|
+
return 0
|
340
|
+
|
341
|
+
|
342
|
+
#########################################################################
|
343
|
+
## Basic Read API ##
|
344
|
+
#########################################################################
|
345
|
+
|
346
|
+
|
347
|
+
def load_report(path: str | Path, base_path: str | Path) -> Report | None:
|
348
|
+
if isinstance(base_path, str) and base_path.startswith('ssh://'):
|
349
|
+
with SSHClient() as client:
|
350
|
+
cfg = parse_ssh_uri(base_path)
|
351
|
+
remote_base_path = cfg.pop('remote_file_path')
|
352
|
+
client.load_system_host_keys()
|
353
|
+
client.connect(**cfg)
|
354
|
+
report = load_report_from_scp(path, remote_base_path, client)
|
355
|
+
report.base_path = base_path
|
356
|
+
return report
|
357
|
+
|
358
|
+
base_path = Path(base_path)
|
359
|
+
if zipfile.is_zipfile(base_path):
|
360
|
+
return load_report_from_zipfile(path, base_path)
|
361
|
+
|
362
|
+
path = base_path / 'reports' / path
|
363
|
+
|
364
|
+
with open(base_path / 'reports' / path, "rb") as f:
|
365
|
+
index = int.from_bytes(f.read(8), 'big')
|
366
|
+
report = pickle.loads(lzma.decompress(f.read()))
|
367
|
+
report.base_path = base_path
|
368
|
+
report.index = index
|
369
|
+
return report
|
370
|
+
|
371
|
+
|
372
|
+
def get_heads(base_path: str | Path) -> Path | None:
|
373
|
+
if isinstance(base_path, str) and base_path.startswith('ssh://'):
|
374
|
+
with SSHClient() as client:
|
375
|
+
cfg = parse_ssh_uri(base_path)
|
376
|
+
remote_base_path = cfg.pop('remote_file_path')
|
377
|
+
client.load_system_host_keys()
|
378
|
+
client.connect(**cfg)
|
379
|
+
return get_heads_from_scp(remote_base_path, client)
|
380
|
+
|
381
|
+
base_path = Path(base_path)
|
382
|
+
if zipfile.is_zipfile(base_path):
|
383
|
+
return get_heads_from_zipfile(base_path)
|
384
|
+
try:
|
385
|
+
with open(base_path / "heads", "rb") as f:
|
386
|
+
heads = pickle.load(f)
|
387
|
+
return heads
|
388
|
+
except:
|
389
|
+
return {}
|
390
|
+
|
391
|
+
|
392
|
+
@lru_cache(maxsize=4096)
|
393
|
+
def query_index(name: str, base_path: str | Path, index: int):
|
394
|
+
if isinstance(base_path, str) and base_path.startswith('ssh://'):
|
395
|
+
with SSHClient() as client:
|
396
|
+
cfg = parse_ssh_uri(base_path)
|
397
|
+
remote_base_path = cfg.pop('remote_file_path')
|
398
|
+
client.load_system_host_keys()
|
399
|
+
client.connect(**cfg)
|
400
|
+
return query_index_from_scp(name, remote_base_path, client, index)
|
401
|
+
|
402
|
+
base_path = Path(base_path)
|
403
|
+
if zipfile.is_zipfile(base_path):
|
404
|
+
return query_index_from_zipfile(name, base_path, index)
|
405
|
+
path = Path(base_path) / "index" / name
|
406
|
+
width = int(path.with_suffix('.width').read_text())
|
407
|
+
|
408
|
+
with path.with_suffix('.idx').open("r") as f:
|
409
|
+
f.seek(index * (width + 1))
|
410
|
+
context = f.read(width)
|
411
|
+
return context.rstrip()
|
412
|
+
|
413
|
+
|
364
414
|
@lru_cache(maxsize=4096)
|
365
|
-
def load_item(id,
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
415
|
+
def load_item(id, base_path):
|
416
|
+
if isinstance(base_path, str) and base_path.startswith('ssh://'):
|
417
|
+
with SSHClient() as client:
|
418
|
+
cfg = parse_ssh_uri(base_path)
|
419
|
+
remote_base_path = cfg.pop('remote_file_path')
|
420
|
+
client.load_system_host_keys()
|
421
|
+
client.connect(**cfg)
|
422
|
+
buf = load_item_buf_from_scp(id, remote_base_path, client)
|
423
|
+
else:
|
424
|
+
base_path = Path(base_path)
|
425
|
+
if zipfile.is_zipfile(base_path):
|
426
|
+
buf = load_item_buf_from_zipfile(id, base_path)
|
427
|
+
else:
|
428
|
+
path = Path(base_path) / 'items' / id
|
429
|
+
with open(path, 'rb') as f:
|
430
|
+
buf = f.read()
|
431
|
+
item = pickle.loads(lzma.decompress(buf))
|
432
|
+
return item
|
433
|
+
|
434
|
+
|
435
|
+
def find_config_key_history(key: str, base_path: str | Path) -> Report | None:
|
436
|
+
global __current_config_cache
|
437
|
+
base_path = Path(base_path) / 'state'
|
438
|
+
if __current_config_cache is None:
|
439
|
+
if (base_path / 'parameters.pkl').exists():
|
440
|
+
with open(base_path / 'parameters.pkl', 'rb') as f:
|
441
|
+
__current_config_cache = pickle.load(f)
|
442
|
+
else:
|
443
|
+
__current_config_cache = {}
|
444
|
+
|
445
|
+
if key in __current_config_cache:
|
446
|
+
value, calibrated_time, checked_time = __current_config_cache.get(
|
447
|
+
key, None)
|
448
|
+
report = Report(
|
449
|
+
workflow=f'cfg:{key}',
|
450
|
+
bad_data=False,
|
451
|
+
in_spec=True,
|
452
|
+
fully_calibrated=True,
|
453
|
+
parameters={key: value},
|
454
|
+
data=value,
|
455
|
+
calibrated_time=calibrated_time,
|
456
|
+
checked_time=checked_time,
|
457
|
+
)
|
458
|
+
return report
|
459
|
+
return None
|
460
|
+
|
461
|
+
|
462
|
+
#########################################################################
|
463
|
+
## Zipfile support ##
|
464
|
+
#########################################################################
|
465
|
+
|
466
|
+
|
467
|
+
def load_report_from_zipfile(path: str | Path,
|
468
|
+
base_path: str | Path) -> Report | None:
|
469
|
+
path = Path(path)
|
470
|
+
with zipfile.ZipFile(base_path) as zf:
|
471
|
+
path = '/'.join(path.parts)
|
472
|
+
with zf.open(f"{base_path.stem}/reports/{path}") as f:
|
473
|
+
index = int.from_bytes(f.read(8), 'big')
|
474
|
+
report = pickle.loads(lzma.decompress(f.read()))
|
475
|
+
report.base_path = base_path
|
476
|
+
report.index = index
|
477
|
+
return report
|
478
|
+
|
479
|
+
|
480
|
+
def get_heads_from_zipfile(base_path: str | Path) -> Path | None:
|
481
|
+
with zipfile.ZipFile(base_path) as zf:
|
482
|
+
with zf.open(f"{base_path.stem}/heads") as f:
|
483
|
+
heads = pickle.load(f)
|
484
|
+
return heads
|
485
|
+
|
486
|
+
|
487
|
+
def query_index_from_zipfile(name: str, base_path: str | Path, index: int):
|
488
|
+
with zipfile.ZipFile(base_path) as zf:
|
489
|
+
with zf.open(f"{base_path.stem}/index/{name}.width") as f:
|
490
|
+
width = int(f.read().decode())
|
491
|
+
with zf.open(f"{base_path.stem}/index/{name}.idx") as f:
|
492
|
+
f.seek(index * (width + 1))
|
493
|
+
context = f.read(width).decode()
|
494
|
+
return context.rstrip()
|
495
|
+
|
496
|
+
|
497
|
+
def load_item_buf_from_zipfile(id, base_path):
|
498
|
+
with zipfile.ZipFile(base_path) as zf:
|
499
|
+
with zf.open(f"{base_path.stem}/items/{id}") as f:
|
500
|
+
return f.read()
|
501
|
+
|
502
|
+
|
503
|
+
#########################################################################
|
504
|
+
## SCP support ##
|
505
|
+
#########################################################################
|
506
|
+
|
507
|
+
|
508
|
+
def parse_ssh_uri(uri):
|
509
|
+
"""
|
510
|
+
解析 SSH URI 字符串,返回包含连接参数和路径的字典。
|
511
|
+
|
512
|
+
格式:ssh://[{username}[:{password}]@]{host}[:{port}][?key_filename={key_path}][/{remote_file_path}]
|
513
|
+
|
514
|
+
返回示例:
|
515
|
+
{
|
516
|
+
"username": "user",
|
517
|
+
"password": "pass",
|
518
|
+
"host": "example.com",
|
519
|
+
"port": 22,
|
520
|
+
"key_filename": "/path/to/key",
|
521
|
+
"remote_file_path": "/data/file.txt"
|
522
|
+
}
|
523
|
+
"""
|
524
|
+
pattern = re.compile(
|
525
|
+
r"^ssh://" # 协议头
|
526
|
+
r"(?:([^:@/]+))(?::([^@/]+))?@?" # 用户名和密码(可选)
|
527
|
+
r"([^:/?#]+)" # 主机名(必须)
|
528
|
+
r"(?::(\d+))?" # 端口(可选)
|
529
|
+
r"(/?[^?#]*)?" # 远程路径(可选)
|
530
|
+
r"(?:\?([^#]+))?" # 查询参数(如 key_filename)
|
531
|
+
r"$",
|
532
|
+
re.IGNORECASE)
|
533
|
+
|
534
|
+
match = pattern.match(uri)
|
535
|
+
if not match:
|
536
|
+
raise ValueError(f"Invalid SSH URI format: {uri}")
|
537
|
+
|
538
|
+
# 提取分组
|
539
|
+
username, password, host, port, path, query = match.groups()
|
540
|
+
|
541
|
+
# 处理查询参数
|
542
|
+
key_filename = None
|
543
|
+
if query:
|
544
|
+
params = parse_qs(query)
|
545
|
+
key_filename = params.get("key_filename", [None])[0] # 取第一个值
|
546
|
+
|
547
|
+
# 清理路径开头的斜杠
|
548
|
+
remote_file_path = path
|
549
|
+
|
550
|
+
return {
|
551
|
+
"username": username,
|
552
|
+
"password": password,
|
553
|
+
"hostname": host,
|
554
|
+
"port": int(port) if port else 22, # 默认端口 22
|
555
|
+
"key_filename": key_filename,
|
556
|
+
"remote_file_path": remote_file_path
|
557
|
+
}
|
558
|
+
|
559
|
+
|
560
|
+
def load_report_from_scp(path: str | Path, base_path: Path,
|
561
|
+
client: SSHClient) -> Report:
|
562
|
+
try:
|
563
|
+
path = Path(path)
|
564
|
+
with client.open_sftp() as sftp:
|
565
|
+
with sftp.open(str(Path(base_path) / 'reports' / path), 'rb') as f:
|
566
|
+
index = int.from_bytes(f.read(8), 'big')
|
567
|
+
report = pickle.loads(lzma.decompress(f.read()))
|
568
|
+
report.index = index
|
569
|
+
return report
|
570
|
+
except SSHException:
|
571
|
+
raise ValueError(f"Can't load report from {path}")
|
572
|
+
|
573
|
+
|
574
|
+
def get_heads_from_scp(base_path: Path, client: SSHClient) -> Path | None:
|
575
|
+
try:
|
576
|
+
with client.open_sftp() as sftp:
|
577
|
+
with sftp.open(str(Path(base_path) / 'heads'), 'rb') as f:
|
578
|
+
heads = pickle.load(f)
|
579
|
+
return heads
|
580
|
+
except SSHException:
|
581
|
+
return None
|
582
|
+
|
583
|
+
|
584
|
+
def query_index_from_scp(name: str, base_path: Path, client: SSHClient,
|
585
|
+
index: int):
|
586
|
+
try:
|
587
|
+
with client.open_sftp() as sftp:
|
588
|
+
s = str(Path(base_path) / 'index' / f'{name}.width')
|
589
|
+
with sftp.open(s, 'rb') as f:
|
590
|
+
width = int(f.read().decode())
|
591
|
+
with sftp.open(str(Path(base_path) / 'index' / f'{name}.idx'),
|
592
|
+
'rb') as f:
|
593
|
+
f.seek(index * (width + 1))
|
594
|
+
context = f.read(width).decode()
|
595
|
+
return context.rstrip()
|
596
|
+
except SSHException:
|
597
|
+
return None
|
598
|
+
|
599
|
+
|
600
|
+
def load_item_buf_from_scp(id: str, base_path: Path, client: SSHClient):
|
601
|
+
try:
|
602
|
+
with client.open_sftp() as sftp:
|
603
|
+
with sftp.open(str(Path(base_path) / 'items' / str(id)),
|
604
|
+
'rb') as f:
|
605
|
+
return f.read()
|
606
|
+
except SSHException:
|
607
|
+
return None
|
qulab/executor/template.py
CHANGED
@@ -8,6 +8,12 @@ import string
|
|
8
8
|
import textwrap
|
9
9
|
from typing import Any
|
10
10
|
|
11
|
+
_notset = object()
|
12
|
+
|
13
|
+
|
14
|
+
def VAR(name: str, /, *, default: Any = _notset) -> Any:
|
15
|
+
return name
|
16
|
+
|
11
17
|
|
12
18
|
def encode_mapping(mapping):
|
13
19
|
mapping_bytes = lzma.compress(pickle.dumps(mapping))
|
@@ -68,26 +74,67 @@ class TemplateVarExtractor(ast.NodeVisitor):
|
|
68
74
|
def visit_Call(self, node):
|
69
75
|
if isinstance(node.func, ast.Name) and node.func.id == 'VAR':
|
70
76
|
arg = node.args[0]
|
71
|
-
if isinstance(arg, ast.Constant)
|
72
|
-
|
73
|
-
|
74
|
-
|
77
|
+
if not isinstance(arg, ast.Constant) or not isinstance(
|
78
|
+
arg.value, str):
|
79
|
+
raise SyntaxError(
|
80
|
+
f"Argument of VAR function must be a string. {self.fname}:{node.lineno}"
|
81
|
+
)
|
82
|
+
if len(node.args) != 1:
|
83
|
+
raise SyntaxError(
|
84
|
+
f"VAR function only accept one argument. {self.fname}:{node.lineno}"
|
85
|
+
)
|
86
|
+
default = _notset
|
87
|
+
for k in node.keywords:
|
88
|
+
if k.arg == 'default':
|
89
|
+
pass
|
90
|
+
# if isinstance(k.value, ast.Constant):
|
91
|
+
# # default = k.value.value
|
92
|
+
# default = k.value
|
93
|
+
# else:
|
94
|
+
# default = k.value
|
95
|
+
# # raise SyntaxError(
|
96
|
+
# # f"Argument of 'default' must be a constant. {self.fname}:{node.lineno}"
|
97
|
+
# # )
|
98
|
+
else:
|
99
|
+
raise SyntaxError(
|
100
|
+
f"VAR function only accept keyword argument 'default'. {self.fname}:{node.lineno}"
|
75
101
|
)
|
76
|
-
|
102
|
+
|
103
|
+
if default is _notset:
|
77
104
|
# new_node = ast.Subscript(value=ast.Name(id="__VAR",
|
78
105
|
# ctx=ast.Load()),
|
79
106
|
# slice=ast.Constant(value=arg.value),
|
80
107
|
# ctx=ast.Load())
|
81
|
-
|
82
|
-
|
108
|
+
if arg.value not in self.mapping:
|
109
|
+
raise TemplateKeyError(
|
110
|
+
f"The variable '{arg.value}' is not provided in mapping. {self.fname}:{node.lineno}"
|
111
|
+
)
|
83
112
|
self.replacements[(node.lineno, node.end_lineno,
|
84
113
|
node.col_offset,
|
85
114
|
node.end_col_offset)] = ('VAR', arg.value,
|
86
115
|
None, None)
|
87
116
|
else:
|
88
|
-
|
89
|
-
|
90
|
-
)
|
117
|
+
# new_node = ast.Call(
|
118
|
+
# func=ast.Attribute(value=ast.Name(id='__VAR',
|
119
|
+
# ctx=ast.Load()),
|
120
|
+
# attr='get',
|
121
|
+
# ctx=ast.Load()),
|
122
|
+
# args=[ast.Constant(value=arg.value)],
|
123
|
+
# keywords=[
|
124
|
+
# ast.keyword(arg='default',
|
125
|
+
# value=value)
|
126
|
+
# ],
|
127
|
+
# ctx=ast.Load())
|
128
|
+
self.replacements[(node.lineno, node.end_lineno,
|
129
|
+
node.col_offset,
|
130
|
+
node.end_col_offset)] = ('VAR', arg.value,
|
131
|
+
None, None)
|
132
|
+
# ast.fix_missing_locations(new_node)
|
133
|
+
# new_source = ast.unparse(new_node)
|
134
|
+
# print(new_source)
|
135
|
+
|
136
|
+
self.variables.add(arg.value)
|
137
|
+
|
91
138
|
self.generic_visit(node)
|
92
139
|
|
93
140
|
def _process_string(self, s: str, lineno: int, col_offset: int,
|
@@ -160,8 +207,24 @@ def inject_mapping(source: str, mapping: dict[str, Any],
|
|
160
207
|
lines_offset[end_lineno - 1] += len(
|
161
208
|
lines[end_lineno - 1]) - length_of_last_line
|
162
209
|
else:
|
163
|
-
pattern = re.compile(
|
164
|
-
|
210
|
+
pattern = re.compile(
|
211
|
+
r'VAR\s*\(\s*' # VAR(
|
212
|
+
r'(["\'])(\w+)\1' # 第一个参数(引号包裹的变量名)
|
213
|
+
r'(?:\s*,\s*(.*?))?' # 可选的其他参数
|
214
|
+
r'\s*\)', # 闭合括号
|
215
|
+
re.DOTALL # 允许.匹配换行符
|
216
|
+
) # yapf: disable
|
217
|
+
|
218
|
+
def replacement(match):
|
219
|
+
quote = match.group(1)
|
220
|
+
var_name = match.group(2)
|
221
|
+
extra_args = match.group(3)
|
222
|
+
base = f'__VAR_{hash_str}'
|
223
|
+
if extra_args is not None:
|
224
|
+
return f'{base}.get({quote}{var_name}{quote}, {extra_args})'
|
225
|
+
else:
|
226
|
+
return f'{base}[{quote}{var_name}{quote}]'
|
227
|
+
|
165
228
|
new_content = re.sub(pattern, replacement, content)
|
166
229
|
|
167
230
|
if lineno == end_lineno:
|
@@ -172,7 +235,7 @@ def inject_mapping(source: str, mapping: dict[str, Any],
|
|
172
235
|
lines[lineno - 1] = head + new_content[:-1]
|
173
236
|
for i in range(lineno, end_lineno - 1):
|
174
237
|
lines[i] = ''
|
175
|
-
lines[end_lineno - 1] =
|
238
|
+
lines[end_lineno - 1] = new_content[-1] + tail
|
176
239
|
lines_offset[end_lineno - 1] += len(
|
177
240
|
lines[end_lineno - 1]) - length_of_last_line
|
178
241
|
|
qulab/fun.cpython-311-darwin.so
CHANGED
Binary file
|
qulab/utils.py
CHANGED
@@ -42,7 +42,7 @@ def _unix_detach_with_tmux_or_screen(executable_path):
|
|
42
42
|
"-d",
|
43
43
|
"-s",
|
44
44
|
session_name,
|
45
|
-
|
45
|
+
executable_path + " ; tmux wait-for -S finished", # 等待命令结束
|
46
46
|
";",
|
47
47
|
"tmux",
|
48
48
|
"wait-for",
|
@@ -55,7 +55,7 @@ def _unix_detach_with_tmux_or_screen(executable_path):
|
|
55
55
|
|
56
56
|
# 尝试 screen
|
57
57
|
elif _check_command_exists("screen"):
|
58
|
-
command = ["screen", "-dmS", session_name,
|
58
|
+
command = ["screen", "-dmS", session_name, executable_path]
|
59
59
|
subprocess.Popen(command, start_new_session=True)
|
60
60
|
click.echo(f"已启动 screen 会话: {session_name}")
|
61
61
|
click.echo(f"你可以使用 `screen -r {session_name}` 来查看输出")
|
@@ -66,18 +66,18 @@ def _unix_detach_with_tmux_or_screen(executable_path):
|
|
66
66
|
|
67
67
|
def run_detached_with_terminal(executable_path):
|
68
68
|
"""回退到带终端窗口的方案"""
|
69
|
-
safe_path = shlex.quote(executable_path)
|
70
69
|
if sys.platform == 'win32':
|
71
70
|
_windows_start(executable_path)
|
72
71
|
elif sys.platform == 'darwin':
|
73
|
-
script = f'tell app "Terminal" to do script "{
|
72
|
+
script = f'tell app "Terminal" to do script "{executable_path}"'
|
74
73
|
subprocess.Popen(["osascript", "-e", script], start_new_session=True)
|
75
74
|
else:
|
76
75
|
try:
|
77
|
-
subprocess.Popen(
|
78
|
-
|
76
|
+
subprocess.Popen(
|
77
|
+
["gnome-terminal", "--", "sh", "-c", executable_path],
|
78
|
+
start_new_session=True)
|
79
79
|
except FileNotFoundError:
|
80
|
-
subprocess.Popen(["xterm", "-e",
|
80
|
+
subprocess.Popen(["xterm", "-e", executable_path],
|
81
81
|
start_new_session=True)
|
82
82
|
|
83
83
|
|
qulab/version.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__version__ = "2.
|
1
|
+
__version__ = "2.9.0"
|
@@ -1,6 +1,6 @@
|
|
1
|
-
Metadata-Version: 2.
|
1
|
+
Metadata-Version: 2.4
|
2
2
|
Name: QuLab
|
3
|
-
Version: 2.
|
3
|
+
Version: 2.9.0
|
4
4
|
Summary: contral instruments and manage data
|
5
5
|
Author-email: feihoo87 <feihoo87@gmail.com>
|
6
6
|
Maintainer-email: feihoo87 <feihoo87@gmail.com>
|
@@ -44,6 +44,7 @@ Requires-Dist: watchdog>=4.0.0
|
|
44
44
|
Requires-Dist: waveforms>=1.9.4
|
45
45
|
Provides-Extra: full
|
46
46
|
Requires-Dist: uvloop>=0.19.0; extra == "full"
|
47
|
+
Dynamic: license-file
|
47
48
|
|
48
49
|
# QuLab
|
49
50
|
[](https://travis-ci.org/feihoo87/QuLab)
|
@@ -1,19 +1,19 @@
|
|
1
|
-
qulab/__init__.py,sha256=
|
1
|
+
qulab/__init__.py,sha256=JZ3bn_kTVlnY-P8JXQ5xEdViieFXqfxX8ReLuiiXIpo,321
|
2
2
|
qulab/__main__.py,sha256=fjaRSL_uUjNIzBGNgjlGswb9TJ2VD5qnkZHW3hItrD4,68
|
3
3
|
qulab/dicttree.py,sha256=tRRMpGZYVOLw0TEByE3_2Ss8FdOmzuGL9e1DWbs8qoY,13684
|
4
|
-
qulab/fun.cpython-311-darwin.so,sha256=
|
4
|
+
qulab/fun.cpython-311-darwin.so,sha256=ub1obycqKPeAfCYWqXF_0Nf_tkVwuNzEOmb_UEv-A6c,126848
|
5
5
|
qulab/typing.py,sha256=vg62sGqxuD9CI5677ejlzAmf2fVdAESZCQjAE_xSxPg,69
|
6
|
-
qulab/utils.py,sha256=
|
7
|
-
qulab/version.py,sha256=
|
6
|
+
qulab/utils.py,sha256=BdLdlfjpe6m6gSeONYmpAKTTqxDaYHNk4exlz8kZxTg,2982
|
7
|
+
qulab/version.py,sha256=a31Mxt9ZbwomFaywnjMS--InOBZRSD7ltY06SX9P5q0,21
|
8
8
|
qulab/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
9
9
|
qulab/cli/commands.py,sha256=tgDIkkeIoasQXAifJZ6NU8jDgpNgb2a-B0C4nF0evrE,559
|
10
10
|
qulab/cli/config.py,sha256=Ei7eSYnbwPPlluDnm8YmWONYiI4g7WtvlZGQdr1Z6vo,3688
|
11
11
|
qulab/executor/__init__.py,sha256=LosPzOMaljSZY1thy_Fxtbrgq7uubJszMABEB7oM7tU,101
|
12
|
-
qulab/executor/cli.py,sha256=
|
12
|
+
qulab/executor/cli.py,sha256=8d-8bRWZ5lmsMtjASsl1zu1rV-syeAESMNVthvIQxlo,10018
|
13
13
|
qulab/executor/load.py,sha256=YndvzagvWR8Sg6WHZ-gP-Of0FrFOyh_E_a3VXsjDf1Q,17502
|
14
14
|
qulab/executor/schedule.py,sha256=0BV5LGxhqdIlGwW6-o5_5mljAtdtL1La8EDNBFi8pzU,18585
|
15
|
-
qulab/executor/storage.py,sha256=
|
16
|
-
qulab/executor/template.py,sha256=
|
15
|
+
qulab/executor/storage.py,sha256=OA_XMDoFDfPZCU89caf9-VZ3D6qaWib8MpJno10KUfc,20770
|
16
|
+
qulab/executor/template.py,sha256=BhNP8GrTk0pCK3GwijI08Q2jmYPbLi_7A5XWZXhCtUo,10123
|
17
17
|
qulab/executor/transform.py,sha256=BDx0c4nqTHMAOLVqju0Ydd91uxNm6EpVIfssjZse0bI,2284
|
18
18
|
qulab/executor/utils.py,sha256=l_b0y2kMwYKyyXeFtoblPYwKNU-wiFQ9PMo9QlWl9wE,6213
|
19
19
|
qulab/monitor/__init__.py,sha256=nTHelnDpxRS_fl_B38TsN0njgq8eVTEz9IAnN3NbDlM,42
|
@@ -97,9 +97,9 @@ qulab/visualization/plot_seq.py,sha256=UWTS6p9nfX_7B8ehcYo6UnSTUCjkBsNU9jiOeW2ca
|
|
97
97
|
qulab/visualization/qdat.py,sha256=ZeevBYWkzbww4xZnsjHhw7wRorJCBzbG0iEu-XQB4EA,5735
|
98
98
|
qulab/visualization/rot3d.py,sha256=lMrEJlRLwYe6NMBlGkKYpp_V9CTipOAuDy6QW_cQK00,734
|
99
99
|
qulab/visualization/widgets.py,sha256=6KkiTyQ8J-ei70LbPQZAK35wjktY47w2IveOa682ftA,3180
|
100
|
-
qulab-2.
|
101
|
-
qulab-2.
|
102
|
-
qulab-2.
|
103
|
-
qulab-2.
|
104
|
-
qulab-2.
|
105
|
-
qulab-2.
|
100
|
+
qulab-2.9.0.dist-info/licenses/LICENSE,sha256=PRzIKxZtpQcH7whTG6Egvzl1A0BvnSf30tmR2X2KrpA,1065
|
101
|
+
qulab-2.9.0.dist-info/METADATA,sha256=gci3kQAblFeB_opSNNDQhNyQAShjlv1NnEWYxGcFlqI,3720
|
102
|
+
qulab-2.9.0.dist-info/WHEEL,sha256=JaRfGby1TTSqNmKjNEsuDj4aQSHVLA9Uc6dzGgAYXuw,114
|
103
|
+
qulab-2.9.0.dist-info/entry_points.txt,sha256=b0v1GXOwmxY-nCCsPN_rHZZvY9CtTbWqrGj8u1m8yHo,45
|
104
|
+
qulab-2.9.0.dist-info/top_level.txt,sha256=3T886LbAsbvjonu_TDdmgxKYUn939BVTRPxPl9r4cEg,6
|
105
|
+
qulab-2.9.0.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|