QuLab 2.1.2__tar.gz → 2.1.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {qulab-2.1.2 → qulab-2.1.3}/PKG-INFO +1 -1
- {qulab-2.1.2 → qulab-2.1.3}/QuLab.egg-info/PKG-INFO +1 -1
- {qulab-2.1.2 → qulab-2.1.3}/QuLab.egg-info/SOURCES.txt +0 -1
- {qulab-2.1.2 → qulab-2.1.3}/qulab/__main__.py +0 -2
- {qulab-2.1.2 → qulab-2.1.3}/qulab/scan/curd.py +78 -1
- {qulab-2.1.2 → qulab-2.1.3}/qulab/scan/models.py +21 -6
- {qulab-2.1.2 → qulab-2.1.3}/qulab/scan/query.py +1 -1
- {qulab-2.1.2 → qulab-2.1.3}/qulab/scan/record.py +44 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/scan/scan.py +130 -6
- qulab-2.1.2/qulab/scan/recorder.py → qulab-2.1.3/qulab/scan/server.py +74 -4
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/rpc/zmq_socket.py +7 -1
- qulab-2.1.3/qulab/version.py +1 -0
- qulab-2.1.2/qulab/scan/server.py +0 -104
- qulab-2.1.2/qulab/version.py +0 -1
- {qulab-2.1.2 → qulab-2.1.3}/LICENSE +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/MANIFEST.in +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/QuLab.egg-info/dependency_links.txt +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/QuLab.egg-info/entry_points.txt +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/QuLab.egg-info/requires.txt +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/QuLab.egg-info/top_level.txt +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/README.md +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/pyproject.toml +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/__init__.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/monitor/__init__.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/monitor/__main__.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/monitor/config.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/monitor/dataset.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/monitor/event_queue.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/monitor/mainwindow.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/monitor/monitor.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/monitor/ploter.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/monitor/qt_compat.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/monitor/toolbar.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/scan/__init__.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/scan/expression.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/scan/optimize.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/scan/space.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/scan/utils.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/storage/__init__.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/storage/__main__.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/storage/backend/__init__.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/storage/backend/redis.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/storage/base_dataset.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/storage/chunk.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/storage/dataset.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/storage/file.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/storage/models/__init__.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/storage/models/base.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/storage/models/config.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/storage/models/file.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/storage/models/ipy.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/storage/models/models.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/storage/models/record.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/storage/models/report.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/storage/models/tag.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/storage/storage.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/__init__.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/chat.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/device/__init__.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/device/basedevice.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/device/loader.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/device/utils.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/drivers/FakeInstrument.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/drivers/__init__.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/ipy_events.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/net/__init__.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/net/bencoder.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/net/cli.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/net/dhcp.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/net/dhcpd.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/net/kad.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/net/kcp.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/net/nginx.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/progress.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/rpc/__init__.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/rpc/client.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/rpc/exceptions.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/rpc/msgpack.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/rpc/msgpack.pyi +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/rpc/rpc.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/rpc/serialize.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/rpc/server.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/rpc/socket.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/rpc/utils.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/sys/rpc/worker.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/visualization/__init__.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/visualization/__main__.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/visualization/_autoplot.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/visualization/plot_layout.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/visualization/plot_seq.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/visualization/qdat.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/qulab/visualization/widgets.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/setup.cfg +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/setup.py +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/src/qulab.h +0 -0
- {qulab-2.1.2 → qulab-2.1.3}/tests/test_scan.py +0 -0
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import click
|
|
2
2
|
|
|
3
3
|
from .monitor.__main__ import main as monitor
|
|
4
|
-
from .scan.recorder import record
|
|
5
4
|
from .scan.server import server
|
|
6
5
|
from .sys.net.cli import dht
|
|
7
6
|
from .visualization.__main__ import plot
|
|
@@ -21,7 +20,6 @@ def hello():
|
|
|
21
20
|
main.add_command(monitor)
|
|
22
21
|
main.add_command(plot)
|
|
23
22
|
main.add_command(dht)
|
|
24
|
-
main.add_command(record)
|
|
25
23
|
main.add_command(server)
|
|
26
24
|
|
|
27
25
|
if __name__ == '__main__':
|
|
@@ -1,4 +1,7 @@
|
|
|
1
|
+
import lzma
|
|
2
|
+
import pickle
|
|
1
3
|
from datetime import date, datetime, timezone
|
|
4
|
+
from pathlib import Path
|
|
2
5
|
from typing import Sequence, Type, Union
|
|
3
6
|
|
|
4
7
|
from sqlalchemy.orm import Query, Session, aliased
|
|
@@ -6,7 +9,8 @@ from sqlalchemy.orm.exc import NoResultFound
|
|
|
6
9
|
from sqlalchemy.orm.session import Session
|
|
7
10
|
from waveforms.dicttree import foldDict
|
|
8
11
|
|
|
9
|
-
from .models import Comment,
|
|
12
|
+
from .models import (Cell, Comment, Config, InputText, Notebook, Record,
|
|
13
|
+
Report, Sample, Tag, utcnow)
|
|
10
14
|
|
|
11
15
|
|
|
12
16
|
def tag(session: Session, tag_text: str) -> Tag:
|
|
@@ -142,3 +146,76 @@ def remove_tags(session: Session, record_id: int, tags: Sequence[str]):
|
|
|
142
146
|
session.rollback()
|
|
143
147
|
return False
|
|
144
148
|
return True
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def create_notebook(session: Session, notebook_name: str) -> Notebook:
|
|
152
|
+
"""Create a notebook in the database."""
|
|
153
|
+
notebook = Notebook(name=notebook_name)
|
|
154
|
+
session.add(notebook)
|
|
155
|
+
return notebook
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def create_input_text(session: Session, input_text: str) -> InputText:
|
|
159
|
+
"""Create an input text in the database."""
|
|
160
|
+
input = InputText()
|
|
161
|
+
input.text = input_text
|
|
162
|
+
try:
|
|
163
|
+
input = session.query(InputText).filter(
|
|
164
|
+
InputText.hash == input.hash,
|
|
165
|
+
InputText.text_field == input_text).one()
|
|
166
|
+
except NoResultFound:
|
|
167
|
+
session.add(input)
|
|
168
|
+
return input
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def create_cell(session: Session, notebook: Notebook, input_text: str) -> Cell:
|
|
172
|
+
"""Create a cell in the database."""
|
|
173
|
+
cell = Cell()
|
|
174
|
+
cell.notebook = notebook
|
|
175
|
+
cell.input = create_input_text(session, input_text)
|
|
176
|
+
cell.index = len(notebook.cells) - 1
|
|
177
|
+
session.add(cell)
|
|
178
|
+
notebook.atime = cell.ctime
|
|
179
|
+
return cell
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def create_config(session: Session, config: dict | bytes, base: Path,
|
|
183
|
+
filename: str) -> Config:
|
|
184
|
+
"""Create a config in the database."""
|
|
185
|
+
|
|
186
|
+
if not isinstance(config, bytes):
|
|
187
|
+
buf = pickle.dumps(config)
|
|
188
|
+
buf = lzma.compress(buf)
|
|
189
|
+
content_type = 'application/pickle+lzma'
|
|
190
|
+
else:
|
|
191
|
+
buf = config
|
|
192
|
+
content_type = 'application/octet-stream'
|
|
193
|
+
config = Config(buf)
|
|
194
|
+
config.content_type = content_type
|
|
195
|
+
for cfg in session.query(Config).filter(Config.hash == config.hash).all():
|
|
196
|
+
with open(base / cfg.file, 'rb') as f:
|
|
197
|
+
if f.read() == buf:
|
|
198
|
+
cfg.atime = utcnow()
|
|
199
|
+
return cfg
|
|
200
|
+
else:
|
|
201
|
+
path = base / filename
|
|
202
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
203
|
+
with open(path, 'wb') as f:
|
|
204
|
+
f.write(buf)
|
|
205
|
+
config.file = filename
|
|
206
|
+
session.add(config)
|
|
207
|
+
return config
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def get_config(session: Session, config_id: int, base: Path):
|
|
211
|
+
config = session.get(Config, config_id)
|
|
212
|
+
if config is None:
|
|
213
|
+
return None
|
|
214
|
+
config.atime = utcnow()
|
|
215
|
+
path = base / config.file
|
|
216
|
+
with open(path, 'rb') as f:
|
|
217
|
+
buf = f.read()
|
|
218
|
+
if config.content_type == 'application/pickle+lzma':
|
|
219
|
+
buf = lzma.decompress(buf)
|
|
220
|
+
buf = pickle.loads(buf)
|
|
221
|
+
return buf
|
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
import hashlib
|
|
2
2
|
import pickle
|
|
3
|
-
import time
|
|
4
3
|
from datetime import datetime, timezone
|
|
5
4
|
from functools import singledispatchmethod
|
|
6
|
-
from typing import Optional
|
|
7
5
|
|
|
8
|
-
from sqlalchemy import (
|
|
6
|
+
from sqlalchemy import (Column, DateTime, Float, ForeignKey, Integer,
|
|
9
7
|
LargeBinary, String, Table, Text, create_engine)
|
|
10
8
|
from sqlalchemy.orm import (backref, declarative_base, relationship,
|
|
11
9
|
sessionmaker)
|
|
@@ -325,7 +323,7 @@ class InputText(Base):
|
|
|
325
323
|
__tablename__ = 'inputs'
|
|
326
324
|
|
|
327
325
|
id = Column(Integer, primary_key=True)
|
|
328
|
-
hash = Column(LargeBinary(20))
|
|
326
|
+
hash = Column(LargeBinary(20), index=True)
|
|
329
327
|
text_field = Column(Text, unique=True)
|
|
330
328
|
|
|
331
329
|
@property
|
|
@@ -432,6 +430,22 @@ class SampleTransfer(Base):
|
|
|
432
430
|
comments = relationship("Comment", secondary=sample_transfer_comments)
|
|
433
431
|
|
|
434
432
|
|
|
433
|
+
class Config(Base):
|
|
434
|
+
__tablename__ = 'configs'
|
|
435
|
+
|
|
436
|
+
id = Column(Integer, primary_key=True)
|
|
437
|
+
hash = Column(LargeBinary(20), index=True)
|
|
438
|
+
file = Column(String)
|
|
439
|
+
content_type = Column(String, default='application/pickle')
|
|
440
|
+
ctime = Column(DateTime, default=utcnow)
|
|
441
|
+
atime = Column(DateTime, default=utcnow)
|
|
442
|
+
|
|
443
|
+
records = relationship("Record", back_populates="config")
|
|
444
|
+
|
|
445
|
+
def __init__(self, data: bytes) -> None:
|
|
446
|
+
self.hash = hashlib.sha1(data).digest()
|
|
447
|
+
|
|
448
|
+
|
|
435
449
|
class Record(Base):
|
|
436
450
|
__tablename__ = 'records'
|
|
437
451
|
|
|
@@ -440,14 +454,14 @@ class Record(Base):
|
|
|
440
454
|
mtime = Column(DateTime, default=utcnow)
|
|
441
455
|
atime = Column(DateTime, default=utcnow)
|
|
442
456
|
user_id = Column(Integer, ForeignKey('users.id'))
|
|
457
|
+
config_id = Column(Integer, ForeignKey('configs.id'))
|
|
443
458
|
parent_id = Column(Integer, ForeignKey('records.id'))
|
|
444
459
|
cell_id = Column(Integer, ForeignKey('cells.id'))
|
|
445
460
|
|
|
446
461
|
app = Column(String)
|
|
447
462
|
file = Column(String)
|
|
463
|
+
content_type = Column(String, default='application/pickle')
|
|
448
464
|
key = Column(String)
|
|
449
|
-
config = Column(JSON)
|
|
450
|
-
task_hash = Column(LargeBinary(32))
|
|
451
465
|
|
|
452
466
|
parent = relationship("Record",
|
|
453
467
|
remote_side=[id],
|
|
@@ -456,6 +470,7 @@ class Record(Base):
|
|
|
456
470
|
remote_side=[parent_id],
|
|
457
471
|
back_populates="parent")
|
|
458
472
|
|
|
473
|
+
config = relationship("Config", back_populates="records")
|
|
459
474
|
user = relationship("User")
|
|
460
475
|
samples = relationship("Sample",
|
|
461
476
|
secondary=sample_records,
|
|
@@ -95,6 +95,11 @@ class BufferList():
|
|
|
95
95
|
dill.dump(item, f)
|
|
96
96
|
self._list.clear()
|
|
97
97
|
|
|
98
|
+
def delete(self):
|
|
99
|
+
if isinstance(self.file, Path):
|
|
100
|
+
self.file.unlink()
|
|
101
|
+
self.file = None
|
|
102
|
+
|
|
98
103
|
def append(self, pos, value, dims=None):
|
|
99
104
|
if dims is not None:
|
|
100
105
|
if any([p != 0 for i, p in enumerate(pos) if i not in dims]):
|
|
@@ -450,6 +455,20 @@ class Record():
|
|
|
450
455
|
with open(self._file, 'wb') as f:
|
|
451
456
|
dill.dump(self, f)
|
|
452
457
|
|
|
458
|
+
def delete(self):
|
|
459
|
+
if self.is_remote_record():
|
|
460
|
+
with ZMQContextManager(zmq.DEALER,
|
|
461
|
+
connect=self.database) as socket:
|
|
462
|
+
socket.send_pyobj({
|
|
463
|
+
'method': 'record_delete',
|
|
464
|
+
'record_id': self.id
|
|
465
|
+
})
|
|
466
|
+
elif self.is_local_record():
|
|
467
|
+
for key, value in self._items.items():
|
|
468
|
+
if isinstance(value, BufferList):
|
|
469
|
+
value.delete()
|
|
470
|
+
self._file.unlink()
|
|
471
|
+
|
|
453
472
|
def export(self, file):
|
|
454
473
|
with zipfile.ZipFile(file,
|
|
455
474
|
'w',
|
|
@@ -470,8 +489,33 @@ class Record():
|
|
|
470
489
|
else:
|
|
471
490
|
items[key] = value
|
|
472
491
|
with z.open('record.pkl', 'w') as f:
|
|
492
|
+
self.description['entry']['scripts'] = self.scripts()
|
|
473
493
|
dill.dump((self.description, items), f)
|
|
474
494
|
|
|
495
|
+
def scripts(self, session=None):
|
|
496
|
+
scripts = self.description['entry']['scripts']
|
|
497
|
+
if isinstance(scripts, list):
|
|
498
|
+
return scripts
|
|
499
|
+
else:
|
|
500
|
+
cell_id = scripts
|
|
501
|
+
|
|
502
|
+
if self.is_remote_record():
|
|
503
|
+
with ZMQContextManager(zmq.DEALER,
|
|
504
|
+
connect=self.database) as socket:
|
|
505
|
+
socket.send_pyobj({
|
|
506
|
+
'method': 'notebook_history',
|
|
507
|
+
'cell_id': cell_id
|
|
508
|
+
})
|
|
509
|
+
return socket.recv_pyobj()
|
|
510
|
+
elif self.is_local_record():
|
|
511
|
+
from .models import Cell
|
|
512
|
+
assert session is not None, "session is required for local record"
|
|
513
|
+
cell = session.get(Cell, cell_id)
|
|
514
|
+
return [
|
|
515
|
+
cell.input.text
|
|
516
|
+
for cell in cell.notebook.cells[1:cell.index + 2]
|
|
517
|
+
]
|
|
518
|
+
|
|
475
519
|
@classmethod
|
|
476
520
|
def load(cls, file: str):
|
|
477
521
|
with zipfile.ZipFile(file, 'r') as z:
|
|
@@ -2,8 +2,12 @@ import asyncio
|
|
|
2
2
|
import copy
|
|
3
3
|
import inspect
|
|
4
4
|
import itertools
|
|
5
|
+
import lzma
|
|
5
6
|
import os
|
|
7
|
+
import pickle
|
|
8
|
+
import platform
|
|
6
9
|
import re
|
|
10
|
+
import subprocess
|
|
7
11
|
import sys
|
|
8
12
|
import uuid
|
|
9
13
|
from concurrent.futures import ProcessPoolExecutor
|
|
@@ -19,7 +23,7 @@ from ..sys.rpc.zmq_socket import ZMQContextManager
|
|
|
19
23
|
from .expression import Env, Expression, Symbol
|
|
20
24
|
from .optimize import NgOptimizer
|
|
21
25
|
from .record import Record
|
|
22
|
-
from .
|
|
26
|
+
from .server import default_record_port
|
|
23
27
|
from .space import Optimizer, OptimizeSpace, Space
|
|
24
28
|
from .utils import async_zip, call_function, dump_globals
|
|
25
29
|
|
|
@@ -41,6 +45,7 @@ except:
|
|
|
41
45
|
|
|
42
46
|
__process_uuid = uuid.uuid1()
|
|
43
47
|
__task_counter = itertools.count()
|
|
48
|
+
__notebook_id = None
|
|
44
49
|
|
|
45
50
|
if os.getenv('QULAB_SERVER'):
|
|
46
51
|
default_server = os.getenv('QULAB_SERVER')
|
|
@@ -52,6 +57,105 @@ else:
|
|
|
52
57
|
default_executor = default_server
|
|
53
58
|
|
|
54
59
|
|
|
60
|
+
def yapf_reformat(cell_text):
|
|
61
|
+
try:
|
|
62
|
+
import isort
|
|
63
|
+
import yapf.yapflib.yapf_api
|
|
64
|
+
|
|
65
|
+
fname = f"f{uuid.uuid1().hex}"
|
|
66
|
+
|
|
67
|
+
def wrap(source):
|
|
68
|
+
lines = [f"async def {fname}():"]
|
|
69
|
+
for line in source.split('\n'):
|
|
70
|
+
lines.append(" " + line)
|
|
71
|
+
return '\n'.join(lines)
|
|
72
|
+
|
|
73
|
+
def unwrap(source):
|
|
74
|
+
lines = []
|
|
75
|
+
for line in source.split('\n'):
|
|
76
|
+
if line.startswith(f"async def {fname}():"):
|
|
77
|
+
continue
|
|
78
|
+
lines.append(line[4:])
|
|
79
|
+
return '\n'.join(lines)
|
|
80
|
+
|
|
81
|
+
cell_text = re.sub('^%', '#%#', cell_text, flags=re.M)
|
|
82
|
+
reformated_text = unwrap(
|
|
83
|
+
yapf.yapflib.yapf_api.FormatCode(wrap(isort.code(cell_text)))[0])
|
|
84
|
+
return re.sub('^#%#', '%', reformated_text, flags=re.M)
|
|
85
|
+
except:
|
|
86
|
+
return cell_text
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def get_installed_packages():
|
|
90
|
+
result = subprocess.run([sys.executable, '-m', 'pip', 'freeze'],
|
|
91
|
+
stdout=subprocess.PIPE,
|
|
92
|
+
text=True)
|
|
93
|
+
|
|
94
|
+
lines = result.stdout.split('\n')
|
|
95
|
+
packages = []
|
|
96
|
+
for line in lines:
|
|
97
|
+
if line:
|
|
98
|
+
packages.append(line)
|
|
99
|
+
return packages
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def get_system_info():
|
|
103
|
+
info = {
|
|
104
|
+
'OS': platform.uname()._asdict(),
|
|
105
|
+
'Python': sys.version,
|
|
106
|
+
'PythonExecutable': sys.executable,
|
|
107
|
+
'PythonPath': sys.path,
|
|
108
|
+
'packages': get_installed_packages()
|
|
109
|
+
}
|
|
110
|
+
return info
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def current_notebook():
|
|
114
|
+
return __notebook_id
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
async def create_notebook(name: str, database=default_server, socket=None):
|
|
118
|
+
global __notebook_id
|
|
119
|
+
|
|
120
|
+
async with ZMQContextManager(zmq.DEALER, connect=database,
|
|
121
|
+
socket=socket) as socket:
|
|
122
|
+
await socket.send_pyobj({'method': 'notebook_create', 'name': name})
|
|
123
|
+
__notebook_id = await socket.recv_pyobj()
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
async def save_input_cells(notebook_id,
|
|
127
|
+
input_cells,
|
|
128
|
+
database=default_server,
|
|
129
|
+
socket=None):
|
|
130
|
+
async with ZMQContextManager(zmq.DEALER, connect=database,
|
|
131
|
+
socket=socket) as socket:
|
|
132
|
+
await socket.send_pyobj({
|
|
133
|
+
'method': 'notebook_extend',
|
|
134
|
+
'notebook_id': notebook_id,
|
|
135
|
+
'input_cells': input_cells
|
|
136
|
+
})
|
|
137
|
+
return await socket.recv_pyobj()
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
async def create_config(config: dict, database=default_server, socket=None):
|
|
141
|
+
async with ZMQContextManager(zmq.DEALER, connect=database,
|
|
142
|
+
socket=socket) as socket:
|
|
143
|
+
buf = lzma.compress(pickle.dumps(config))
|
|
144
|
+
await socket.send_pyobj({'method': 'config_update', 'update': buf})
|
|
145
|
+
return await socket.recv_pyobj()
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
async def get_config(config_id: int, database=default_server, socket=None):
|
|
149
|
+
async with ZMQContextManager(zmq.DEALER, connect=database,
|
|
150
|
+
socket=socket) as socket:
|
|
151
|
+
await socket.send_pyobj({
|
|
152
|
+
'method': 'config_get',
|
|
153
|
+
'config_id': config_id
|
|
154
|
+
})
|
|
155
|
+
buf = await socket.recv_pyobj()
|
|
156
|
+
return pickle.loads(lzma.decompress(buf))
|
|
157
|
+
|
|
158
|
+
|
|
55
159
|
def task_uuid():
|
|
56
160
|
return uuid.uuid3(__process_uuid, str(next(__task_counter)))
|
|
57
161
|
|
|
@@ -135,10 +239,11 @@ class Scan():
|
|
|
135
239
|
mixin=None):
|
|
136
240
|
self.id = task_uuid()
|
|
137
241
|
self.record = None
|
|
138
|
-
self.
|
|
242
|
+
self.config = None
|
|
139
243
|
self.description = {
|
|
140
244
|
'app': app,
|
|
141
245
|
'tags': tags,
|
|
246
|
+
'config': None,
|
|
142
247
|
'loops': {},
|
|
143
248
|
'intrinsic_loops': {},
|
|
144
249
|
'consts': {},
|
|
@@ -157,9 +262,11 @@ class Scan():
|
|
|
157
262
|
'database': database,
|
|
158
263
|
'hiden': ['self', r'^__.*', r'.*__$'],
|
|
159
264
|
'entry': {
|
|
265
|
+
'system': get_system_info(),
|
|
160
266
|
'env': {},
|
|
161
267
|
'shell': '',
|
|
162
|
-
'cmds': []
|
|
268
|
+
'cmds': [],
|
|
269
|
+
'scripts': []
|
|
163
270
|
},
|
|
164
271
|
}
|
|
165
272
|
self._current_level = 0
|
|
@@ -287,8 +394,6 @@ class Scan():
|
|
|
287
394
|
def get(self, name: str):
|
|
288
395
|
if name in self.description['consts']:
|
|
289
396
|
return self.description['consts'][name]
|
|
290
|
-
elif name in self.namespace:
|
|
291
|
-
return self.namespace.get(name)
|
|
292
397
|
else:
|
|
293
398
|
return Symbol(name)
|
|
294
399
|
|
|
@@ -442,6 +547,17 @@ class Scan():
|
|
|
442
547
|
|
|
443
548
|
async def run(self):
|
|
444
549
|
assymbly(self.description)
|
|
550
|
+
if self.config:
|
|
551
|
+
self.description['config'] = await create_config(
|
|
552
|
+
self.config, self.description['database'], self._sock)
|
|
553
|
+
if current_notebook() is None:
|
|
554
|
+
await create_notebook('untitle', self.description['database'],
|
|
555
|
+
self._sock)
|
|
556
|
+
cell_id = await save_input_cells(current_notebook(),
|
|
557
|
+
self.description['entry']['scripts'],
|
|
558
|
+
self.description['database'],
|
|
559
|
+
self._sock)
|
|
560
|
+
self.description['entry']['scripts'] = cell_id
|
|
445
561
|
if isinstance(
|
|
446
562
|
self.description['database'],
|
|
447
563
|
str) and self.description['database'].startswith("tcp://"):
|
|
@@ -635,13 +751,21 @@ def assymbly(description):
|
|
|
635
751
|
ipy = get_ipython()
|
|
636
752
|
if ipy is not None:
|
|
637
753
|
description['entry']['shell'] = 'ipython'
|
|
638
|
-
description['entry']['
|
|
754
|
+
description['entry']['scripts'] = [
|
|
755
|
+
yapf_reformat(cell_text) for cell_text in ipy.user_ns['In']
|
|
756
|
+
]
|
|
639
757
|
else:
|
|
640
758
|
try:
|
|
641
759
|
description['entry']['shell'] = 'shell'
|
|
642
760
|
description['entry']['cmds'] = [
|
|
643
761
|
sys.executable, __main__.__file__, *sys.argv[1:]
|
|
644
762
|
]
|
|
763
|
+
description['entry']['scripts'] = []
|
|
764
|
+
try:
|
|
765
|
+
with open(__main__.__file__) as f:
|
|
766
|
+
description['entry']['scripts'].append(f.read())
|
|
767
|
+
except:
|
|
768
|
+
pass
|
|
645
769
|
except:
|
|
646
770
|
pass
|
|
647
771
|
|
|
@@ -11,10 +11,12 @@ from loguru import logger
|
|
|
11
11
|
|
|
12
12
|
from qulab.sys.rpc.zmq_socket import ZMQContextManager
|
|
13
13
|
|
|
14
|
-
from .curd import
|
|
14
|
+
from .curd import (create_cell, create_config, create_notebook, get_config,
|
|
15
|
+
query_record, remove_tags, tag, update_tags)
|
|
16
|
+
from .models import Cell, Notebook
|
|
15
17
|
from .models import Record as RecordInDB
|
|
16
18
|
from .models import Session, create_engine, create_tables, sessionmaker, utcnow
|
|
17
|
-
from .record import Record
|
|
19
|
+
from .record import BufferList, Record, random_path
|
|
18
20
|
|
|
19
21
|
try:
|
|
20
22
|
default_record_port = int(os.getenv('QULAB_RECORD_PORT', 6789))
|
|
@@ -30,6 +32,8 @@ datapath.mkdir(parents=True, exist_ok=True)
|
|
|
30
32
|
record_cache = {}
|
|
31
33
|
CACHE_SIZE = 1024
|
|
32
34
|
|
|
35
|
+
pool = {}
|
|
36
|
+
|
|
33
37
|
|
|
34
38
|
class Request():
|
|
35
39
|
__slots__ = ['sock', 'identity', 'msg', 'method']
|
|
@@ -62,6 +66,10 @@ def flush_cache():
|
|
|
62
66
|
def get_local_record(session: Session, id: int, datapath: Path) -> Record:
|
|
63
67
|
record_in_db = session.get(RecordInDB, id)
|
|
64
68
|
record_in_db.atime = utcnow()
|
|
69
|
+
|
|
70
|
+
if record_in_db.file.endswith('.zip'):
|
|
71
|
+
return Record.load(datapath / 'objects' / record_in_db.file)
|
|
72
|
+
|
|
65
73
|
path = datapath / 'objects' / record_in_db.file
|
|
66
74
|
with open(path, 'rb') as f:
|
|
67
75
|
record = dill.load(f)
|
|
@@ -88,6 +96,7 @@ def record_create(session: Session, description: dict, datapath: Path) -> int:
|
|
|
88
96
|
if 'tags' in description:
|
|
89
97
|
record_in_db.tags = [tag(session, t) for t in description['tags']]
|
|
90
98
|
record_in_db.file = '/'.join(record._file.parts[-4:])
|
|
99
|
+
record_in_db.config_id = description['config']
|
|
91
100
|
record._file = datapath / 'objects' / record_in_db.file
|
|
92
101
|
session.add(record_in_db)
|
|
93
102
|
try:
|
|
@@ -115,6 +124,14 @@ def record_append(session: Session, record_id: int, level: int, step: int,
|
|
|
115
124
|
raise
|
|
116
125
|
|
|
117
126
|
|
|
127
|
+
def record_delete(session: Session, record_id: int, datapath: Path):
|
|
128
|
+
record = get_local_record(session, record_id, datapath)
|
|
129
|
+
record.delete()
|
|
130
|
+
record_in_db = session.get(RecordInDB, record_id)
|
|
131
|
+
session.delete(record_in_db)
|
|
132
|
+
session.commit()
|
|
133
|
+
|
|
134
|
+
|
|
118
135
|
@logger.catch
|
|
119
136
|
async def handle(session: Session, request: Request, datapath: Path):
|
|
120
137
|
|
|
@@ -162,6 +179,59 @@ async def handle(session: Session, request: Request, datapath: Path):
|
|
|
162
179
|
update_tags(session, msg['record_id'], msg['tags'], True)
|
|
163
180
|
case 'record_replace_tags':
|
|
164
181
|
update_tags(session, msg['record_id'], msg['tags'], False)
|
|
182
|
+
case 'notebook_create':
|
|
183
|
+
notebook = create_notebook(session, msg['name'])
|
|
184
|
+
session.commit()
|
|
185
|
+
await reply(request, notebook.id)
|
|
186
|
+
case 'notebook_extend':
|
|
187
|
+
notebook = session.get(Notebook, msg['notebook_id'])
|
|
188
|
+
inputCells = msg.get('input_cells', [""])
|
|
189
|
+
aready_saved = len(notebook.cells)
|
|
190
|
+
if len(inputCells) > aready_saved:
|
|
191
|
+
for cell in inputCells[aready_saved:]:
|
|
192
|
+
cell = create_cell(session, notebook, cell)
|
|
193
|
+
session.commit()
|
|
194
|
+
await reply(request, cell.id)
|
|
195
|
+
else:
|
|
196
|
+
await reply(request, None)
|
|
197
|
+
case 'notebook_history':
|
|
198
|
+
cell = session.get(Cell, msg['cell_id'])
|
|
199
|
+
if cell:
|
|
200
|
+
await reply(request, [
|
|
201
|
+
cell.input.text
|
|
202
|
+
for cell in cell.notebook.cells[1:cell.index + 2]
|
|
203
|
+
])
|
|
204
|
+
else:
|
|
205
|
+
await reply(request, None)
|
|
206
|
+
case 'config_get':
|
|
207
|
+
config = get_config(session, msg['config_id'], base=datapath)
|
|
208
|
+
session.commit()
|
|
209
|
+
await reply(request, config)
|
|
210
|
+
case 'config_update':
|
|
211
|
+
config = create_config(session,
|
|
212
|
+
msg['update'],
|
|
213
|
+
base=datapath,
|
|
214
|
+
filename='/'.join(
|
|
215
|
+
random_path(datapath).parts[-4:]))
|
|
216
|
+
session.commit()
|
|
217
|
+
await reply(request, config.id)
|
|
218
|
+
case 'submit':
|
|
219
|
+
from .scan import Scan
|
|
220
|
+
description = dill.loads(msg['description'])
|
|
221
|
+
task = Scan()
|
|
222
|
+
task.description = description
|
|
223
|
+
task.start()
|
|
224
|
+
pool[task.id] = task
|
|
225
|
+
await reply(request, task.id)
|
|
226
|
+
case 'get_record_id':
|
|
227
|
+
task = pool.get(msg['id'])
|
|
228
|
+
for _ in range(10):
|
|
229
|
+
if task.record:
|
|
230
|
+
await reply(request, task.record.id)
|
|
231
|
+
break
|
|
232
|
+
await asyncio.sleep(1)
|
|
233
|
+
else:
|
|
234
|
+
await reply(request, None)
|
|
165
235
|
case _:
|
|
166
236
|
logger.error(f"Unknown method: {msg['method']}")
|
|
167
237
|
|
|
@@ -239,9 +309,9 @@ async def main(port, datapath, url, timeout=1, buffer=1024, interval=60):
|
|
|
239
309
|
@click.option('--interval',
|
|
240
310
|
default=60,
|
|
241
311
|
help='Interval of flush cache, in unit of second.')
|
|
242
|
-
def
|
|
312
|
+
def server(port, datapath, url, timeout, buffer, interval):
|
|
243
313
|
asyncio.run(main(port, Path(datapath), url, timeout, buffer, interval))
|
|
244
314
|
|
|
245
315
|
|
|
246
316
|
if __name__ == "__main__":
|
|
247
|
-
|
|
317
|
+
server()
|
|
@@ -98,7 +98,8 @@ class ZMQContextManager:
|
|
|
98
98
|
public_keys_location: Optional[str] = None,
|
|
99
99
|
secret_key: Optional[bytes] = None,
|
|
100
100
|
public_key: Optional[bytes] = None,
|
|
101
|
-
server_public_key: Optional[bytes] = None
|
|
101
|
+
server_public_key: Optional[bytes] = None,
|
|
102
|
+
socket: Optional[zmq.Socket] = None):
|
|
102
103
|
self.socket_type = socket_type
|
|
103
104
|
if bind is None and connect is None:
|
|
104
105
|
raise ValueError("Either 'bind' or 'connect' must be specified.")
|
|
@@ -129,6 +130,7 @@ class ZMQContextManager:
|
|
|
129
130
|
self.auth = None
|
|
130
131
|
self.context = None
|
|
131
132
|
self.socket = None
|
|
133
|
+
self._external_socket = socket
|
|
132
134
|
|
|
133
135
|
def _create_socket(self, asyncio=False) -> zmq.Socket:
|
|
134
136
|
"""
|
|
@@ -138,6 +140,8 @@ class ZMQContextManager:
|
|
|
138
140
|
Returns:
|
|
139
141
|
zmq.Socket: The configured ZeroMQ socket.
|
|
140
142
|
"""
|
|
143
|
+
if self._external_socket:
|
|
144
|
+
return self._external_socket
|
|
141
145
|
if asyncio:
|
|
142
146
|
self.context = zmq.asyncio.Context()
|
|
143
147
|
else:
|
|
@@ -185,6 +189,8 @@ class ZMQContextManager:
|
|
|
185
189
|
Closes the ZeroMQ socket and the context, and stops the authenticator
|
|
186
190
|
if it was started.
|
|
187
191
|
"""
|
|
192
|
+
if self._external_socket:
|
|
193
|
+
return
|
|
188
194
|
if self.observer:
|
|
189
195
|
self.observer.stop()
|
|
190
196
|
self.observer.join()
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "2.1.3"
|
qulab-2.1.2/qulab/scan/server.py
DELETED
|
@@ -1,104 +0,0 @@
|
|
|
1
|
-
import asyncio
|
|
2
|
-
import pickle
|
|
3
|
-
|
|
4
|
-
import click
|
|
5
|
-
import dill
|
|
6
|
-
import zmq
|
|
7
|
-
from loguru import logger
|
|
8
|
-
|
|
9
|
-
from qulab.sys.rpc.zmq_socket import ZMQContextManager
|
|
10
|
-
|
|
11
|
-
from .scan import Scan
|
|
12
|
-
|
|
13
|
-
pool = {}
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
class Request():
|
|
17
|
-
__slots__ = ['sock', 'identity', 'msg', 'method']
|
|
18
|
-
|
|
19
|
-
def __init__(self, sock, identity, msg):
|
|
20
|
-
self.sock = sock
|
|
21
|
-
self.identity = identity
|
|
22
|
-
self.msg = pickle.loads(msg)
|
|
23
|
-
self.method = self.msg.get('method', '')
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
async def reply(req, resp):
|
|
27
|
-
await req.sock.send_multipart([req.identity, pickle.dumps(resp)])
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
@logger.catch
|
|
31
|
-
async def handle(request: Request):
|
|
32
|
-
|
|
33
|
-
msg = request.msg
|
|
34
|
-
|
|
35
|
-
match request.method:
|
|
36
|
-
case 'ping':
|
|
37
|
-
await reply(request, 'pong')
|
|
38
|
-
case 'submit':
|
|
39
|
-
description = dill.loads(msg['description'])
|
|
40
|
-
task = Scan()
|
|
41
|
-
task.description = description
|
|
42
|
-
task.start()
|
|
43
|
-
pool[task.id] = task
|
|
44
|
-
await reply(request, task.id)
|
|
45
|
-
case 'get_record_id':
|
|
46
|
-
task = pool.get(msg['id'])
|
|
47
|
-
for _ in range(10):
|
|
48
|
-
if task.record:
|
|
49
|
-
await reply(request, task.record.id)
|
|
50
|
-
break
|
|
51
|
-
await asyncio.sleep(1)
|
|
52
|
-
else:
|
|
53
|
-
await reply(request, None)
|
|
54
|
-
case _:
|
|
55
|
-
logger.error(f"Unknown method: {msg['method']}")
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
async def _handle(request: Request):
|
|
59
|
-
try:
|
|
60
|
-
await handle(request)
|
|
61
|
-
except:
|
|
62
|
-
await reply(request, 'error')
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
async def serv(port):
|
|
66
|
-
logger.info('Server starting.')
|
|
67
|
-
async with ZMQContextManager(zmq.ROUTER, bind=f"tcp://*:{port}") as sock:
|
|
68
|
-
logger.info('Server started.')
|
|
69
|
-
while True:
|
|
70
|
-
identity, msg = await sock.recv_multipart()
|
|
71
|
-
req = Request(sock, identity, msg)
|
|
72
|
-
asyncio.create_task(_handle(req))
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
async def watch(port, timeout=1):
|
|
76
|
-
with ZMQContextManager(zmq.DEALER,
|
|
77
|
-
connect=f"tcp://127.0.0.1:{port}") as sock:
|
|
78
|
-
sock.setsockopt(zmq.LINGER, 0)
|
|
79
|
-
while True:
|
|
80
|
-
try:
|
|
81
|
-
sock.send_pyobj({"method": "ping"})
|
|
82
|
-
if sock.poll(int(1000 * timeout)):
|
|
83
|
-
sock.recv()
|
|
84
|
-
else:
|
|
85
|
-
raise asyncio.TimeoutError()
|
|
86
|
-
except (zmq.error.ZMQError, asyncio.TimeoutError):
|
|
87
|
-
return asyncio.create_task(serv(port))
|
|
88
|
-
await asyncio.sleep(timeout)
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
async def main(port, timeout=1):
|
|
92
|
-
task = await watch(port=port, timeout=timeout)
|
|
93
|
-
await task
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
@click.command()
|
|
97
|
-
@click.option('--port', default=6788, help='Port of the server.')
|
|
98
|
-
@click.option('--timeout', default=1, help='Timeout of ping.')
|
|
99
|
-
def server(port, timeout):
|
|
100
|
-
asyncio.run(main(port, timeout))
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
if __name__ == "__main__":
|
|
104
|
-
server()
|
qulab-2.1.2/qulab/version.py
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "2.1.2"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|