QuLab 2.1.2__tar.gz → 2.1.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. {qulab-2.1.2 → qulab-2.1.4}/PKG-INFO +1 -1
  2. {qulab-2.1.2 → qulab-2.1.4}/QuLab.egg-info/PKG-INFO +1 -1
  3. {qulab-2.1.2 → qulab-2.1.4}/QuLab.egg-info/SOURCES.txt +0 -1
  4. {qulab-2.1.2 → qulab-2.1.4}/qulab/__main__.py +0 -2
  5. {qulab-2.1.2 → qulab-2.1.4}/qulab/scan/curd.py +78 -1
  6. {qulab-2.1.2 → qulab-2.1.4}/qulab/scan/models.py +21 -6
  7. {qulab-2.1.2 → qulab-2.1.4}/qulab/scan/query.py +1 -1
  8. {qulab-2.1.2 → qulab-2.1.4}/qulab/scan/record.py +44 -0
  9. {qulab-2.1.2 → qulab-2.1.4}/qulab/scan/scan.py +131 -8
  10. qulab-2.1.2/qulab/scan/recorder.py → qulab-2.1.4/qulab/scan/server.py +77 -4
  11. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/rpc/zmq_socket.py +7 -1
  12. qulab-2.1.4/qulab/version.py +1 -0
  13. qulab-2.1.2/qulab/scan/server.py +0 -104
  14. qulab-2.1.2/qulab/version.py +0 -1
  15. {qulab-2.1.2 → qulab-2.1.4}/LICENSE +0 -0
  16. {qulab-2.1.2 → qulab-2.1.4}/MANIFEST.in +0 -0
  17. {qulab-2.1.2 → qulab-2.1.4}/QuLab.egg-info/dependency_links.txt +0 -0
  18. {qulab-2.1.2 → qulab-2.1.4}/QuLab.egg-info/entry_points.txt +0 -0
  19. {qulab-2.1.2 → qulab-2.1.4}/QuLab.egg-info/requires.txt +0 -0
  20. {qulab-2.1.2 → qulab-2.1.4}/QuLab.egg-info/top_level.txt +0 -0
  21. {qulab-2.1.2 → qulab-2.1.4}/README.md +0 -0
  22. {qulab-2.1.2 → qulab-2.1.4}/pyproject.toml +0 -0
  23. {qulab-2.1.2 → qulab-2.1.4}/qulab/__init__.py +0 -0
  24. {qulab-2.1.2 → qulab-2.1.4}/qulab/monitor/__init__.py +0 -0
  25. {qulab-2.1.2 → qulab-2.1.4}/qulab/monitor/__main__.py +0 -0
  26. {qulab-2.1.2 → qulab-2.1.4}/qulab/monitor/config.py +0 -0
  27. {qulab-2.1.2 → qulab-2.1.4}/qulab/monitor/dataset.py +0 -0
  28. {qulab-2.1.2 → qulab-2.1.4}/qulab/monitor/event_queue.py +0 -0
  29. {qulab-2.1.2 → qulab-2.1.4}/qulab/monitor/mainwindow.py +0 -0
  30. {qulab-2.1.2 → qulab-2.1.4}/qulab/monitor/monitor.py +0 -0
  31. {qulab-2.1.2 → qulab-2.1.4}/qulab/monitor/ploter.py +0 -0
  32. {qulab-2.1.2 → qulab-2.1.4}/qulab/monitor/qt_compat.py +0 -0
  33. {qulab-2.1.2 → qulab-2.1.4}/qulab/monitor/toolbar.py +0 -0
  34. {qulab-2.1.2 → qulab-2.1.4}/qulab/scan/__init__.py +0 -0
  35. {qulab-2.1.2 → qulab-2.1.4}/qulab/scan/expression.py +0 -0
  36. {qulab-2.1.2 → qulab-2.1.4}/qulab/scan/optimize.py +0 -0
  37. {qulab-2.1.2 → qulab-2.1.4}/qulab/scan/space.py +0 -0
  38. {qulab-2.1.2 → qulab-2.1.4}/qulab/scan/utils.py +0 -0
  39. {qulab-2.1.2 → qulab-2.1.4}/qulab/storage/__init__.py +0 -0
  40. {qulab-2.1.2 → qulab-2.1.4}/qulab/storage/__main__.py +0 -0
  41. {qulab-2.1.2 → qulab-2.1.4}/qulab/storage/backend/__init__.py +0 -0
  42. {qulab-2.1.2 → qulab-2.1.4}/qulab/storage/backend/redis.py +0 -0
  43. {qulab-2.1.2 → qulab-2.1.4}/qulab/storage/base_dataset.py +0 -0
  44. {qulab-2.1.2 → qulab-2.1.4}/qulab/storage/chunk.py +0 -0
  45. {qulab-2.1.2 → qulab-2.1.4}/qulab/storage/dataset.py +0 -0
  46. {qulab-2.1.2 → qulab-2.1.4}/qulab/storage/file.py +0 -0
  47. {qulab-2.1.2 → qulab-2.1.4}/qulab/storage/models/__init__.py +0 -0
  48. {qulab-2.1.2 → qulab-2.1.4}/qulab/storage/models/base.py +0 -0
  49. {qulab-2.1.2 → qulab-2.1.4}/qulab/storage/models/config.py +0 -0
  50. {qulab-2.1.2 → qulab-2.1.4}/qulab/storage/models/file.py +0 -0
  51. {qulab-2.1.2 → qulab-2.1.4}/qulab/storage/models/ipy.py +0 -0
  52. {qulab-2.1.2 → qulab-2.1.4}/qulab/storage/models/models.py +0 -0
  53. {qulab-2.1.2 → qulab-2.1.4}/qulab/storage/models/record.py +0 -0
  54. {qulab-2.1.2 → qulab-2.1.4}/qulab/storage/models/report.py +0 -0
  55. {qulab-2.1.2 → qulab-2.1.4}/qulab/storage/models/tag.py +0 -0
  56. {qulab-2.1.2 → qulab-2.1.4}/qulab/storage/storage.py +0 -0
  57. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/__init__.py +0 -0
  58. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/chat.py +0 -0
  59. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/device/__init__.py +0 -0
  60. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/device/basedevice.py +0 -0
  61. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/device/loader.py +0 -0
  62. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/device/utils.py +0 -0
  63. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/drivers/FakeInstrument.py +0 -0
  64. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/drivers/__init__.py +0 -0
  65. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/ipy_events.py +0 -0
  66. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/net/__init__.py +0 -0
  67. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/net/bencoder.py +0 -0
  68. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/net/cli.py +0 -0
  69. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/net/dhcp.py +0 -0
  70. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/net/dhcpd.py +0 -0
  71. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/net/kad.py +0 -0
  72. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/net/kcp.py +0 -0
  73. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/net/nginx.py +0 -0
  74. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/progress.py +0 -0
  75. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/rpc/__init__.py +0 -0
  76. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/rpc/client.py +0 -0
  77. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/rpc/exceptions.py +0 -0
  78. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/rpc/msgpack.py +0 -0
  79. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/rpc/msgpack.pyi +0 -0
  80. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/rpc/rpc.py +0 -0
  81. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/rpc/serialize.py +0 -0
  82. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/rpc/server.py +0 -0
  83. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/rpc/socket.py +0 -0
  84. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/rpc/utils.py +0 -0
  85. {qulab-2.1.2 → qulab-2.1.4}/qulab/sys/rpc/worker.py +0 -0
  86. {qulab-2.1.2 → qulab-2.1.4}/qulab/visualization/__init__.py +0 -0
  87. {qulab-2.1.2 → qulab-2.1.4}/qulab/visualization/__main__.py +0 -0
  88. {qulab-2.1.2 → qulab-2.1.4}/qulab/visualization/_autoplot.py +0 -0
  89. {qulab-2.1.2 → qulab-2.1.4}/qulab/visualization/plot_layout.py +0 -0
  90. {qulab-2.1.2 → qulab-2.1.4}/qulab/visualization/plot_seq.py +0 -0
  91. {qulab-2.1.2 → qulab-2.1.4}/qulab/visualization/qdat.py +0 -0
  92. {qulab-2.1.2 → qulab-2.1.4}/qulab/visualization/widgets.py +0 -0
  93. {qulab-2.1.2 → qulab-2.1.4}/setup.cfg +0 -0
  94. {qulab-2.1.2 → qulab-2.1.4}/setup.py +0 -0
  95. {qulab-2.1.2 → qulab-2.1.4}/src/qulab.h +0 -0
  96. {qulab-2.1.2 → qulab-2.1.4}/tests/test_scan.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: QuLab
3
- Version: 2.1.2
3
+ Version: 2.1.4
4
4
  Summary: contral instruments and manage data
5
5
  Author-email: feihoo87 <feihoo87@gmail.com>
6
6
  Maintainer-email: feihoo87 <feihoo87@gmail.com>
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: QuLab
3
- Version: 2.1.2
3
+ Version: 2.1.4
4
4
  Summary: contral instruments and manage data
5
5
  Author-email: feihoo87 <feihoo87@gmail.com>
6
6
  Maintainer-email: feihoo87 <feihoo87@gmail.com>
@@ -29,7 +29,6 @@ qulab/scan/models.py
29
29
  qulab/scan/optimize.py
30
30
  qulab/scan/query.py
31
31
  qulab/scan/record.py
32
- qulab/scan/recorder.py
33
32
  qulab/scan/scan.py
34
33
  qulab/scan/server.py
35
34
  qulab/scan/space.py
@@ -1,7 +1,6 @@
1
1
  import click
2
2
 
3
3
  from .monitor.__main__ import main as monitor
4
- from .scan.recorder import record
5
4
  from .scan.server import server
6
5
  from .sys.net.cli import dht
7
6
  from .visualization.__main__ import plot
@@ -21,7 +20,6 @@ def hello():
21
20
  main.add_command(monitor)
22
21
  main.add_command(plot)
23
22
  main.add_command(dht)
24
- main.add_command(record)
25
23
  main.add_command(server)
26
24
 
27
25
  if __name__ == '__main__':
@@ -1,4 +1,7 @@
1
+ import lzma
2
+ import pickle
1
3
  from datetime import date, datetime, timezone
4
+ from pathlib import Path
2
5
  from typing import Sequence, Type, Union
3
6
 
4
7
  from sqlalchemy.orm import Query, Session, aliased
@@ -6,7 +9,8 @@ from sqlalchemy.orm.exc import NoResultFound
6
9
  from sqlalchemy.orm.session import Session
7
10
  from waveforms.dicttree import foldDict
8
11
 
9
- from .models import Comment, Record, Report, Sample, Tag
12
+ from .models import (Cell, Comment, Config, InputText, Notebook, Record,
13
+ Report, Sample, Tag, utcnow)
10
14
 
11
15
 
12
16
  def tag(session: Session, tag_text: str) -> Tag:
@@ -142,3 +146,76 @@ def remove_tags(session: Session, record_id: int, tags: Sequence[str]):
142
146
  session.rollback()
143
147
  return False
144
148
  return True
149
+
150
+
151
+ def create_notebook(session: Session, notebook_name: str) -> Notebook:
152
+ """Create a notebook in the database."""
153
+ notebook = Notebook(name=notebook_name)
154
+ session.add(notebook)
155
+ return notebook
156
+
157
+
158
+ def create_input_text(session: Session, input_text: str) -> InputText:
159
+ """Create an input text in the database."""
160
+ input = InputText()
161
+ input.text = input_text
162
+ try:
163
+ input = session.query(InputText).filter(
164
+ InputText.hash == input.hash,
165
+ InputText.text_field == input_text).one()
166
+ except NoResultFound:
167
+ session.add(input)
168
+ return input
169
+
170
+
171
+ def create_cell(session: Session, notebook: Notebook, input_text: str) -> Cell:
172
+ """Create a cell in the database."""
173
+ cell = Cell()
174
+ cell.notebook = notebook
175
+ cell.input = create_input_text(session, input_text)
176
+ cell.index = len(notebook.cells) - 1
177
+ session.add(cell)
178
+ notebook.atime = cell.ctime
179
+ return cell
180
+
181
+
182
+ def create_config(session: Session, config: dict | bytes, base: Path,
183
+ filename: str) -> Config:
184
+ """Create a config in the database."""
185
+
186
+ if not isinstance(config, bytes):
187
+ buf = pickle.dumps(config)
188
+ buf = lzma.compress(buf)
189
+ content_type = 'application/pickle+lzma'
190
+ else:
191
+ buf = config
192
+ content_type = 'application/octet-stream'
193
+ config = Config(buf)
194
+ config.content_type = content_type
195
+ for cfg in session.query(Config).filter(Config.hash == config.hash).all():
196
+ with open(base / cfg.file, 'rb') as f:
197
+ if f.read() == buf:
198
+ cfg.atime = utcnow()
199
+ return cfg
200
+ else:
201
+ path = base / filename
202
+ path.parent.mkdir(parents=True, exist_ok=True)
203
+ with open(path, 'wb') as f:
204
+ f.write(buf)
205
+ config.file = filename
206
+ session.add(config)
207
+ return config
208
+
209
+
210
+ def get_config(session: Session, config_id: int, base: Path):
211
+ config = session.get(Config, config_id)
212
+ if config is None:
213
+ return None
214
+ config.atime = utcnow()
215
+ path = base / config.file
216
+ with open(path, 'rb') as f:
217
+ buf = f.read()
218
+ if config.content_type == 'application/pickle+lzma':
219
+ buf = lzma.decompress(buf)
220
+ buf = pickle.loads(buf)
221
+ return buf
@@ -1,11 +1,9 @@
1
1
  import hashlib
2
2
  import pickle
3
- import time
4
3
  from datetime import datetime, timezone
5
4
  from functools import singledispatchmethod
6
- from typing import Optional
7
5
 
8
- from sqlalchemy import (JSON, Column, DateTime, Float, ForeignKey, Integer,
6
+ from sqlalchemy import (Column, DateTime, Float, ForeignKey, Integer,
9
7
  LargeBinary, String, Table, Text, create_engine)
10
8
  from sqlalchemy.orm import (backref, declarative_base, relationship,
11
9
  sessionmaker)
@@ -325,7 +323,7 @@ class InputText(Base):
325
323
  __tablename__ = 'inputs'
326
324
 
327
325
  id = Column(Integer, primary_key=True)
328
- hash = Column(LargeBinary(20))
326
+ hash = Column(LargeBinary(20), index=True)
329
327
  text_field = Column(Text, unique=True)
330
328
 
331
329
  @property
@@ -432,6 +430,22 @@ class SampleTransfer(Base):
432
430
  comments = relationship("Comment", secondary=sample_transfer_comments)
433
431
 
434
432
 
433
+ class Config(Base):
434
+ __tablename__ = 'configs'
435
+
436
+ id = Column(Integer, primary_key=True)
437
+ hash = Column(LargeBinary(20), index=True)
438
+ file = Column(String)
439
+ content_type = Column(String, default='application/pickle')
440
+ ctime = Column(DateTime, default=utcnow)
441
+ atime = Column(DateTime, default=utcnow)
442
+
443
+ records = relationship("Record", back_populates="config")
444
+
445
+ def __init__(self, data: bytes) -> None:
446
+ self.hash = hashlib.sha1(data).digest()
447
+
448
+
435
449
  class Record(Base):
436
450
  __tablename__ = 'records'
437
451
 
@@ -440,14 +454,14 @@ class Record(Base):
440
454
  mtime = Column(DateTime, default=utcnow)
441
455
  atime = Column(DateTime, default=utcnow)
442
456
  user_id = Column(Integer, ForeignKey('users.id'))
457
+ config_id = Column(Integer, ForeignKey('configs.id'))
443
458
  parent_id = Column(Integer, ForeignKey('records.id'))
444
459
  cell_id = Column(Integer, ForeignKey('cells.id'))
445
460
 
446
461
  app = Column(String)
447
462
  file = Column(String)
463
+ content_type = Column(String, default='application/pickle')
448
464
  key = Column(String)
449
- config = Column(JSON)
450
- task_hash = Column(LargeBinary(32))
451
465
 
452
466
  parent = relationship("Record",
453
467
  remote_side=[id],
@@ -456,6 +470,7 @@ class Record(Base):
456
470
  remote_side=[parent_id],
457
471
  back_populates="parent")
458
472
 
473
+ config = relationship("Config", back_populates="records")
459
474
  user = relationship("User")
460
475
  samples = relationship("Sample",
461
476
  secondary=sample_records,
@@ -12,7 +12,7 @@ from sqlalchemy.orm import sessionmaker
12
12
  from qulab.sys.rpc.zmq_socket import ZMQContextManager
13
13
 
14
14
  from .record import Record
15
- from .recorder import get_local_record
15
+ from .server import get_local_record
16
16
  from .scan import default_server
17
17
 
18
18
 
@@ -95,6 +95,11 @@ class BufferList():
95
95
  dill.dump(item, f)
96
96
  self._list.clear()
97
97
 
98
+ def delete(self):
99
+ if isinstance(self.file, Path):
100
+ self.file.unlink()
101
+ self.file = None
102
+
98
103
  def append(self, pos, value, dims=None):
99
104
  if dims is not None:
100
105
  if any([p != 0 for i, p in enumerate(pos) if i not in dims]):
@@ -450,6 +455,20 @@ class Record():
450
455
  with open(self._file, 'wb') as f:
451
456
  dill.dump(self, f)
452
457
 
458
+ def delete(self):
459
+ if self.is_remote_record():
460
+ with ZMQContextManager(zmq.DEALER,
461
+ connect=self.database) as socket:
462
+ socket.send_pyobj({
463
+ 'method': 'record_delete',
464
+ 'record_id': self.id
465
+ })
466
+ elif self.is_local_record():
467
+ for key, value in self._items.items():
468
+ if isinstance(value, BufferList):
469
+ value.delete()
470
+ self._file.unlink()
471
+
453
472
  def export(self, file):
454
473
  with zipfile.ZipFile(file,
455
474
  'w',
@@ -470,8 +489,33 @@ class Record():
470
489
  else:
471
490
  items[key] = value
472
491
  with z.open('record.pkl', 'w') as f:
492
+ self.description['entry']['scripts'] = self.scripts()
473
493
  dill.dump((self.description, items), f)
474
494
 
495
+ def scripts(self, session=None):
496
+ scripts = self.description['entry']['scripts']
497
+ if isinstance(scripts, list):
498
+ return scripts
499
+ else:
500
+ cell_id = scripts
501
+
502
+ if self.is_remote_record():
503
+ with ZMQContextManager(zmq.DEALER,
504
+ connect=self.database) as socket:
505
+ socket.send_pyobj({
506
+ 'method': 'notebook_history',
507
+ 'cell_id': cell_id
508
+ })
509
+ return socket.recv_pyobj()
510
+ elif self.is_local_record():
511
+ from .models import Cell
512
+ assert session is not None, "session is required for local record"
513
+ cell = session.get(Cell, cell_id)
514
+ return [
515
+ cell.input.text
516
+ for cell in cell.notebook.cells[1:cell.index + 2]
517
+ ]
518
+
475
519
  @classmethod
476
520
  def load(cls, file: str):
477
521
  with zipfile.ZipFile(file, 'r') as z:
@@ -2,8 +2,12 @@ import asyncio
2
2
  import copy
3
3
  import inspect
4
4
  import itertools
5
+ import lzma
5
6
  import os
7
+ import pickle
8
+ import platform
6
9
  import re
10
+ import subprocess
7
11
  import sys
8
12
  import uuid
9
13
  from concurrent.futures import ProcessPoolExecutor
@@ -19,7 +23,7 @@ from ..sys.rpc.zmq_socket import ZMQContextManager
19
23
  from .expression import Env, Expression, Symbol
20
24
  from .optimize import NgOptimizer
21
25
  from .record import Record
22
- from .recorder import default_record_port
26
+ from .server import default_record_port
23
27
  from .space import Optimizer, OptimizeSpace, Space
24
28
  from .utils import async_zip, call_function, dump_globals
25
29
 
@@ -41,6 +45,7 @@ except:
41
45
 
42
46
  __process_uuid = uuid.uuid1()
43
47
  __task_counter = itertools.count()
48
+ __notebook_id = None
44
49
 
45
50
  if os.getenv('QULAB_SERVER'):
46
51
  default_server = os.getenv('QULAB_SERVER')
@@ -52,6 +57,105 @@ else:
52
57
  default_executor = default_server
53
58
 
54
59
 
60
+ def yapf_reformat(cell_text):
61
+ try:
62
+ import isort
63
+ import yapf.yapflib.yapf_api
64
+
65
+ fname = f"f{uuid.uuid1().hex}"
66
+
67
+ def wrap(source):
68
+ lines = [f"async def {fname}():"]
69
+ for line in source.split('\n'):
70
+ lines.append(" " + line)
71
+ return '\n'.join(lines)
72
+
73
+ def unwrap(source):
74
+ lines = []
75
+ for line in source.split('\n'):
76
+ if line.startswith(f"async def {fname}():"):
77
+ continue
78
+ lines.append(line[4:])
79
+ return '\n'.join(lines)
80
+
81
+ cell_text = re.sub('^%', '#%#', cell_text, flags=re.M)
82
+ reformated_text = unwrap(
83
+ yapf.yapflib.yapf_api.FormatCode(wrap(isort.code(cell_text)))[0])
84
+ return re.sub('^#%#', '%', reformated_text, flags=re.M)
85
+ except:
86
+ return cell_text
87
+
88
+
89
+ def get_installed_packages():
90
+ result = subprocess.run([sys.executable, '-m', 'pip', 'freeze'],
91
+ stdout=subprocess.PIPE,
92
+ text=True)
93
+
94
+ lines = result.stdout.split('\n')
95
+ packages = []
96
+ for line in lines:
97
+ if line:
98
+ packages.append(line)
99
+ return packages
100
+
101
+
102
+ def get_system_info():
103
+ info = {
104
+ 'OS': platform.uname()._asdict(),
105
+ 'Python': sys.version,
106
+ 'PythonExecutable': sys.executable,
107
+ 'PythonPath': sys.path,
108
+ 'packages': get_installed_packages()
109
+ }
110
+ return info
111
+
112
+
113
+ def current_notebook():
114
+ return __notebook_id
115
+
116
+
117
+ async def create_notebook(name: str, database=default_server, socket=None):
118
+ global __notebook_id
119
+
120
+ async with ZMQContextManager(zmq.DEALER, connect=database,
121
+ socket=socket) as socket:
122
+ await socket.send_pyobj({'method': 'notebook_create', 'name': name})
123
+ __notebook_id = await socket.recv_pyobj()
124
+
125
+
126
+ async def save_input_cells(notebook_id,
127
+ input_cells,
128
+ database=default_server,
129
+ socket=None):
130
+ async with ZMQContextManager(zmq.DEALER, connect=database,
131
+ socket=socket) as socket:
132
+ await socket.send_pyobj({
133
+ 'method': 'notebook_extend',
134
+ 'notebook_id': notebook_id,
135
+ 'input_cells': input_cells
136
+ })
137
+ return await socket.recv_pyobj()
138
+
139
+
140
+ async def create_config(config: dict, database=default_server, socket=None):
141
+ async with ZMQContextManager(zmq.DEALER, connect=database,
142
+ socket=socket) as socket:
143
+ buf = lzma.compress(pickle.dumps(config))
144
+ await socket.send_pyobj({'method': 'config_update', 'update': buf})
145
+ return await socket.recv_pyobj()
146
+
147
+
148
+ async def get_config(config_id: int, database=default_server, socket=None):
149
+ async with ZMQContextManager(zmq.DEALER, connect=database,
150
+ socket=socket) as socket:
151
+ await socket.send_pyobj({
152
+ 'method': 'config_get',
153
+ 'config_id': config_id
154
+ })
155
+ buf = await socket.recv_pyobj()
156
+ return pickle.loads(lzma.decompress(buf))
157
+
158
+
55
159
  def task_uuid():
56
160
  return uuid.uuid3(__process_uuid, str(next(__task_counter)))
57
161
 
@@ -135,10 +239,11 @@ class Scan():
135
239
  mixin=None):
136
240
  self.id = task_uuid()
137
241
  self.record = None
138
- self.namespace = {}
242
+ self.config = None
139
243
  self.description = {
140
244
  'app': app,
141
245
  'tags': tags,
246
+ 'config': None,
142
247
  'loops': {},
143
248
  'intrinsic_loops': {},
144
249
  'consts': {},
@@ -157,9 +262,11 @@ class Scan():
157
262
  'database': database,
158
263
  'hiden': ['self', r'^__.*', r'.*__$'],
159
264
  'entry': {
265
+ 'system': get_system_info(),
160
266
  'env': {},
161
267
  'shell': '',
162
- 'cmds': []
268
+ 'cmds': [],
269
+ 'scripts': []
163
270
  },
164
271
  }
165
272
  self._current_level = 0
@@ -248,8 +355,7 @@ class Scan():
248
355
 
249
356
  def emit(self, current_level, step, position, variables: dict[str, Any]):
250
357
  self._msg_queue.put_nowait(
251
- asyncio.create_task(
252
- self._emit(current_level, step, position, variables.copy())))
358
+ self._emit(current_level, step, position, variables.copy()))
253
359
 
254
360
  def hide(self, name: str):
255
361
  self.description['hiden'].append(name)
@@ -287,8 +393,6 @@ class Scan():
287
393
  def get(self, name: str):
288
394
  if name in self.description['consts']:
289
395
  return self.description['consts'][name]
290
- elif name in self.namespace:
291
- return self.namespace.get(name)
292
396
  else:
293
397
  return Symbol(name)
294
398
 
@@ -442,6 +546,17 @@ class Scan():
442
546
 
443
547
  async def run(self):
444
548
  assymbly(self.description)
549
+ if self.config:
550
+ self.description['config'] = await create_config(
551
+ self.config, self.description['database'], self._sock)
552
+ if current_notebook() is None:
553
+ await create_notebook('untitle', self.description['database'],
554
+ self._sock)
555
+ cell_id = await save_input_cells(current_notebook(),
556
+ self.description['entry']['scripts'],
557
+ self.description['database'],
558
+ self._sock)
559
+ self.description['entry']['scripts'] = cell_id
445
560
  if isinstance(
446
561
  self.description['database'],
447
562
  str) and self.description['database'].startswith("tcp://"):
@@ -635,13 +750,21 @@ def assymbly(description):
635
750
  ipy = get_ipython()
636
751
  if ipy is not None:
637
752
  description['entry']['shell'] = 'ipython'
638
- description['entry']['cmds'] = ipy.user_ns['In']
753
+ description['entry']['scripts'] = [
754
+ yapf_reformat(cell_text) for cell_text in ipy.user_ns['In']
755
+ ]
639
756
  else:
640
757
  try:
641
758
  description['entry']['shell'] = 'shell'
642
759
  description['entry']['cmds'] = [
643
760
  sys.executable, __main__.__file__, *sys.argv[1:]
644
761
  ]
762
+ description['entry']['scripts'] = []
763
+ try:
764
+ with open(__main__.__file__) as f:
765
+ description['entry']['scripts'].append(f.read())
766
+ except:
767
+ pass
645
768
  except:
646
769
  pass
647
770
 
@@ -11,10 +11,12 @@ from loguru import logger
11
11
 
12
12
  from qulab.sys.rpc.zmq_socket import ZMQContextManager
13
13
 
14
- from .curd import query_record, remove_tags, tag, update_tags
14
+ from .curd import (create_cell, create_config, create_notebook, get_config,
15
+ query_record, remove_tags, tag, update_tags)
16
+ from .models import Cell, Notebook
15
17
  from .models import Record as RecordInDB
16
18
  from .models import Session, create_engine, create_tables, sessionmaker, utcnow
17
- from .record import Record
19
+ from .record import BufferList, Record, random_path
18
20
 
19
21
  try:
20
22
  default_record_port = int(os.getenv('QULAB_RECORD_PORT', 6789))
@@ -30,6 +32,8 @@ datapath.mkdir(parents=True, exist_ok=True)
30
32
  record_cache = {}
31
33
  CACHE_SIZE = 1024
32
34
 
35
+ pool = {}
36
+
33
37
 
34
38
  class Request():
35
39
  __slots__ = ['sock', 'identity', 'msg', 'method']
@@ -62,6 +66,10 @@ def flush_cache():
62
66
  def get_local_record(session: Session, id: int, datapath: Path) -> Record:
63
67
  record_in_db = session.get(RecordInDB, id)
64
68
  record_in_db.atime = utcnow()
69
+
70
+ if record_in_db.file.endswith('.zip'):
71
+ return Record.load(datapath / 'objects' / record_in_db.file)
72
+
65
73
  path = datapath / 'objects' / record_in_db.file
66
74
  with open(path, 'rb') as f:
67
75
  record = dill.load(f)
@@ -88,6 +96,7 @@ def record_create(session: Session, description: dict, datapath: Path) -> int:
88
96
  if 'tags' in description:
89
97
  record_in_db.tags = [tag(session, t) for t in description['tags']]
90
98
  record_in_db.file = '/'.join(record._file.parts[-4:])
99
+ record_in_db.config_id = description['config']
91
100
  record._file = datapath / 'objects' / record_in_db.file
92
101
  session.add(record_in_db)
93
102
  try:
@@ -115,6 +124,14 @@ def record_append(session: Session, record_id: int, level: int, step: int,
115
124
  raise
116
125
 
117
126
 
127
+ def record_delete(session: Session, record_id: int, datapath: Path):
128
+ record = get_local_record(session, record_id, datapath)
129
+ record.delete()
130
+ record_in_db = session.get(RecordInDB, record_id)
131
+ session.delete(record_in_db)
132
+ session.commit()
133
+
134
+
118
135
  @logger.catch
119
136
  async def handle(session: Session, request: Request, datapath: Path):
120
137
 
@@ -162,6 +179,62 @@ async def handle(session: Session, request: Request, datapath: Path):
162
179
  update_tags(session, msg['record_id'], msg['tags'], True)
163
180
  case 'record_replace_tags':
164
181
  update_tags(session, msg['record_id'], msg['tags'], False)
182
+ case 'notebook_create':
183
+ notebook = create_notebook(session, msg['name'])
184
+ session.commit()
185
+ await reply(request, notebook.id)
186
+ case 'notebook_extend':
187
+ notebook = session.get(Notebook, msg['notebook_id'])
188
+ inputCells = msg.get('input_cells', [""])
189
+ aready_saved = len(notebook.cells)
190
+ if len(inputCells) > aready_saved:
191
+ for cell in inputCells[aready_saved:]:
192
+ cell = create_cell(session, notebook, cell)
193
+ session.commit()
194
+ await reply(request, cell.id)
195
+ else:
196
+ await reply(request, None)
197
+ case 'notebook_history':
198
+ cell = session.get(Cell, msg['cell_id'])
199
+ if cell:
200
+ await reply(request, [
201
+ cell.input.text
202
+ for cell in cell.notebook.cells[1:cell.index + 2]
203
+ ])
204
+ else:
205
+ await reply(request, None)
206
+ case 'config_get':
207
+ config = get_config(session,
208
+ msg['config_id'],
209
+ base=datapath / 'objects')
210
+ session.commit()
211
+ await reply(request, config)
212
+ case 'config_update':
213
+ config = create_config(session,
214
+ msg['update'],
215
+ base=datapath / 'objects',
216
+ filename='/'.join(
217
+ random_path(datapath /
218
+ 'objects').parts[-4:]))
219
+ session.commit()
220
+ await reply(request, config.id)
221
+ case 'submit':
222
+ from .scan import Scan
223
+ description = dill.loads(msg['description'])
224
+ task = Scan()
225
+ task.description = description
226
+ task.start()
227
+ pool[task.id] = task
228
+ await reply(request, task.id)
229
+ case 'get_record_id':
230
+ task = pool.get(msg['id'])
231
+ for _ in range(10):
232
+ if task.record:
233
+ await reply(request, task.record.id)
234
+ break
235
+ await asyncio.sleep(1)
236
+ else:
237
+ await reply(request, None)
165
238
  case _:
166
239
  logger.error(f"Unknown method: {msg['method']}")
167
240
 
@@ -239,9 +312,9 @@ async def main(port, datapath, url, timeout=1, buffer=1024, interval=60):
239
312
  @click.option('--interval',
240
313
  default=60,
241
314
  help='Interval of flush cache, in unit of second.')
242
- def record(port, datapath, url, timeout, buffer, interval):
315
+ def server(port, datapath, url, timeout, buffer, interval):
243
316
  asyncio.run(main(port, Path(datapath), url, timeout, buffer, interval))
244
317
 
245
318
 
246
319
  if __name__ == "__main__":
247
- record()
320
+ server()
@@ -98,7 +98,8 @@ class ZMQContextManager:
98
98
  public_keys_location: Optional[str] = None,
99
99
  secret_key: Optional[bytes] = None,
100
100
  public_key: Optional[bytes] = None,
101
- server_public_key: Optional[bytes] = None):
101
+ server_public_key: Optional[bytes] = None,
102
+ socket: Optional[zmq.Socket] = None):
102
103
  self.socket_type = socket_type
103
104
  if bind is None and connect is None:
104
105
  raise ValueError("Either 'bind' or 'connect' must be specified.")
@@ -129,6 +130,7 @@ class ZMQContextManager:
129
130
  self.auth = None
130
131
  self.context = None
131
132
  self.socket = None
133
+ self._external_socket = socket
132
134
 
133
135
  def _create_socket(self, asyncio=False) -> zmq.Socket:
134
136
  """
@@ -138,6 +140,8 @@ class ZMQContextManager:
138
140
  Returns:
139
141
  zmq.Socket: The configured ZeroMQ socket.
140
142
  """
143
+ if self._external_socket:
144
+ return self._external_socket
141
145
  if asyncio:
142
146
  self.context = zmq.asyncio.Context()
143
147
  else:
@@ -185,6 +189,8 @@ class ZMQContextManager:
185
189
  Closes the ZeroMQ socket and the context, and stops the authenticator
186
190
  if it was started.
187
191
  """
192
+ if self._external_socket:
193
+ return
188
194
  if self.observer:
189
195
  self.observer.stop()
190
196
  self.observer.join()
@@ -0,0 +1 @@
1
+ __version__ = "2.1.4"
@@ -1,104 +0,0 @@
1
- import asyncio
2
- import pickle
3
-
4
- import click
5
- import dill
6
- import zmq
7
- from loguru import logger
8
-
9
- from qulab.sys.rpc.zmq_socket import ZMQContextManager
10
-
11
- from .scan import Scan
12
-
13
- pool = {}
14
-
15
-
16
- class Request():
17
- __slots__ = ['sock', 'identity', 'msg', 'method']
18
-
19
- def __init__(self, sock, identity, msg):
20
- self.sock = sock
21
- self.identity = identity
22
- self.msg = pickle.loads(msg)
23
- self.method = self.msg.get('method', '')
24
-
25
-
26
- async def reply(req, resp):
27
- await req.sock.send_multipart([req.identity, pickle.dumps(resp)])
28
-
29
-
30
- @logger.catch
31
- async def handle(request: Request):
32
-
33
- msg = request.msg
34
-
35
- match request.method:
36
- case 'ping':
37
- await reply(request, 'pong')
38
- case 'submit':
39
- description = dill.loads(msg['description'])
40
- task = Scan()
41
- task.description = description
42
- task.start()
43
- pool[task.id] = task
44
- await reply(request, task.id)
45
- case 'get_record_id':
46
- task = pool.get(msg['id'])
47
- for _ in range(10):
48
- if task.record:
49
- await reply(request, task.record.id)
50
- break
51
- await asyncio.sleep(1)
52
- else:
53
- await reply(request, None)
54
- case _:
55
- logger.error(f"Unknown method: {msg['method']}")
56
-
57
-
58
- async def _handle(request: Request):
59
- try:
60
- await handle(request)
61
- except:
62
- await reply(request, 'error')
63
-
64
-
65
- async def serv(port):
66
- logger.info('Server starting.')
67
- async with ZMQContextManager(zmq.ROUTER, bind=f"tcp://*:{port}") as sock:
68
- logger.info('Server started.')
69
- while True:
70
- identity, msg = await sock.recv_multipart()
71
- req = Request(sock, identity, msg)
72
- asyncio.create_task(_handle(req))
73
-
74
-
75
- async def watch(port, timeout=1):
76
- with ZMQContextManager(zmq.DEALER,
77
- connect=f"tcp://127.0.0.1:{port}") as sock:
78
- sock.setsockopt(zmq.LINGER, 0)
79
- while True:
80
- try:
81
- sock.send_pyobj({"method": "ping"})
82
- if sock.poll(int(1000 * timeout)):
83
- sock.recv()
84
- else:
85
- raise asyncio.TimeoutError()
86
- except (zmq.error.ZMQError, asyncio.TimeoutError):
87
- return asyncio.create_task(serv(port))
88
- await asyncio.sleep(timeout)
89
-
90
-
91
- async def main(port, timeout=1):
92
- task = await watch(port=port, timeout=timeout)
93
- await task
94
-
95
-
96
- @click.command()
97
- @click.option('--port', default=6788, help='Port of the server.')
98
- @click.option('--timeout', default=1, help='Timeout of ping.')
99
- def server(port, timeout):
100
- asyncio.run(main(port, timeout))
101
-
102
-
103
- if __name__ == "__main__":
104
- server()
@@ -1 +0,0 @@
1
- __version__ = "2.1.2"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes