QuLab 2.1.0__tar.gz → 2.1.2__tar.gz

{qulab-2.1.0 → qulab-2.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: QuLab
-Version: 2.1.0
+Version: 2.1.2
 Summary: contral instruments and manage data
 Author-email: feihoo87 <feihoo87@gmail.com>
 Maintainer-email: feihoo87 <feihoo87@gmail.com>

{qulab-2.1.0 → qulab-2.1.2}/QuLab.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: QuLab
-Version: 2.1.0
+Version: 2.1.2
 Summary: contral instruments and manage data
 Author-email: feihoo87 <feihoo87@gmail.com>
 Maintainer-email: feihoo87 <feihoo87@gmail.com>

{qulab-2.1.0 → qulab-2.1.2}/QuLab.egg-info/SOURCES.txt

@@ -27,10 +27,12 @@ qulab/scan/curd.py
 qulab/scan/expression.py
 qulab/scan/models.py
 qulab/scan/optimize.py
-qulab/scan/query_record.py
+qulab/scan/query.py
+qulab/scan/record.py
 qulab/scan/recorder.py
 qulab/scan/scan.py
 qulab/scan/server.py
+qulab/scan/space.py
 qulab/scan/utils.py
 qulab/storage/__init__.py
 qulab/storage/__main__.py

qulab-2.1.2/qulab/__init__.py

@@ -0,0 +1,3 @@
+from .scan import Scan, get_record, load_record, lookup, lookup_list
+from .version import __version__
+from .visualization import autoplot

{qulab-2.1.0 → qulab-2.1.2}/qulab/scan/__init__.py

@@ -1,3 +1,3 @@
 from .expression import Expression, Symbol
-from .query_record import get_record, lookup, lookup_list
+from .query import get_record, load_record, lookup, lookup_list
 from .scan import Scan

qulab-2.1.0/qulab/scan/query_record.py → qulab-2.1.2/qulab/scan/query.py

@@ -6,13 +6,17 @@ import dill
 import ipywidgets as widgets
 import zmq
 from IPython.display import display
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
 
 from qulab.sys.rpc.zmq_socket import ZMQContextManager
 
-from .recorder import Record
+from .record import Record
+from .recorder import get_local_record
+from .scan import default_server
 
 
-def get_record(id, database='tcp://127.0.0.1:6789'):
+def get_record(id, database=default_server) -> Record:
     if isinstance(database, str) and database.startswith('tcp://'):
         with ZMQContextManager(zmq.DEALER, connect=database) as socket:
             socket.send_pyobj({
@@ -20,20 +24,20 @@ def get_record(id, database='tcp://127.0.0.1:6789'):
                 'record_id': id
             })
             d = dill.loads(socket.recv_pyobj())
-            return Record(id, database, d)
+            d.id = id
+            d.database = database
+            d._file = None
+            return d
     else:
-        from .models import Record as RecordInDB
-        from .models import create_engine, sessionmaker
-
         db_file = Path(database) / 'data.db'
         engine = create_engine(f'sqlite:///{db_file}')
         Session = sessionmaker(bind=engine)
         with Session() as session:
-            path = Path(database) / 'objects' / session.get(RecordInDB,
-                                                            id).file
-            with open(path, 'rb') as f:
-                record = dill.load(f)
-            return record
+            return get_local_record(session, id, database)
+
+
+def load_record(file):
+    return Record.load(file)
 
 
 def _format_tag(tag):
@@ -292,7 +296,7 @@ def _on_tags_submit(tags, ui_widgets):
     _update_view(ui_widgets)
 
 
-def lookup(app=None, limit=10, database='tcp://127.0.0.1:6789'):
+def lookup(app=None, limit=10, database=default_server):
     after = widgets.DatePicker()
     before = widgets.DatePicker()
     app_prefix = widgets.Label('App:')

qulab-2.1.0/qulab/scan/recorder.py → qulab-2.1.2/qulab/scan/record.py

@@ -1,41 +1,20 @@
-import asyncio
 import itertools
-import os
-import pickle
 import sys
-import time
 import uuid
-from collections import defaultdict
+import zipfile
 from pathlib import Path
 from threading import Lock
 from types import EllipsisType
 
-import click
 import dill
 import numpy as np
 import zmq
-from loguru import logger
 
 from qulab.sys.rpc.zmq_socket import ZMQContextManager
 
-from .curd import query_record, remove_tags, tag, update_tags
-from .models import Record as RecordInDB
-from .models import Session, create_engine, create_tables, sessionmaker, utcnow
+from .space import OptimizeSpace, Space
 
-_notgiven = object()
-
-try:
-    default_record_port = int(os.getenv('QULAB_RECORD_PORT', 6789))
-except:
-    default_record_port = 6789
-
-if os.getenv('QULAB_RECORD_PATH'):
-    datapath = Path(os.getenv('QULAB_RECORD_PATH'))
-else:
-    datapath = Path.home() / 'qulab' / 'data'
-datapath.mkdir(parents=True, exist_ok=True)
-
-record_cache = {}
+_not_given = object()
 
 
 def random_path(base):
@@ -142,6 +121,18 @@ class BufferList():
                             yield pos, value
                         except EOFError:
                             break
+        elif isinstance(
+                self.file, tuple) and len(self.file) == 2 and isinstance(
+                    self.file[0], str) and self.file[0].endswith('.zip'):
+            f, name = self.file
+            with zipfile.ZipFile(f, 'r') as z:
+                with z.open(name, 'r') as f:
+                    while True:
+                        try:
+                            pos, value = dill.load(f)
+                            yield pos, value
+                        except EOFError:
+                            break
 
     def iter(self):
         if self._data_id is None:
@@ -297,51 +288,13 @@ class BufferList():
 class Record():
 
     def __init__(self, id, database, description=None):
-        from .scan import OptimizeSpace
-
         self.id = id
         self.database = database
         self.description = description
-        self._keys = set()
         self._items = {}
-        self._index = []
         self._pos = []
         self._last_vars = set()
         self._file = None
-        self.independent_variables = {}
-        self.constants = {}
-        self.dims = {}
-
-        for name, value in self.description['consts'].items():
-            if name not in self._items:
-                self._items[name] = value
-            self.constants[name] = value
-            self.dims[name] = ()
-        for level, range_list in self.description['loops'].items():
-            for name, iterable in range_list:
-                if isinstance(iterable, OptimizeSpace):
-                    self.dims[name] = tuple(range(level + 1))
-                    continue
-                elif isinstance(iterable, (np.ndarray, list, tuple, range)):
-                    self._items[name] = iterable
-                    self.independent_variables[name] = iterable
-                self.dims[name] = (level, )
-
-        for level, group in self.description['order'].items():
-            for names in group:
-                for name in names:
-                    if name not in self.description['dependents']:
-                        if name not in self.dims:
-                            self.dims[name] = (level, )
-                    else:
-                        d = set()
-                        for n in self.description['dependents'][name]:
-                            d.update(self.dims[n])
-                        if name not in self.dims:
-                            self.dims[name] = tuple(sorted(d))
-                        else:
-                            self.dims[name] = tuple(
-                                sorted(set(self.dims[name]) | d))
 
         if self.is_local_record():
             self.database = Path(self.database)
@@ -351,32 +304,23 @@ class Record():
     def __getstate__(self) -> dict:
         return {
             'id': self.id,
-            'database': self.database,
             'description': self.description,
-            '_keys': self._keys,
             '_items': self._items,
-            '_index': self._index,
-            '_pos': self._pos,
-            '_last_vars': self._last_vars,
-            'independent_variables': self.independent_variables,
-            'constants': self.constants,
-            'dims': self.dims,
         }
 
     def __setstate__(self, state: dict):
         self.id = state['id']
-        self.database = state['database']
         self.description = state['description']
-        self._keys = state['_keys']
         self._items = state['_items']
-        self._index = state['_index']
-        self._pos = state['_pos']
-        self._last_vars = state['_last_vars']
-        self.independent_variables = state['independent_variables']
-        self.constants = state['constants']
-        self.dims = state['dims']
+        self._pos = []
+        self._last_vars = set()
+        self.database = None
         self._file = None
 
+    @property
+    def axis(self):
+        return self.description.get('axis', {})
+
     def is_local_record(self):
         return not self.is_cache_record() and not self.is_remote_record()
 
@@ -391,9 +335,12 @@ class Record():
         self.flush()
 
     def __getitem__(self, key):
-        return self.get(key, buffer_to_array=True)
+        ret = self.get(key, buffer_to_array=True)
+        if isinstance(ret, Space):
+            ret = ret.toarray()
+        return ret
 
-    def get(self, key, default=_notgiven, buffer_to_array=False, slice=None):
+    def get(self, key, default=_not_given, buffer_to_array=False, slice=None):
         if self.is_remote_record():
             with ZMQContextManager(zmq.DEALER,
                                    connect=self.database) as socket:
@@ -421,7 +368,7 @@ class Record():
                 else:
                     return ret
         else:
-            if default is _notgiven:
+            if default is _not_given:
                 d = self._items.get(key)
             else:
                 d = self._items.get(key, default)
@@ -446,7 +393,7 @@ class Record():
                 })
                 return socket.recv_pyobj()
         else:
-            return list(self._keys)
+            return list(self._items.keys())
 
     def append(self, level, step, position, variables):
         if level < 0:
@@ -454,34 +401,29 @@ class Record():
             return
 
         for key in set(variables.keys()) - self._last_vars:
-            if key not in self.dims:
-                self.dims[key] = tuple(range(level + 1))
+            if key not in self.axis:
+                self.axis[key] = tuple(range(level + 1))
 
         self._last_vars = set(variables.keys())
-        self._keys.update(variables.keys())
 
         if level >= len(self._pos):
             l = level + 1 - len(self._pos)
-            self._index.extend(([0] * (l - 1)) + [step])
             self._pos.extend(([0] * (l - 1)) + [position])
             pos = tuple(self._pos)
         elif level == len(self._pos) - 1:
-            self._index[-1] = step
             self._pos[-1] = position
             pos = tuple(self._pos)
         else:
-            self._index = self._index[:level + 1]
             self._pos = self._pos[:level + 1]
-            self._index[-1] = step + 1
             self._pos[-1] = position
             pos = tuple(self._pos)
             self._pos[-1] += 1
 
         for key, value in variables.items():
-            if self.dims[key] == ():
+            if self.axis[key] == ():
                 if key not in self._items:
                     self._items[key] = value
-            elif level == self.dims[key][-1]:
+            elif level == self.axis[key][-1]:
                 if key not in self._items:
                     if self.is_local_record():
                         bufferlist_file = random_path(self.database /
@@ -493,9 +435,9 @@ class Record():
                         self._items[key] = BufferList()
                     self._items[key].lu = pos
                     self._items[key].rd = tuple([i + 1 for i in pos])
-                    self._items[key].append(pos, value, self.dims[key])
+                    self._items[key].append(pos, value, self.axis[key])
                 elif isinstance(self._items[key], BufferList):
-                    self._items[key].append(pos, value, self.dims[key])
+                    self._items[key].append(pos, value, self.axis[key])
 
     def flush(self):
         if self.is_remote_record() or self.is_cache_record():
@@ -508,6 +450,40 @@ class Record():
         with open(self._file, 'wb') as f:
             dill.dump(self, f)
 
+    def export(self, file):
+        with zipfile.ZipFile(file,
+                             'w',
+                             compression=zipfile.ZIP_DEFLATED,
+                             compresslevel=9) as z:
+            items = {}
+            for key in self.keys():
+                value = self.get(key)
+                if isinstance(value, BufferList):
+                    v = BufferList()
+                    v.lu = value.lu
+                    v.rd = value.rd
+                    v.inner_shape = value.inner_shape
+                    items[key] = v
+                    with z.open(f'{key}.buf', 'w') as f:
+                        for pos, data in value.iter():
+                            dill.dump((pos, data), f)
+                else:
+                    items[key] = value
+            with z.open('record.pkl', 'w') as f:
+                dill.dump((self.description, items), f)
+
+    @classmethod
+    def load(cls, file: str):
+        with zipfile.ZipFile(file, 'r') as z:
+            with z.open('record.pkl', 'r') as f:
+                description, items = dill.load(f)
+            record = cls(None, None, description)
+            for key, value in items.items():
+                if isinstance(value, BufferList):
+                    value.file = file, f'{key}.buf'
+                record._items[key] = value
+        return record
+
     def __repr__(self):
         return f"<Record: id={self.id} app={self.description['app']}, keys={self.keys()}>"
 
@@ -515,214 +491,5 @@ class Record():
     #     return f"""
     #     <h3>Record: id={self.id}, app={self.description['app']}</h3>
     #     <p>keys={self.keys()}</p>
-    #     <p>dims={self.dims}</p>
+    #     <p>axis={self.axis}</p>
     #     """
-
-
-class Request():
-    __slots__ = ['sock', 'identity', 'msg', 'method']
-
-    def __init__(self, sock, identity, msg):
-        self.sock = sock
-        self.identity = identity
-        self.msg = pickle.loads(msg)
-        self.method = self.msg.get('method', '')
-
-
-async def reply(req, resp):
-    await req.sock.send_multipart([req.identity, pickle.dumps(resp)])
-
-
-def clear_cache():
-    if len(record_cache) < 1024:
-        return
-
-    for k, (t, _) in zip(sorted(record_cache.items(), key=lambda x: x[1][0]),
-                         range(len(record_cache) - 1024)):
-        del record_cache[k]
-
-
-def flush_cache():
-    for k, (t, r) in record_cache.items():
-        r.flush()
-
-
-def get_record(session: Session, id: int, datapath: Path) -> Record:
-    if id not in record_cache:
-        record_in_db = session.get(RecordInDB, id)
-        record_in_db.atime = utcnow()
-        path = datapath / 'objects' / record_in_db.file
-        with open(path, 'rb') as f:
-            record = dill.load(f)
-            record._file = path
-    else:
-        record = record_cache[id][1]
-    clear_cache()
-    record_cache[id] = time.time(), record
-    return record
-
-
-def record_create(session: Session, description: dict, datapath: Path) -> int:
-    record = Record(None, datapath, description)
-    record_in_db = RecordInDB()
-    if 'app' in description:
-        record_in_db.app = description['app']
-    if 'tags' in description:
-        record_in_db.tags = [tag(session, t) for t in description['tags']]
-    record_in_db.file = '/'.join(record._file.parts[-4:])
-    session.add(record_in_db)
-    try:
-        session.commit()
-        record.id = record_in_db.id
-        clear_cache()
-        record_cache[record.id] = time.time(), record
-        return record.id
-    except:
-        session.rollback()
-        raise
-
-
-def record_append(session: Session, record_id: int, level: int, step: int,
-                  position: int, variables: dict, datapath: Path):
-    record = get_record(session, record_id, datapath)
-    record.append(level, step, position, variables)
-    try:
-        record_in_db = session.get(RecordInDB, record_id)
-        record_in_db.mtime = utcnow()
-        record_in_db.atime = utcnow()
-        session.commit()
-    except:
-        session.rollback()
-        raise
-
-
-@logger.catch
-async def handle(session: Session, request: Request, datapath: Path):
-
-    msg = request.msg
-
-    match request.method:
-        case 'ping':
-            await reply(request, 'pong')
-        case 'bufferlist_slice':
-            record = get_record(session, msg['record_id'], datapath)
-            bufferlist = record.get(msg['key'],
-                                    buffer_to_array=False,
-                                    slice=msg['slice'])
-            await reply(request, list(bufferlist.iter()))
-        case 'record_create':
-            description = dill.loads(msg['description'])
-            await reply(request, record_create(session, description, datapath))
-        case 'record_append':
-            record_append(session, msg['record_id'], msg['level'], msg['step'],
-                          msg['position'], msg['variables'], datapath)
-        case 'record_description':
-            record = get_record(session, msg['record_id'], datapath)
-            await reply(request, dill.dumps(record.description))
-        case 'record_getitem':
-            record = get_record(session, msg['record_id'], datapath)
-            await reply(request, record.get(msg['key'], buffer_to_array=False))
-        case 'record_keys':
-            record = get_record(session, msg['record_id'], datapath)
-            await reply(request, record.keys())
-        case 'record_query':
-            total, apps, table = query_record(session,
-                                              offset=msg.get('offset', 0),
-                                              limit=msg.get('limit', 10),
-                                              app=msg.get('app', None),
-                                              tags=msg.get('tags', ()),
-                                              before=msg.get('before', None),
-                                              after=msg.get('after', None))
-            await reply(request, (total, apps, table))
-        case 'record_get_tags':
-            record_in_db = session.get(RecordInDB, msg['record_id'])
-            await reply(request, [t.name for t in record_in_db.tags])
-        case 'record_remove_tags':
-            remove_tags(session, msg['record_id'], msg['tags'])
-        case 'record_add_tags':
-            update_tags(session, msg['record_id'], msg['tags'], True)
-        case 'record_replace_tags':
-            update_tags(session, msg['record_id'], msg['tags'], False)
-        case _:
-            logger.error(f"Unknown method: {msg['method']}")
-
-
-async def _handle(session: Session, request: Request, datapath: Path):
-    try:
-        await handle(session, request, datapath)
-    except:
-        await reply(request, 'error')
-
-
-async def serv(port,
-               datapath,
-               url=None,
-               buffer_size=1024 * 1024 * 1024,
-               interval=60):
-    logger.info('Server starting.')
-    async with ZMQContextManager(zmq.ROUTER, bind=f"tcp://*:{port}") as sock:
-        if url is None:
-            url = 'sqlite:///' + str(datapath / 'data.db')
-        engine = create_engine(url)
-        create_tables(engine)
-        Session = sessionmaker(engine)
-        with Session() as session:
-            logger.info('Server started.')
-            received = 0
-            last_flush_time = time.time()
-            while True:
-                identity, msg = await sock.recv_multipart()
-                received += len(msg)
-                req = Request(sock, identity, msg)
-                asyncio.create_task(_handle(session, req, datapath))
-                if received > buffer_size or time.time(
-                ) - last_flush_time > interval:
-                    flush_cache()
-                    received = 0
-                    last_flush_time = time.time()
-
-
-async def watch(port, datapath, url=None, timeout=1, buffer=1024, interval=60):
-    with ZMQContextManager(zmq.DEALER,
-                           connect=f"tcp://127.0.0.1:{port}") as sock:
-        sock.setsockopt(zmq.LINGER, 0)
-        while True:
-            try:
-                sock.send_pyobj({"method": "ping"})
-                if sock.poll(int(1000 * timeout)):
-                    sock.recv()
-                else:
-                    raise asyncio.TimeoutError()
-            except (zmq.error.ZMQError, asyncio.TimeoutError):
-                return asyncio.create_task(
-                    serv(port, datapath, url, buffer * 1024 * 1024, interval))
-            await asyncio.sleep(timeout)
-
-
-async def main(port, datapath, url, timeout=1, buffer=1024, interval=60):
-    task = await watch(port=port,
-                       datapath=datapath,
-                       url=url,
-                       timeout=timeout,
-                       buffer=buffer,
-                       interval=interval)
-    await task
-
-
-@click.command()
-@click.option('--port',
-              default=os.getenv('QULAB_RECORD_PORT', 6789),
-              help='Port of the server.')
-@click.option('--datapath', default=datapath, help='Path of the data.')
-@click.option('--url', default=None, help='URL of the database.')
-@click.option('--timeout', default=1, help='Timeout of ping.')
-@click.option('--buffer', default=1024, help='Buffer size (MB).')
-@click.option('--interval',
-              default=60,
-              help='Interval of flush cache, in unit of second.')
-def record(port, datapath, url, timeout, buffer, interval):
-    asyncio.run(main(port, Path(datapath), url, timeout, buffer, interval))
-
-
-if __name__ == "__main__":
-    record()