QuLab 2.1.1__tar.gz → 2.1.2__tar.gz

{qulab-2.1.1 → qulab-2.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: QuLab
-Version: 2.1.1
+Version: 2.1.2
 Summary: contral instruments and manage data
 Author-email: feihoo87 <feihoo87@gmail.com>
 Maintainer-email: feihoo87 <feihoo87@gmail.com>

{qulab-2.1.1 → qulab-2.1.2}/QuLab.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: QuLab
-Version: 2.1.1
+Version: 2.1.2
 Summary: contral instruments and manage data
 Author-email: feihoo87 <feihoo87@gmail.com>
 Maintainer-email: feihoo87 <feihoo87@gmail.com>

qulab-2.1.2/qulab/__init__.py

@@ -0,0 +1,3 @@
+from .scan import Scan, get_record, load_record, lookup, lookup_list
+from .version import __version__
+from .visualization import autoplot

{qulab-2.1.1 → qulab-2.1.2}/qulab/scan/__init__.py

@@ -1,3 +1,3 @@
 from .expression import Expression, Symbol
-from .query import get_record, lookup, lookup_list
+from .query import get_record, load_record, lookup, lookup_list
 from .scan import Scan

{qulab-2.1.1 → qulab-2.1.2}/qulab/scan/query.py

@@ -6,10 +6,13 @@ import dill
 import ipywidgets as widgets
 import zmq
 from IPython.display import display
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
 
 from qulab.sys.rpc.zmq_socket import ZMQContextManager
 
 from .record import Record
+from .recorder import get_local_record
 from .scan import default_server
 
 
@@ -26,18 +29,15 @@ def get_record(id, database=default_server) -> Record:
             d._file = None
             return d
     else:
-        from .models import Record as RecordInDB
-        from .models import create_engine, sessionmaker
-
         db_file = Path(database) / 'data.db'
         engine = create_engine(f'sqlite:///{db_file}')
         Session = sessionmaker(bind=engine)
         with Session() as session:
-            path = Path(database) / 'objects' / session.get(RecordInDB,
-                                                            id).file
-            with open(path, 'rb') as f:
-                record = dill.load(f)
-            return record
+            return get_local_record(session, id, database)
+
+
+def load_record(file):
+    return Record.load(file)
 
 
 def _format_tag(tag):

{qulab-2.1.1 → qulab-2.1.2}/qulab/scan/record.py

@@ -1,6 +1,7 @@
 import itertools
 import sys
 import uuid
+import zipfile
 from pathlib import Path
 from threading import Lock
 from types import EllipsisType
@@ -13,7 +14,7 @@ from qulab.sys.rpc.zmq_socket import ZMQContextManager
 
 from .space import OptimizeSpace, Space
 
-_notgiven = object()
+_not_given = object()
 
 
 def random_path(base):
@@ -120,6 +121,18 @@ class BufferList():
                             yield pos, value
                         except EOFError:
                             break
+        elif isinstance(
+                self.file, tuple) and len(self.file) == 2 and isinstance(
+                    self.file[0], str) and self.file[0].endswith('.zip'):
+            f, name = self.file
+            with zipfile.ZipFile(f, 'r') as z:
+                with z.open(name, 'r') as f:
+                    while True:
+                        try:
+                            pos, value = dill.load(f)
+                            yield pos, value
+                        except EOFError:
+                            break
 
     def iter(self):
         if self._data_id is None:
@@ -327,7 +340,7 @@ class Record():
             ret = ret.toarray()
         return ret
 
-    def get(self, key, default=_notgiven, buffer_to_array=False, slice=None):
+    def get(self, key, default=_not_given, buffer_to_array=False, slice=None):
         if self.is_remote_record():
             with ZMQContextManager(zmq.DEALER,
                                    connect=self.database) as socket:
@@ -355,7 +368,7 @@ class Record():
                 else:
                     return ret
         else:
-            if default is _notgiven:
+            if default is _not_given:
                 d = self._items.get(key)
             else:
                 d = self._items.get(key, default)
@@ -437,6 +450,40 @@ class Record():
         with open(self._file, 'wb') as f:
             dill.dump(self, f)
 
+    def export(self, file):
+        with zipfile.ZipFile(file,
+                             'w',
+                             compression=zipfile.ZIP_DEFLATED,
+                             compresslevel=9) as z:
+            items = {}
+            for key in self.keys():
+                value = self.get(key)
+                if isinstance(value, BufferList):
+                    v = BufferList()
+                    v.lu = value.lu
+                    v.rd = value.rd
+                    v.inner_shape = value.inner_shape
+                    items[key] = v
+                    with z.open(f'{key}.buf', 'w') as f:
+                        for pos, data in value.iter():
+                            dill.dump((pos, data), f)
+                else:
+                    items[key] = value
+            with z.open('record.pkl', 'w') as f:
+                dill.dump((self.description, items), f)
+
+    @classmethod
+    def load(cls, file: str):
+        with zipfile.ZipFile(file, 'r') as z:
+            with z.open('record.pkl', 'r') as f:
+                description, items = dill.load(f)
+            record = cls(None, None, description)
+            for key, value in items.items():
+                if isinstance(value, BufferList):
+                    value.file = file, f'{key}.buf'
+                record._items[key] = value
+        return record
+
     def __repr__(self):
         return f"<Record: id={self.id} app={self.description['app']}, keys={self.keys()}>"
 

{qulab-2.1.1 → qulab-2.1.2}/qulab/scan/recorder.py

@@ -28,6 +28,7 @@ else:
 datapath.mkdir(parents=True, exist_ok=True)
 
 record_cache = {}
+CACHE_SIZE = 1024
 
 
 class Request():
@@ -45,11 +46,11 @@ async def reply(req, resp):
 
 
 def clear_cache():
-    if len(record_cache) < 1024:
+    if len(record_cache) < CACHE_SIZE:
         return
 
     for k, (t, _) in zip(sorted(record_cache.items(), key=lambda x: x[1][0]),
-                         range(len(record_cache) - 1024)):
+                         range(len(record_cache) - CACHE_SIZE)):
         del record_cache[k]
 
 
@@ -58,15 +59,20 @@ def flush_cache():
         r.flush()
 
 
+def get_local_record(session: Session, id: int, datapath: Path) -> Record:
+    record_in_db = session.get(RecordInDB, id)
+    record_in_db.atime = utcnow()
+    path = datapath / 'objects' / record_in_db.file
+    with open(path, 'rb') as f:
+        record = dill.load(f)
+        record.database = datapath
+        record._file = path
+    return record
+
+
 def get_record(session: Session, id: int, datapath: Path) -> Record:
     if id not in record_cache:
-        record_in_db = session.get(RecordInDB, id)
-        record_in_db.atime = utcnow()
-        path = datapath / 'objects' / record_in_db.file
-        with open(path, 'rb') as f:
-            record = dill.load(f)
-            record.database = datapath
-            record._file = path
+        record = get_local_record(session, id, datapath)
     else:
         record = record_cache[id][1]
     clear_cache()

{qulab-2.1.1 → qulab-2.1.2}/qulab/scan/scan.py

@@ -46,6 +46,10 @@ if os.getenv('QULAB_SERVER'):
     default_server = os.getenv('QULAB_SERVER')
 else:
     default_server = f'tcp://127.0.0.1:{default_record_port}'
+if os.getenv('QULAB_EXECUTOR'):
+    default_executor = os.getenv('QULAB_EXECUTOR')
+else:
+    default_executor = default_server
 
 
 def task_uuid():
@@ -136,6 +140,7 @@ class Scan():
             'app': app,
             'tags': tags,
             'loops': {},
+            'intrinsic_loops': {},
             'consts': {},
             'functions': {},
             'getters': {},
@@ -287,10 +292,10 @@ class Scan():
         else:
             return Symbol(name)
 
-    def _add_loop_var(self, name: str, level: int, range):
+    def _add_search_space(self, name: str, level: int, space):
         if level not in self.description['loops']:
             self.description['loops'][level] = []
-        self.description['loops'][level].append((name, range))
+        self.description['loops'][level].append((name, space))
 
     def add_depends(self, name: str, depends: list[str]):
         if isinstance(depends, str):
@@ -299,7 +304,7 @@ class Scan():
             self.description['dependents'][name] = set()
         self.description['dependents'][name].update(depends)
 
-    def add_filter(self, func: Callable, level: int):
+    def add_filter(self, func: Callable, level: int = -1):
         """
         Add a filter function to the scan.
 
@@ -345,26 +350,33 @@ class Scan():
 
     def search(self,
                name: str,
-               range: Iterable | Expression | Callable | OptimizeSpace,
+               space: Iterable | Expression | Callable | OptimizeSpace,
                level: int | None = None,
-               setter: Callable | None = None):
+               setter: Callable | None = None,
+               intrinsic: bool = False):
         if level is not None:
-            assert level >= 0, 'level must be greater than or equal to 0.'
-        if isinstance(range, OptimizeSpace):
-            range.name = name
-            range.optimizer.dimensions[name] = range.space
-            self._add_loop_var(name, range.optimizer.level, range)
-            self.add_depends(range.optimizer.name, [name])
+            if not intrinsic:
+                assert level >= 0, 'level must be greater than or equal to 0.'
+        if intrinsic:
+            assert isinstance(space, (np.ndarray, list, tuple, range, Space)), \
+                'space must be an instance of np.ndarray, list, tuple, range or Space.'
+            self.description['intrinsic_loops'][name] = level
+            self.set(name, space)
+        elif isinstance(space, OptimizeSpace):
+            space.name = name
+            space.optimizer.dimensions[name] = space.space
+            self._add_search_space(name, space.optimizer.level, space)
+            self.add_depends(space.optimizer.name, [name])
         else:
             if level is None:
                 raise ValueError('level must be provided.')
             try:
-                range = Space.fromarray(range)
+                space = Space.fromarray(space)
             except:
                 pass
-            self._add_loop_var(name, level, range)
-            if isinstance(range, Expression) or callable(range):
-                self.add_depends(name, range.symbols())
+            self._add_search_space(name, level, space)
+            if isinstance(space, Expression) or callable(space):
+                self.add_depends(name, space.symbols())
         if setter:
             self.description['setters'][name] = setter
 
@@ -500,7 +512,7 @@ class Scan():
         import asyncio
         self._main_task = asyncio.create_task(self.run())
 
-    async def submit(self, server='tcp://127.0.0.1:6788'):
+    async def submit(self, server=default_executor):
         assymbly(self.description)
         async with ZMQContextManager(zmq.DEALER, connect=server) as socket:
             await socket.send_pyobj({
@@ -740,7 +752,7 @@ def assymbly(description):
                 keys -= set(ready)
 
     axis = {}
-    independent_variables = set()
+    independent_variables = set(description['intrinsic_loops'].keys())
 
     for name in description['consts']:
         axis[name] = ()

qulab-2.1.2/qulab/version.py

@@ -0,0 +1 @@
+__version__ = "2.1.2"

qulab-2.1.1/qulab/__init__.py

@@ -1 +0,0 @@
-from .version import __version__

qulab-2.1.1/qulab/version.py

@@ -1 +0,0 @@
-__version__ = "2.1.1"