QuLab 2.5.4__tar.gz → 2.6.0__tar.gz

{qulab-2.5.4 → qulab-2.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: QuLab
-Version: 2.5.4
+Version: 2.6.0
 Summary: contral instruments and manage data
 Author-email: feihoo87 <feihoo87@gmail.com>
 Maintainer-email: feihoo87 <feihoo87@gmail.com>

{qulab-2.5.4 → qulab-2.6.0}/QuLab.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: QuLab
-Version: 2.5.4
+Version: 2.6.0
 Summary: contral instruments and manage data
 Author-email: feihoo87 <feihoo87@gmail.com>
 Maintainer-email: feihoo87 <feihoo87@gmail.com>

{qulab-2.5.4 → qulab-2.6.0}/qulab/executor/cli.py

@@ -102,7 +102,7 @@ def set(key, value, api):
     from . import transform
     if api is not None:
         api = importlib.import_module(api)
-        set_config_api(api.query_config, api.update_config)
+        set_config_api(api.query_config, api.update_config, api.export_config)
     try:
         value = eval(value)
     except:
@@ -124,7 +124,7 @@ def get(key, api):
     from . import transform
     if api is not None:
         api = importlib.import_module(api)
-        set_config_api(api.query_config, api.update_config)
+        set_config_api(api.query_config, api.update_config, api.export_config)
     click.echo(transform.query_config(key))
 
 
@@ -147,7 +147,7 @@ def run(workflow, code, data, api, plot, no_dependents, update):
     )
     if api is not None:
         api = importlib.import_module(api)
-        set_config_api(api.query_config, api.update_config)
+        set_config_api(api.query_config, api.update_config, api.export_config)
     if code is None:
         code = Path.cwd()
     if data is None:
@@ -197,7 +197,7 @@ def maintain(workflow, code, data, api, plot):
     )
     if api is not None:
         api = importlib.import_module(api)
-        set_config_api(api.query_config, api.update_config)
+        set_config_api(api.query_config, api.update_config, api.export_config)
     if code is None:
         code = Path.cwd()
     if data is None:

{qulab-2.5.4 → qulab-2.6.0}/qulab/executor/load.py

@@ -11,6 +11,7 @@ import warnings
 from importlib.util import module_from_spec, spec_from_file_location
 from pathlib import Path
 from types import ModuleType
+from typing import Any
 
 from loguru import logger
 
@@ -107,7 +108,8 @@ def verify_calibrate_method(module: WorkflowType):
 def verify_check_method(module: WorkflowType):
     if not hasattr(module, 'check'):
         warnings.warn(
-            f"Workflow {module.__file__} does not have 'check' function")
+            f"Workflow {module.__file__} does not have 'check' function, it will be set to 'calibrate' function"
+        )
     else:
         if not can_call_without_args(module.check):
             raise AttributeError(
@@ -120,6 +122,93 @@ def verify_check_method(module: WorkflowType):
             )
 
 
+def verify_dependence_key(workflow: str | tuple[str, dict[str, Any]]
+                          | tuple[str, str, dict[str, Any]]):
+    if isinstance(workflow, str):
+        return
+    if not isinstance(workflow, tuple) or len(workflow) not in [2, 3]:
+        raise ValueError(f"Invalid workflow: {workflow}")
+
+    if len(workflow) == 2:
+        file_name, mapping = workflow
+        if not Path(file_name).exists():
+            raise FileNotFoundError(f"File not found: {file_name}")
+    elif len(workflow) == 3:
+        template_path, target_path, mapping = workflow
+        if not Path(template_path).exists():
+            raise FileNotFoundError(f"File not found: {template_path}")
+        if not isinstance(target_path, (Path, str)) or target_path == '':
+            raise ValueError(f"Invalid target_path: {target_path}")
+        if not isinstance(target_path, (Path, str)):
+            raise ValueError(f"Invalid target_path: {target_path}")
+        if Path(target_path).suffix != '.py':
+            raise ValueError(
+                f"Invalid target_path: {target_path}. Only .py file is supported"
+            )
+    else:
+        raise ValueError(f"Invalid workflow: {workflow}")
+
+    if not isinstance(mapping, dict):
+        raise ValueError(f"Invalid mapping: {mapping}")
+
+    for key, value in mapping.items():
+        if not isinstance(key, str):
+            raise ValueError(
+                f"Invalid key: {key}, should be str type and valid identifier")
+        if not key.isidentifier():
+            raise ValueError(f"Invalid key: {key}, should be identifier")
+        try:
+            pickle.dumps(value)
+        except Exception as e:
+            raise ValueError(
+                f"Invalid value: {key}: {value}, should be pickleable") from e
+    return
+
+
+def verify_depends(module: WorkflowType):
+    if not hasattr(module, 'depends'):
+        return
+
+    deps = []
+
+    if callable(module.depends):
+        if not can_call_without_args(module.depends):
+            raise AttributeError(
+                f"Workflow {module.__file__} 'depends' function should not have any parameters"
+            )
+        deps = list(module.depends())
+    elif isinstance(module.depends, (list, tuple)):
+        deps = module.depends
+    else:
+        raise AttributeError(
+            f"Workflow {module.__file__} 'depends' should be a callable or a list"
+        )
+    for workflow in deps:
+        verify_dependence_key(workflow)
+
+
+def verify_entries(module: WorkflowType):
+    if not hasattr(module, 'entries'):
+        return
+
+    deps = []
+
+    if callable(module.entries):
+        if not can_call_without_args(module.entries):
+            raise AttributeError(
+                f"Workflow {module.__file__} 'entries' function should not have any parameters"
+            )
+        deps = list(module.entries())
+    elif isinstance(module.entries, (list, tuple)):
+        deps = module.entries
+    else:
+        raise AttributeError(
+            f"Workflow {module.__file__} 'entries' should be a callable or a list"
+        )
+    for workflow in deps:
+        verify_dependence_key(workflow)
+
+
 def is_workflow(module: ModuleType) -> bool:
     try:
         verify_calibrate_method(module)
@@ -218,14 +307,15 @@ def load_workflow_from_file(file_name: str,
     module.__mtime__ = (base_path / path).stat().st_mtime
 
     if hasattr(module, 'entries'):
+        verify_entries(module)
         return module
 
     if not hasattr(module, '__timeout__'):
         module.__timeout__ = None
 
     if not hasattr(module, 'depends'):
-        module.depends = lambda: [[]]
-
+        module.depends = lambda: []
+    verify_depends(module)
     verify_calibrate_method(module)
     verify_check_method(module)
 
@@ -415,5 +505,7 @@ def make_graph(workflow: WorkflowType, graph: dict, code_path: str | Path):
         for w in get_dependents(workflow, code_path):
             graph[workflow.__workflow_id__].append(w.__workflow_id__)
             make_graph(w, graph=graph, code_path=code_path)
+    if graph[workflow.__workflow_id__] == []:
+        del graph[workflow.__workflow_id__]
 
     return graph

{qulab-2.5.4 → qulab-2.6.0}/qulab/executor/schedule.py

@@ -1,4 +1,5 @@
 import functools
+import pickle
 import uuid
 from datetime import datetime, timedelta
 from pathlib import Path
@@ -8,13 +9,34 @@ from loguru import logger
 from .load import WorkflowType, get_dependents
 from .storage import (Result, find_result, renew_result, revoke_result,
                       save_result)
-from .transform import update_parameters
+from .transform import current_config, update_parameters
 
 
 class CalibrationFailedError(Exception):
     pass
 
 
+def is_pickleable(obj) -> bool:
+    try:
+        pickle.dumps(obj)
+        return True
+    except:
+        return False
+
+
+def veryfy_analyzed_result(result: Result, script: str, method: str):
+    if not isinstance(result, Result):
+        raise TypeError(f'"{script}" : "{method}" must return a Result object')
+    if not is_pickleable(result.parameters):
+        raise TypeError(
+            f'"{script}" : "{method}" return not pickleable data in .parameters'
+        )
+    if not is_pickleable(result.other_infomation):
+        raise TypeError(
+            f'"{script}" : "{method}" return not pickleable data in .other_infomation'
+        )
+
+
 def check_state(workflow: WorkflowType, code_path: str | Path,
                 state_path: str | Path) -> bool:
     """
@@ -73,17 +95,11 @@ def call_analyzer(node,
                   plot=False) -> Result:
     if check:
         result = node.check_analyze(result, history=history)
-        if not isinstance(result, Result):
-            raise TypeError(
-                f'"{node.__workflow_id__}" : "check_analyze" must return a Result object'
-            )
+        veryfy_analyzed_result(result, node.__workflow_id__, "check_analyze")
         result.fully_calibrated = False
     else:
         result = node.analyze(result, history=history)
-        if not isinstance(result, Result):
-            raise TypeError(
-                f'"{node.__workflow_id__}" : "analyze" must return a Result object'
-            )
+        veryfy_analyzed_result(result, node.__workflow_id__, "analyze")
         result.fully_calibrated = True
         if plot:
             call_plot(node, result)
@@ -108,7 +124,8 @@ def check_data(workflow: WorkflowType, code_path: str | Path,
 
     if history is None:
         logger.debug(f'No history found for "{workflow.__workflow_id__}"')
-        result = Result()
+        result = Result(workflow=workflow.__workflow_id__,
+                        config_path=current_config(state_path))
         result.in_spec = False
         result.bad_data = False
         return result
@@ -130,8 +147,13 @@ def check_data(workflow: WorkflowType, code_path: str | Path,
         logger.debug(
             f'Checking "{workflow.__workflow_id__}" with "check" method ...')
         data = workflow.check()
-        result = Result()
-        result.data = data
+        if not is_pickleable(data):
+            raise TypeError(
+                f'"{workflow.__workflow_id__}" : "check" return not pickleable data'
+            )
+        result = Result(workflow=workflow.__workflow_id__,
+                        data=data,
+                        config_path=current_config(state_path))
         #save_result(workflow.__workflow_id__, result, state_path)
 
         logger.debug(f'Checked "{workflow.__workflow_id__}" !')
@@ -155,8 +177,13 @@ def check_data(workflow: WorkflowType, code_path: str | Path,
             f'Checking "{workflow.__workflow_id__}" with "calibrate" method ...'
         )
         data = workflow.calibrate()
-        result = Result()
-        result.data = data
+        if not is_pickleable(data):
+            raise TypeError(
+                f'"{workflow.__workflow_id__}" : "calibrate" return not pickleable data'
+            )
+        result = Result(workflow=workflow.__workflow_id__,
+                        data=data,
+                        config_path=current_config(state_path))
         save_result(workflow.__workflow_id__, result, state_path)
 
         logger.debug(f'Calibrated "{workflow}" !')
@@ -180,8 +207,13 @@ def calibrate(workflow: WorkflowType, code_path: str | Path,
 
     logger.debug(f'Calibrating "{workflow.__workflow_id__}" ...')
     data = workflow.calibrate()
-    result = Result()
-    result.data = data
+    if not is_pickleable(data):
+        raise TypeError(
+            f'"{workflow.__workflow_id__}" : "calibrate" return not pickleable data'
+        )
+    result = Result(workflow=workflow.__workflow_id__,
+                    data=data,
+                    config_path=current_config(state_path))
     save_result(workflow.__workflow_id__, result, state_path)
     logger.debug(f'Calibrated "{workflow.__workflow_id__}" !')
     result = call_analyzer(workflow, result, history, check=False, plot=plot)
@@ -243,7 +275,7 @@ def diagnose(workflow: WorkflowType, code_path: str | Path,
         raise CalibrationFailedError(
             f'"{workflow.__workflow_id__}": All dependents passed, but calibration failed!'
         )
-    update_parameters(result)
+    update_parameters(result, state_path)
     return True
 
 
@@ -299,7 +331,7 @@ def maintain(workflow: WorkflowType,
             f'"{workflow.__workflow_id__}": All dependents passed, but calibration failed!'
         )
     if update:
-        update_parameters(result)
+        update_parameters(result, state_path)
     return
 
 
@@ -321,5 +353,5 @@ def run(workflow: WorkflowType,
             f'"{workflow.__workflow_id__}": All dependents passed, but calibration failed!'
         )
     if update:
-        update_parameters(result)
+        update_parameters(result, state_path)
     return

{qulab-2.5.4 → qulab-2.6.0}/qulab/executor/storage.py

@@ -3,6 +3,7 @@ import pickle
 import uuid
 from dataclasses import dataclass, field
 from datetime import datetime, timedelta
+from functools import lru_cache
 from pathlib import Path
 from typing import Any, Literal
 
@@ -13,6 +14,7 @@ from ..cli.config import get_config_value
 
 @dataclass
 class Result():
+    workflow: str = ''
     in_spec: bool = False
     bad_data: bool = False
     fully_calibrated: bool = False
@@ -25,6 +27,7 @@ class Result():
     index: int = -1
     previous_path: Path | None = None
     base_path: Path | None = None
+    config_path: Path | None = None
 
     @property
     def previous(self):
@@ -45,6 +48,14 @@ class Result():
                 state = 'Outdated'
         return state
 
+    @property
+    def config(self):
+        from . import transform
+        if self.config_path is not None and self.base_path is not None:
+            return transform._load_config(self.config_path, self.base_path)
+        else:
+            return None
+
     @state.setter
     def state(self, state: Literal['OK', 'Bad', 'Outdated', 'In spec',
                                    'Out of spec', 'Bad data']):
@@ -181,36 +192,42 @@ def get_heads(base_path: str | Path) -> Path | None:
 def create_index(name: str,
                  base_path: str | Path,
                  context: str,
-                 width: int,
+                 width: int = -1,
                  start: int = 0):
-    path = Path(base_path) / "index" / f"{name}.seq"
-    if path.exists():
-        with open(path, "r") as f:
-            index = int(f.read())
+
+    path = Path(base_path) / "index" / name
+    if width == -1:
+        width = len(context)
     else:
-        index = start
-    path.parent.mkdir(parents=True, exist_ok=True)
-    with open(path, "w") as f:
-        f.write(str(index + 1))
+        width = max(width, len(context))
 
-    path = Path(base_path) / "index" / f"{name}.width"
-    with open(path, "w") as f:
-        f.write(str(width))
+    if path.with_suffix('.width').exists():
+        width = int(path.with_suffix('.width').read_text())
+        index = int(path.with_suffix('.seq').read_text())
+    else:
+        index = start
+    if width < len(context):
+        raise ValueError(
+            f"Context '{context}' is too long, existing width of '{name}' is {width}."
+        )
+    if not path.with_suffix('.width').exists():
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.with_suffix('.width').write_text(str(width))
 
-    path = Path(base_path) / "index" / f"{name}.idx"
-    with open(path, "a") as f:
+    path.with_suffix('.seq').write_text(str(index + 1))
 
+    with path.with_suffix('.idx').open("a") as f:
         f.write(f"{context.ljust(width)}\n")
 
     return index
 
 
+@lru_cache(maxsize=4096)
 def query_index(name: str, base_path: str | Path, index: int):
-    path = Path(base_path) / "index" / f"{name}.width"
-    with open(path, "r") as f:
-        width = int(f.read())
-    path = Path(base_path) / "index" / f"{name}.idx"
-    with open(path, "r") as f:
+    path = Path(base_path) / "index" / name
+    width = int(path.with_suffix('.width').read_text())
+
+    with path.with_suffix('.idx').open("r") as f:
         f.seek(index * (width + 1))
         context = f.read(width)
     return context.rstrip()
@@ -219,5 +236,8 @@ def query_index(name: str, base_path: str | Path, index: int):
 def get_result_by_index(
     index: int, base_path: str | Path = get_config_value("data", Path)
 ) -> Result | None:
-    path = query_index("result", base_path, index)
-    return load_result(path, base_path)
+    try:
+        path = query_index("result", base_path, index)
+        return load_result(path, base_path)
+    except:
+        return None

qulab-2.6.0/qulab/executor/transform.py

@@ -0,0 +1,122 @@
+import functools
+import hashlib
+import lzma
+import pickle
+from pathlib import Path
+
+from .storage import Result
+
+__config_id = None
+
+
+def _query_config(name: str, default=None):
+    import pickle
+
+    try:
+        with open('parameters.pkl', 'rb') as f:
+            parameters = pickle.load(f)
+    except:
+        parameters = {}
+
+    return parameters.get(name, default)
+
+
+def _update_config(updates):
+    import pickle
+
+    try:
+        with open('parameters.pkl', 'rb') as f:
+            parameters = pickle.load(f)
+    except:
+        parameters = {}
+
+    for k, v in updates.items():
+        parameters[k] = v
+
+    with open('parameters.pkl', 'wb') as f:
+        pickle.dump(parameters, f)
+
+
+def _export_config() -> dict:
+    import pickle
+
+    try:
+        with open('parameters.pkl', 'rb') as f:
+            parameters = pickle.load(f)
+    except:
+        parameters = {}
+
+    return parameters
+
+
+def update_parameters(result: Result, data_path):
+    global __config_id
+    update_config(result.parameters)
+    cfg = export_config()
+    __config_id = _save_config(cfg, data_path)
+
+
+def current_config(data_path):
+    global __config_id
+    if __config_id is None:
+        cfg = export_config()
+        __config_id = _save_config(cfg, data_path)
+    return __config_id
+
+
+def _save_config(cfg, data_path):
+    global __config_id
+    i = 0
+    buf = pickle.dumps(cfg)
+    buf = lzma.compress(buf)
+    h = hashlib.md5(buf)
+
+    while True:
+        salt = f"{i:08d}".encode()
+        h.update(salt)
+        hashstr = h.hexdigest()
+        cfg_id = Path(hashstr[:2]) / hashstr[2:4] / hashstr[4:]
+        path = Path(data_path) / 'config' / cfg_id
+        if not path.exists():
+            path.parent.mkdir(parents=True, exist_ok=True)
+            with open(path, 'wb') as f:
+                f.write(buf)
+            break
+        elif path.read_bytes() == buf:
+            break
+        i += 1
+    __config_id = str(cfg_id)
+    return __config_id
+
+
+@functools.lru_cache(maxsize=1024)
+def _load_config(id, data_path):
+    path = Path(data_path) / 'config' / id
+    with open(path, 'rb') as f:
+        buf = f.read()
+    cfg = pickle.loads(lzma.decompress(buf))
+    return cfg
+
+
+query_config = _query_config
+update_config = _update_config
+export_config = _export_config
+
+
+def set_config_api(query_method, update_method, export_method):
+    """
+    Set the query and update methods for the config.
+
+    Args:
+        query_method: The query method.
+            the method should take a key and return the value.
+        update_method: The update method.
+            the method should take a dict of updates.
+    """
+    global query_config, update_config, export_config
+
+    query_config = query_method
+    update_config = update_method
+    export_config = export_method
+
+    return query_config, update_config, export_config

{qulab-2.5.4 → qulab-2.6.0}/qulab/executor/utils.py

@@ -1,5 +1,6 @@
 from pathlib import Path
 
+from ..cli.config import get_config_value
 from .load import load_workflow
 
 
@@ -133,3 +134,23 @@ def check_analyze(result: Result, history: Result | None = None) -> Result:
 
     return result
 """
+
+
+def debug_analyze(
+        result_index: int,
+        code_path: str | Path = get_config_value('code_path', Path),
+        data_path: str | Path = get_config_value('data_path', Path),
+) -> None:
+    from .storage import get_result_by_index
+
+    result = get_result_by_index(result_index, data_path)
+    if result is None:
+        raise ValueError(f'Invalid result index: {result_index}')
+    workflow = result.workflow
+    wf = load_workflow(workflow, code_path)
+    if wf is None:
+        raise ValueError(f'Invalid workflow: {workflow}')
+    result = wf.analyze(result)
+    if hasattr(wf, 'plot'):
+        wf.plot(result)
+    return result

qulab-2.6.0/qulab/version.py

@@ -0,0 +1 @@
+__version__ = "2.6.0"

qulab-2.5.4/qulab/executor/transform.py

@@ -1,55 +0,0 @@
-from .storage import Result
-
-
-def _query_config(name: str, default=None):
-    import pickle
-
-    try:
-        with open('parameters.pkl', 'rb') as f:
-            parameters = pickle.load(f)
-    except:
-        parameters = {}
-
-    return parameters.get(name, default)
-
-
-def _update_config(updates):
-    import pickle
-
-    try:
-        with open('parameters.pkl', 'rb') as f:
-            parameters = pickle.load(f)
-    except:
-        parameters = {}
-
-    for k, v in updates.items():
-        parameters[k] = v
-
-    with open('parameters.pkl', 'wb') as f:
-        pickle.dump(parameters, f)
-
-
-def update_parameters(result: Result):
-    update_config(result.parameters)
-
-
-query_config = _query_config
-update_config = _update_config
-
-
-def set_config_api(query_method, update_method):
-    """
-    Set the query and update methods for the config.
-
-    Args:
-        query_method: The query method.
-            the method should take a key and return the value.
-        update_method: The update method.
-            the method should take a dict of updates.
-    """
-    global query_config, update_config
-
-    query_config = query_method
-    update_config = update_method
-
-    return query_config, update_config

qulab-2.5.4/qulab/version.py

@@ -1 +0,0 @@
-__version__ = "2.5.4"