QuLab 2.7.12__tar.gz → 2.7.14__tar.gz

{qulab-2.7.12 → qulab-2.7.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: QuLab
-Version: 2.7.12
+Version: 2.7.14
 Summary: contral instruments and manage data
 Author-email: feihoo87 <feihoo87@gmail.com>
 Maintainer-email: feihoo87 <feihoo87@gmail.com>

{qulab-2.7.12 → qulab-2.7.14}/QuLab.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: QuLab
-Version: 2.7.12
+Version: 2.7.14
 Summary: contral instruments and manage data
 Author-email: feihoo87 <feihoo87@gmail.com>
 Maintainer-email: feihoo87 <feihoo87@gmail.com>

{qulab-2.7.12 → qulab-2.7.14}/QuLab.egg-info/SOURCES.txt

@@ -23,6 +23,7 @@ qulab/executor/cli.py
 qulab/executor/load.py
 qulab/executor/schedule.py
 qulab/executor/storage.py
+qulab/executor/template.py
 qulab/executor/transform.py
 qulab/executor/utils.py
 qulab/monitor/__init__.py
@@ -95,6 +96,8 @@ qulab/sys/rpc/socket.py
 qulab/sys/rpc/utils.py
 qulab/sys/rpc/worker.py
 qulab/sys/rpc/zmq_socket.py
+qulab/tools/__init__.py
+qulab/tools/connection_helper.py
 qulab/visualization/__init__.py
 qulab/visualization/__main__.py
 qulab/visualization/_autoplot.py

{qulab-2.7.12 → qulab-2.7.14}/qulab/executor/cli.py

@@ -161,6 +161,15 @@ def get(key, api):
 def run(workflow, code, data, api, plot, no_dependents, retry, freeze):
     """
     Run a workflow.
+
+    If the workflow has entries, run all entries.
+    If `--no-dependents` is set, only run the workflow itself.
+    If `--retry` is set, retry the workflow when calibration failed.
+    If `--freeze` is set, freeze the config table.
+    If `--plot` is set, plot the report.
+    If `--api` is set, use the api to get and update the config table.
+    If `--code` is not set, use the current working directory.
+    If `--data` is not set, use the `logs` directory in the code path.
     """
     logger.info(
         f'[CMD]: run {workflow} --code {code} --data {data} --api {api}'
@@ -227,6 +236,13 @@ def run(workflow, code, data, api, plot, no_dependents, retry, freeze):
 def maintain(workflow, code, data, api, retry, plot):
     """
     Maintain a workflow.
+
+    If the workflow has entries, run all entries.
+    If `--retry` is set, retry the workflow when calibration failed.
+    If `--plot` is set, plot the report.
+    If `--api` is set, use the api to get and update the config table.
+    If `--code` is not set, use the current working directory.
+    If `--data` is not set, use the `logs` directory in the code path.
     """
     logger.info(
         f'[CMD]: maintain {workflow} --code {code} --data {data} --api {api}'

{qulab-2.7.12 → qulab-2.7.14}/qulab/executor/load.py

@@ -1,12 +1,6 @@
-import base64
 import graphlib
-import hashlib
 import inspect
-import lzma
 import pickle
-import re
-import string
-import textwrap
 import warnings
 from importlib.util import module_from_spec, spec_from_file_location
 from pathlib import Path
@@ -16,6 +10,8 @@ from typing import Any
 from loguru import logger
 
 from .storage import Report
+from .template import (TemplateKeyError, TemplateTypeError, decode_mapping,
+                       inject_mapping)
 
 
 class SetConfigWorkflow():
@@ -330,22 +326,6 @@ def load_workflow_from_file(file_name: str,
     return module
 
 
-def encode_mapping(mapping):
-    mapping_bytes = lzma.compress(pickle.dumps(mapping))
-    hash_str = hashlib.md5(mapping_bytes).hexdigest()[:8]
-    mappping_code = '\n'.join(
-        textwrap.wrap(base64.b64encode(mapping_bytes).decode(), 100))
-    return hash_str, mappping_code
-
-
-def decode_mapping(hash_str, mappping_code):
-    mapping_bytes = base64.b64decode(mappping_code.replace('\n', ''))
-    if hash_str != hashlib.md5(mapping_bytes).hexdigest()[:8]:
-        raise ValueError("Hash does not match")
-    mapping = pickle.loads(lzma.decompress(mapping_bytes))
-    return mapping
-
-
 def load_workflow_from_template(template_path: str,
                                 mapping: dict[str, str],
                                 base_path: str | Path,
@@ -359,36 +339,9 @@ def load_workflow_from_template(template_path: str,
         content = f.read()
 
     mtime = max((base_path / template_path).stat().st_mtime, mtime)
-    hash_str, mapping_code = encode_mapping(mapping)
-
-    def replace(text):
-        """
-        将给定文本中的所有 VAR("var") 替换为 __VAR_{hash_str}["var"]。
-        
-        Args:
-            text (str): 包含 VAR 调用的字符串。
-        
-        Returns:
-            str: 已经替换的新字符串。
-        """
-        pattern = re.compile(r'VAR\s*\(\s*(["\'])(\w+)\1\s*\)')
-        replacement = f'__VAR_{hash_str}' + r'[\1\2\1]'
-        new_text = re.sub(pattern, replacement, text)
-        return new_text
-
-    template = string.Template(replace(content))
-    keys = template.get_identifiers()
-    missing = set(keys) - set(mapping.keys())
-    if missing:
-        raise KeyError(f"{template_path}: Missing keys in mapping: {missing}")
-    content = template.substitute(mapping)
-
-    inject_code = [
-        "from qulab.executor.load import decode_mapping",
-        f"__VAR_{hash_str} = decode_mapping(\"{hash_str}\",",
-        f"\"\"\"{mapping_code}\"\"\")"
-    ]
-    content = '\n'.join(inject_code + [content])
+
+    content, hash_str = inject_mapping(content, mapping, str(path))
+
     if target_path is None:
         if path.stem == 'template':
             path = path.parent / f'tmp{hash_str}.py'
@@ -424,16 +377,27 @@ def load_workflow_from_template(template_path: str,
 def load_workflow(workflow: str | tuple[str, dict],
                   base_path: str | Path,
                   package='workflows',
-                  mtime: float = 0) -> WorkflowType:
+                  mtime: float = 0,
+                  inject: dict | None = None) -> WorkflowType:
     if isinstance(workflow, tuple):
         if len(workflow) == 2:
             file_name, mapping = workflow
-            w = load_workflow_from_template(file_name, mapping, base_path,
-                                            None, package, mtime)
+            if inject is None:
+                w = load_workflow_from_template(file_name, mapping, base_path,
+                                                None, package, mtime)
+            else:
+                w = load_workflow_from_template(file_name, inject, base_path,
+                                                None, package, mtime)
         elif len(workflow) == 3:
             template_path, target_path, mapping = workflow
-            w = load_workflow_from_template(template_path, mapping, base_path,
-                                            target_path, package, mtime)
+            if inject is None:
+                w = load_workflow_from_template(template_path, mapping,
+                                                base_path, target_path,
+                                                package, mtime)
+            else:
+                w = load_workflow_from_template(template_path, inject,
+                                                base_path, target_path,
+                                                package, mtime)
         else:
             raise ValueError(f"Invalid workflow: {workflow}")
         w.__workflow_id__ = str(Path(w.__file__).relative_to(base_path))
@@ -451,6 +415,22 @@ def load_workflow(workflow: str | tuple[str, dict],
     return w
 
 
+def _load_workflow_list(workflow, lst, code_path):
+    ret = []
+    for i, n in enumerate(lst):
+        try:
+            ret.append(load_workflow(n, code_path, mtime=workflow.__mtime__))
+        except TemplateKeyError:
+            raise TemplateKeyError(
+                f"Workflow {workflow.__workflow_id__} missing key in {i}th {n[0]} dependent mapping."
+            )
+        except TemplateTypeError:
+            raise TemplateTypeError(
+                f"Workflow {workflow.__workflow_id__} type error in {i}th {n[0]} dependent mapping."
+            )
+    return ret
+
+
 def get_dependents(workflow: WorkflowType,
                    code_path: str | Path) -> list[WorkflowType]:
     if callable(getattr(workflow, 'depends', None)):
@@ -458,16 +438,10 @@ def get_dependents(workflow: WorkflowType,
             raise AttributeError(
                 f'Workflow {workflow.__workflow_id__} "depends" function should not have any parameters'
             )
-        return [
-            load_workflow(n, code_path, mtime=workflow.__mtime__)
-            for n in workflow.depends()
-        ]
+        return _load_workflow_list(workflow, workflow.depends(), code_path)
     elif isinstance(getattr(workflow, 'depends', None), (list, tuple)):
-        return [
-            load_workflow(n, code_path, mtime=workflow.__mtime__)
-            for n in workflow.depends
-        ]
-    elif getattr(workflow, 'entries', None) is None:
+        return _load_workflow_list(workflow, workflow.depends, code_path)
+    elif getattr(workflow, 'depends', None) is None:
         return []
     else:
         raise AttributeError(
@@ -482,15 +456,9 @@ def get_entries(workflow: WorkflowType,
             raise AttributeError(
                 f'Workflow {workflow.__workflow_id__} "entries" function should not have any parameters'
             )
-        return [
-            load_workflow(n, code_path, mtime=workflow.__mtime__)
-            for n in workflow.entries()
-        ]
+        return _load_workflow_list(workflow, workflow.entries(), code_path)
     elif isinstance(getattr(workflow, 'entries', None), (list, tuple)):
-        return [
-            load_workflow(n, code_path, mtime=workflow.__mtime__)
-            for n in workflow.entries
-        ]
+        return _load_workflow_list(workflow, workflow.entries, code_path)
     elif getattr(workflow, 'entries', None) is None:
         return []
     else:

{qulab-2.7.12 → qulab-2.7.14}/qulab/executor/schedule.py

@@ -1,4 +1,5 @@
 import functools
+import inspect
 import pickle
 import uuid
 from datetime import datetime, timedelta
@@ -231,21 +232,7 @@ def call_analyzer(node: WorkflowType,
     if hasattr(node, 'oracle') and callable(node.oracle):
         logger.debug(
             f'"{node.__workflow_id__}" has oracle method, calling ...')
-        try:
-            report = node.oracle(report,
-                                 history=history,
-                                 system_state=get_heads(report.base_path))
-        except Exception as e:
-            logger.exception(e)
-            report.oracle = {}
-        if not isinstance(report, Report):
-            raise TypeError(
-                f'"{node.__workflow_id__}" : function "oracle" must return a Report object'
-            )
-        if not is_pickleable(report.oracle):
-            raise TypeError(
-                f'"{node.__workflow_id__}" : function "oracle" return not pickleable data'
-            )
+        report = call_oracle(node, report, history)
     report.fully_calibrated = True
     save_report(node.__workflow_id__, report, state_path, overwrite=True)
     if plot:
@@ -253,6 +240,35 @@ def call_analyzer(node: WorkflowType,
     return report
 
 
+def call_oracle(node: WorkflowType, report: Report, history: Report | None):
+    sig = inspect.signature(node.oracle)
+    try:
+        if 'history' in sig.parameters and 'system_state' in sig.parameters:
+            report = node.oracle(report,
+                                 history=history,
+                                 system_state=get_heads(report.base_path))
+        elif 'history' in sig.parameters:
+            report = node.oracle(report, history=history)
+        elif 'system_state' in sig.parameters:
+            report = node.oracle(report,
+                                 system_state=get_heads(report.base_path))
+        else:
+            report = node.oracle(report)
+    except Exception as e:
+        logger.exception(e)
+        report.oracle = {}
+        return report
+    if not isinstance(report, Report):
+        raise TypeError(
+            f'"{node.__workflow_id__}" : function "oracle" must return a Report object'
+        )
+    if not is_pickleable(report.oracle):
+        raise TypeError(
+            f'"{node.__workflow_id__}" : function "oracle" return not pickleable data'
+        )
+    return report
+
+
 def check_data(workflow: WorkflowType, state_path: str | Path, plot: bool,
                session_id: str) -> Report:
     """

qulab-2.7.14/qulab/executor/template.py

@@ -0,0 +1,173 @@
+import ast
+import base64
+import hashlib
+import lzma
+import pickle
+import re
+import string
+import textwrap
+from typing import Any
+
+
+def encode_mapping(mapping):
+    mapping_bytes = lzma.compress(pickle.dumps(mapping))
+    hash_str = hashlib.md5(mapping_bytes).hexdigest()[:8]
+    mappping_code = '\n'.join(
+        textwrap.wrap(base64.b64encode(mapping_bytes).decode(),
+                      90,
+                      initial_indent='    ',
+                      subsequent_indent='    '))
+    return hash_str, mappping_code
+
+
+def decode_mapping(hash_str, mappping_code):
+    mapping_bytes = base64.b64decode(mappping_code)
+    if hash_str != hashlib.md5(mapping_bytes).hexdigest()[:8]:
+        raise ValueError("Hash does not match")
+    mapping = pickle.loads(lzma.decompress(mapping_bytes))
+    return mapping
+
+
+class TemplateTypeError(TypeError):
+    pass
+
+
+class TemplateKeyError(KeyError):
+    pass
+
+
+class TemplateVarExtractor(ast.NodeVisitor):
+
+    def __init__(self, fname, mapping):
+        self.var_func_def = (0, 0)
+        self.variables = set()
+        self.str_variables = set()
+        self.replacements = {}
+        self.fname = fname
+        self.mapping = mapping
+
+    def visit_Constant(self, node):
+        if isinstance(node.value, str):
+            self._process_string(node.value, node.lineno, node.col_offset,
+                                 node.end_lineno, node.end_col_offset)
+
+    def visit_JoinedStr(self, node):
+        for value in node.values:
+            if isinstance(value, ast.Constant) and isinstance(
+                    value.value, str):
+                self._process_string(value.value, value.lineno,
+                                     value.col_offset, value.end_lineno,
+                                     value.end_col_offset)
+        self.generic_visit(node)
+
+    def visit_FunctionDef(self, node):
+        if node.name == 'VAR':
+            self.var_func_def = (node.lineno, node.end_lineno)
+        self.generic_visit(node)
+
+    def visit_Call(self, node):
+        if isinstance(node.func, ast.Name) and node.func.id == 'VAR':
+            arg = node.args[0]
+            if isinstance(arg, ast.Constant) and isinstance(arg.value, str):
+                if arg.value not in self.mapping:
+                    raise TemplateKeyError(
+                        f"The variable '{arg.value}' is not provided in mapping. {self.fname}:{node.lineno}"
+                    )
+                self.variables.add(arg.value)
+                # new_node = ast.Subscript(value=ast.Name(id="__VAR",
+                #                                         ctx=ast.Load()),
+                #                          slice=ast.Constant(value=arg.value),
+                #                          ctx=ast.Load())
+                # ast.fix_missing_locations(new_node)
+                # new_source = ast.unparse(new_node)
+                self.replacements[(node.lineno, node.end_lineno,
+                                   node.col_offset,
+                                   node.end_col_offset)] = ('VAR', arg.value,
+                                                            None, None)
+            else:
+                raise SyntaxError(
+                    f"Argument of VAR function must be a string. {self.fname}:{node.lineno}"
+                )
+        self.generic_visit(node)
+
+    def _process_string(self, s: str, lineno: int, col_offset: int,
+                        end_lineno: int, end_col_offset: int):
+        """解析字符串内容，提取模板变量"""
+        lines = s.split('\n')
+        for offset, line in enumerate(lines):
+            current_lineno = lineno + offset
+            template = string.Template(line)
+            for var_name in template.get_identifiers():
+                if var_name not in self.mapping:
+                    raise TemplateKeyError(
+                        f"The variable '{var_name}' is not provided in mapping. {self.fname}:{current_lineno}"
+                    )
+                if not isinstance(self.mapping[var_name], str):
+                    raise TemplateTypeError(
+                        f"Mapping value for '{var_name}' must be a string. {self.fname}:{current_lineno}"
+                    )
+                self.str_variables.add(var_name)
+                start, stop = 0, len(line)
+                if current_lineno == lineno:
+                    start = col_offset
+                if current_lineno == end_lineno:
+                    stop = end_col_offset
+                self.replacements[(current_lineno, current_lineno, start,
+                                   stop)] = ('STR', var_name, None, None)
+
+
+def inject_mapping(source: str, mapping: dict[str, Any],
+                   fname: str) -> list[tuple[str, int]]:
+    hash_str, mapping_code = encode_mapping(mapping)
+
+    tree = ast.parse(source)
+    lines = source.splitlines()
+    extractor = TemplateVarExtractor(fname, mapping)
+    extractor.visit(tree)
+
+    # remove VAR function definition
+    if extractor.var_func_def != (0, 0):
+        for i in range(extractor.var_func_def[0] - 1,
+                       extractor.var_func_def[1]):
+            lines[i] = ''
+
+    for (lineno, end_lineno, col_offset,
+         end_col_offset), (kind, name, old_source,
+                           new_source) in extractor.replacements.items():
+        head = lines[lineno - 1][:col_offset]
+        tail = lines[end_lineno - 1][end_col_offset:]
+        content = lines[lineno - 1:end_lineno]
+        content[0] = content[0].removeprefix(head)
+        content[-1] = content[-1].removesuffix(tail)
+        content = '\n'.join(content)
+
+        if kind == 'STR':
+            template = string.Template(content)
+            formated_lines = template.substitute(mapping).splitlines()
+            formated_lines[0] = head + formated_lines[0]
+            formated_lines[-1] = formated_lines[-1] + tail
+            if len(formated_lines) == 1:
+                lines[lineno - 1] = formated_lines[0]
+            else:
+                lines[lineno - 1:end_lineno] = formated_lines
+        else:
+            pattern = re.compile(r'VAR\s*\(\s*(["\'])(\w+)\1\s*\)')
+            replacement = f'__VAR_{hash_str}' + r'[\1\2\1]'
+            new_content = re.sub(pattern, replacement, content)
+
+            if lineno == end_lineno:
+                lines[lineno - 1] = head + new_content + tail
+            else:
+                lines[lineno - 1] = head + new_content[:-1]
+                for i in range(lineno, end_lineno - 1):
+                    lines[i] = ''
+                lines[end_lineno - 1] = ']' + tail
+
+    injected_code = '\n'.join([
+        f"__QULAB_TEMPLATE__ = \"{fname}\"",
+        f"from qulab.executor.template import decode_mapping as __decode_{hash_str}",
+        f"__VAR_{hash_str} = __decode_{hash_str}(\"{hash_str}\", \"\"\"",
+        mapping_code, "    \"\"\")", *lines
+    ])
+
+    return injected_code, hash_str

{qulab-2.7.12 → qulab-2.7.14}/qulab/executor/transform.py

@@ -79,6 +79,8 @@ def set_config_api(query_method, update_method, export_method):
             the method should take a key and return the value.
         update_method: The update method.
             the method should take a dict of updates.
+        export_method: The export method.
+            the method should return a dict of the config.
     """
     global query_config, update_config, export_config
 

{qulab-2.7.12 → qulab-2.7.14}/qulab/executor/utils.py

@@ -169,6 +169,18 @@ def debug_analyze(
     wf = load_workflow(workflow, code_path)
     if wf is None:
         raise ValueError(f'Invalid workflow: {workflow}')
+    if hasattr(wf, '__QULAB_TEMPLATE__'):
+        template_mtime = (Path(code_path) / wf.__QULAB_TEMPLATE__).stat().st_mtime
+        if template_mtime > wf.__mtime__:
+            for k in dir(wf):
+                if k.startswith('__VAR_') and len(k) == len('__VAR_17fb4dde'):
+                    var_dict = getattr(wf, k)
+                    break
+            else:
+                var_dict = {}
+            wf = load_workflow((wf.__QULAB_TEMPLATE__, workflow, var_dict),
+                               code_path)
+
     report = wf.analyze(report, report.previous)
     if hasattr(wf, 'plot'):
         wf.plot(report)

qulab-2.7.14/qulab/tools/connection_helper.py

@@ -0,0 +1,39 @@
+class _Missing():
+
+    def __repr__(self):
+        return "Missing"
+
+
+Missing = _Missing()
+
+
+def connect_trace(*mappings):
+    if not mappings:
+        return {}
+    result = {}
+    first_mapping = mappings[0]
+    for key in first_mapping:
+        current_value = key
+        trajectory = []
+        for mapping in mappings:
+            if current_value is Missing:
+                trajectory.append(Missing)
+                continue
+            if current_value in mapping:
+                next_value = mapping[current_value]
+                trajectory.append(next_value)
+                current_value = next_value
+            else:
+                trajectory.append(Missing)
+                current_value = Missing
+        result[key] = trajectory
+    return result
+
+
+def connect(*mappings):
+    if not mappings:
+        return {}
+    return {
+        k: v[-1]
+        for k, v in connect_trace(*mappings).items() if v[-1] is not Missing
+    }

qulab-2.7.14/qulab/version.py

@@ -0,0 +1 @@
+__version__ = "2.7.14"

qulab-2.7.12/qulab/version.py

@@ -1 +0,0 @@
-__version__ = "2.7.12"