QuLab 2.4.9__tar.gz → 2.4.11__tar.gz

{qulab-2.4.9 → qulab-2.4.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: QuLab
-Version: 2.4.9
+Version: 2.4.11
 Summary: contral instruments and manage data
 Author-email: feihoo87 <feihoo87@gmail.com>
 Maintainer-email: feihoo87 <feihoo87@gmail.com>

{qulab-2.4.9 → qulab-2.4.11}/QuLab.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: QuLab
-Version: 2.4.9
+Version: 2.4.11
 Summary: contral instruments and manage data
 Author-email: feihoo87 <feihoo87@gmail.com>
 Maintainer-email: feihoo87 <feihoo87@gmail.com>

{qulab-2.4.9 → qulab-2.4.11}/qulab/__main__.py

@@ -1,6 +1,6 @@
 import click
 
-from .executor.__main__ import create, maintain, run
+from .executor.__main__ import create, get, maintain, run, set
 from .monitor.__main__ import main as monitor
 from .scan.server import server
 from .sys.net.cli import dht
@@ -25,6 +25,8 @@ cli.add_command(server)
 cli.add_command(maintain)
 cli.add_command(run)
 cli.add_command(create)
+cli.add_command(set)
+cli.add_command(get)
 
 if __name__ == '__main__':
     cli()

qulab-2.4.11/qulab/executor/__main__.py

@@ -0,0 +1,200 @@
+import functools
+import importlib
+import os
+import sys
+from pathlib import Path
+
+import click
+from loguru import logger
+
+from .load import find_unreferenced_workflows, load_workflow
+from .schedule import maintain as maintain_workflow
+from .schedule import run as run_workflow
+from .transform import set_config_api
+from .utils import workflow_template
+
+
+def load_config():
+    import yaml
+
+    config_paths = [
+        Path.home() / ".myapp/config.yaml",  # 用户主目录
+        Path("config.yaml")  # 当前目录
+    ]
+    for path in config_paths:
+        if path.exists():
+            with open(path) as f:
+                return yaml.safe_load(f)
+    return {"defaults": {"log": "default.log", "debug": False}}
+
+
+def get_config_value(option_name):
+    # 1. 尝试从环境变量读取
+    env_value = os.environ.get(f"MYAPP_{option_name.upper()}")
+    if env_value:
+        return env_value
+
+    # 2. 尝试从配置文件读取
+    config = load_config()
+    return config["defaults"].get(option_name)
+
+
+def log_options(func):
+
+    @click.option("--debug", is_flag=True, help="Enable debug mode.")
+    @click.option("--log", type=str, help="Log file path.")
+    @functools.wraps(func)  # 保持函数元信息
+    def wrapper(*args, log=None, debug=False, **kwargs):
+        if log is None and not debug:
+            logger.remove()
+            logger.add(sys.stderr, level='INFO')
+        elif log is None and debug:
+            logger.remove()
+            logger.add(sys.stderr, level='DEBUG')
+        elif log is not None and not debug:
+            logger.configure(handlers=[dict(sink=log, level='INFO')])
+        elif log is not None and debug:
+            logger.configure(handlers=[
+                dict(sink=log, level='DEBUG'),
+                dict(sink=sys.stderr, level='DEBUG')
+            ])
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+@click.group()
+def cli():
+    pass
+
+
+@click.command()
+@click.argument('workflow')
+@click.option('--code', '-c', default=None, help='The path of the code.')
+def create(workflow, code):
+    """
+    Create a new workflow file.
+    """
+    if code is None:
+        code = Path.cwd()
+
+    fname = Path(code) / f'{workflow}'
+    fname = Path(os.path.expanduser(fname))
+    if fname.exists():
+        click.echo(f'{workflow} already exists')
+        return
+
+    fname.parent.mkdir(parents=True, exist_ok=True)
+    deps = find_unreferenced_workflows(code)
+
+    with open(fname, 'w') as f:
+        f.write(workflow_template(list(deps)))
+    click.echo(f'{workflow} created')
+
+
+@click.command()
+@click.argument('key')
+@click.argument('value', type=str)
+@click.option('--api', '-a', default=None, help='The modlule name of the api.')
+def set(key, value, api):
+    """
+    Set a config.
+    """
+    from . import transform
+    if api is not None:
+        api = importlib.import_module(api)
+        set_config_api(api.query_config, api.update_config)
+    try:
+        value = eval(value)
+    except:
+        pass
+    transform.update_config({key: value})
+
+
+@click.command()
+@click.argument('key')
+@click.option('--api', '-a', default=None, help='The modlule name of the api.')
+def get(key, api):
+    """
+    Get a config.
+    """
+    from . import transform
+    if api is not None:
+        api = importlib.import_module(api)
+        set_config_api(api.query_config, api.update_config)
+    click.echo(transform.query_config(key))
+
+
+@click.command()
+@click.argument('workflow')
+@click.option('--code', '-c', default=None, help='The path of the code.')
+@click.option('--data', '-d', default=None, help='The path of the data.')
+@click.option('--api', '-a', default=None, help='The modlule name of the api.')
+@click.option('--plot', '-p', is_flag=True, help='Plot the result.')
+@click.option('--no-dependents',
+              '-n',
+              is_flag=True,
+              help='Do not run dependents.')
+@log_options
+def run(workflow, code, data, api, plot, no_dependents):
+    """
+    Run a workflow.
+    """
+    if api is not None:
+        api = importlib.import_module(api)
+        set_config_api(api.query_config, api.update_config)
+    if code is None:
+        code = Path.cwd()
+    if data is None:
+        data = Path(code) / 'logs'
+
+    code = Path(os.path.expanduser(code))
+    data = Path(os.path.expanduser(data))
+
+    if no_dependents:
+        run_workflow(load_workflow(workflow, code), code, data, plot=plot)
+    else:
+        maintain_workflow(load_workflow(workflow, code),
+                          code,
+                          data,
+                          run=True,
+                          plot=plot)
+
+
+@click.command()
+@click.argument('workflow')
+@click.option('--code', '-c', default=None, help='The path of the code.')
+@click.option('--data', '-d', default=None, help='The path of the data.')
+@click.option('--api', '-a', default=None, help='The modlule name of the api.')
+@click.option('--plot', '-p', is_flag=True, help='Plot the result.')
+@log_options
+def maintain(workflow, code, data, api, plot):
+    """
+    Maintain a workflow.
+    """
+    if api is not None:
+        api = importlib.import_module(api)
+        set_config_api(api.query_config, api.update_config)
+    if code is None:
+        code = Path.cwd()
+    if data is None:
+        data = Path(code) / 'logs'
+
+    code = Path(os.path.expanduser(code))
+    data = Path(os.path.expanduser(data))
+
+    maintain_workflow(load_workflow(workflow, code),
+                      code,
+                      data,
+                      run=False,
+                      plot=plot)
+
+
+cli.add_command(maintain)
+cli.add_command(run)
+cli.add_command(create)
+cli.add_command(set)
+cli.add_command(get)
+
+if __name__ == '__main__':
+    cli()

{qulab-2.4.9 → qulab-2.4.11}/qulab/executor/load.py

@@ -1,10 +1,14 @@
+import hashlib
 import inspect
+import pickle
+import re
+import string
 import warnings
 from importlib.util import module_from_spec, spec_from_file_location
 from pathlib import Path
 from types import ModuleType
 
-import loguru
+from loguru import logger
 
 from .storage import Result
 
@@ -124,12 +128,15 @@ def find_unreferenced_workflows(path: str) -> list[str]:
     for file_path in root.rglob("*.py"):
         if file_path.name == "__init__.py":
             continue
+        if file_path.name.endswith("_template.py") or re.match(
+                r'.*_tmp_[0-9a-fA-F]{8}.py', file_path.name):
+            continue
         try:
             rel_path = file_path.relative_to(root)
         except ValueError:
             continue
 
-        module = load_workflow(str(rel_path), root)
+        module = load_workflow_from_module(str(rel_path), root)
 
         if is_workflow(module):
             rel_str = str(rel_path)
@@ -140,7 +147,7 @@ def find_unreferenced_workflows(path: str) -> list[str]:
 
     # Check dependencies for each workflow module
     for rel_str in workflows:
-        module = load_workflow(rel_str, root)
+        module = load_workflow_from_module(rel_str, root)
 
         depends_func = getattr(module, "depends", None)
         if depends_func and callable(depends_func):
@@ -150,17 +157,15 @@ def find_unreferenced_workflows(path: str) -> list[str]:
                 )
                 continue
             try:
-                depends_list = depends_func()
+                depends_list = depends_func()[0]
             except Exception as e:
                 warnings.warn(f"Error calling depends() in {rel_str}: {e}")
                 continue
 
             if not isinstance(depends_list, list) or not all(
-                isinstance(item, str) for item in depends_list
-            ):
+                    isinstance(item, str) for item in depends_list):
                 warnings.warn(
-                    f"depends() in {rel_str} did not return a list of strings"
-                )
+                    f"depends() in {rel_str} did not return a list of strings")
                 continue
 
             for dep in depends_list:
@@ -178,9 +183,9 @@ def find_unreferenced_workflows(path: str) -> list[str]:
     return unreferenced
 
 
-def load_workflow(file_name: str,
-                  base_path: str | Path,
-                  package='workflows') -> WorkflowType:
+def load_workflow_from_module(file_name: str,
+                              base_path: str | Path,
+                              package='workflows') -> WorkflowType:
     if file_name.startswith('cfg:'):
         return SetConfigWorkflow(file_name[4:])
     base_path = Path(base_path)
@@ -200,3 +205,60 @@ def load_workflow(file_name: str,
     verify_check_method(module)
 
     return module
+
+
+def load_workflow_from_template(file_name: str,
+                                mappping: dict[str, str],
+                                base_path: str | Path,
+                                subtitle: str | None = None,
+                                package='workflows') -> WorkflowType:
+    base_path = Path(base_path)
+    path = Path(file_name)
+
+    with open(base_path / path) as f:
+        content = f.read()
+    template = string.Template(content)
+    content = template.substitute(mappping)
+
+    hash_str = hashlib.md5(pickle.dumps(mappping)).hexdigest()[:8]
+    if subtitle is None:
+        path = path.parent / path.stem.replace('_template',
+                                               f'_tmp{hash_str}.py')
+    else:
+        path = path.parent / path.stem.replace('_template', f'_{subtitle}.py')
+
+    with open(base_path / path, 'w') as f:
+        f.write(content)
+
+    module = load_workflow_from_module(str(path), base_path, package)
+
+    return module
+
+
+def load_workflow(workflow: str | tuple[str, dict],
+                  base_path: str | Path,
+                  package='workflows') -> WorkflowType:
+    if isinstance(workflow, tuple):
+        if len(workflow) == 2:
+            file_name, mapping = workflow
+            w = load_workflow_from_template(file_name, mapping, base_path,
+                                            None, package)
+        elif len(workflow) == 3:
+            file_name, subtitle, mapping = workflow
+            w = load_workflow_from_template(file_name, mapping, base_path,
+                                            subtitle, package)
+        else:
+            raise ValueError(f"Invalid workflow: {workflow}")
+        w.__workflow_id__ = str(Path(w.__file__).relative_to(base_path))
+    elif isinstance(workflow, str):
+        if workflow.startswith('cfg:'):
+            key = workflow[4:]
+            w = SetConfigWorkflow(key)
+            w.__workflow_id__ = workflow
+        else:
+            w = load_workflow_from_module(workflow, base_path, package)
+            w.__workflow_id__ = str(Path(w.__file__).relative_to(base_path))
+    else:
+        raise TypeError(f"Invalid workflow: {workflow}")
+
+    return w

qulab-2.4.11/qulab/executor/schedule.py

@@ -0,0 +1,304 @@
+import functools
+import uuid
+from datetime import datetime, timedelta
+from pathlib import Path
+
+from loguru import logger
+
+from . import transform
+from .load import WorkflowType, load_workflow
+from .storage import (Result, find_result, get_head, renew_result,
+                      revoke_result, save_result)
+
+
+class CalibrationFailedError(Exception):
+    pass
+
+
+def check_state(workflow: WorkflowType, code_path: str | Path,
+                state_path: str | Path) -> bool:
+    """
+    check state should report a pass if and only if the following are satisfied:
+    
+    1. The cal has had check data or calibrate pass within the timeout period.
+    2. The cal has not failed calibrate without resolution.
+    3. No dependencies have been recalibrated since the last time check data or calibrate was run on this cal.
+    4. All dependencies pass check state.
+    """
+    logger.debug(f'check_state: "{workflow}"')
+    result = find_result(workflow.__workflow_id__, state_path)
+    if not result:
+        logger.debug(
+            f'check_state failed: No history found for "{workflow.__workflow_id__}"'
+        )
+        return False
+    if hasattr(workflow, 'check_state') and callable(workflow.check_state):
+        logger.debug(
+            f'check_state: "{workflow.__workflow_id__}" has custom check_state method'
+        )
+        return workflow.check_state(result)
+    if workflow.__timeout__ is not None and datetime.now(
+    ) > result.checked_time + timedelta(seconds=workflow.__timeout__):
+        logger.debug(
+            f'check_state failed: "{workflow.__workflow_id__}" has expired')
+        return False
+    if not result.in_spec:
+        logger.debug(
+            f'check_state failed: "{workflow.__workflow_id__}" is out of spec')
+        return False
+    if result.bad_data:
+        logger.debug(
+            f'check_state failed: "{workflow.__workflow_id__}" has bad data')
+        return False
+    for n in get_dependents(workflow, code_path):
+        r = find_result(n, state_path)
+        if r is None or r.checked_time > result.checked_time:
+            logger.debug(
+                f'check_state failed: "{workflow.__workflow_id__}" has outdated dependencies'
+            )
+            return False
+    for n in get_dependents(workflow, code_path):
+        if not check_state(n, code_path, state_path):
+            logger.debug(
+                f'check_state failed: "{workflow.__workflow_id__}" has bad dependencies'
+            )
+            return False
+    return True
+
+
+def call_analyzer(node, data, history, check=False, plot=False):
+    if check:
+        result = transform.params_to_result(
+            node.check_analyze(*data,
+                               history=transform.result_to_params(history)))
+        result.fully_calibrated = False
+    else:
+        result = transform.params_to_result(
+            node.analyze(*data, history=transform.result_to_params(history)))
+        result.fully_calibrated = True
+        if plot:
+            call_plot(node, result)
+    result.data = data
+    return result
+
+
+@logger.catch()
+def call_plot(node, result, check=False):
+    if hasattr(node, 'plot') and callable(node.plot):
+        state, params, other = transform.result_to_params(result)
+        node.plot(state, params, other)
+
+
+@functools.lru_cache(maxsize=128)
+def check_data(workflow: WorkflowType, code_path: str | Path,
+               state_path: str | Path, plot: bool, session_id: str) -> Result:
+    """
+    check data answers two questions:
+    Is the parameter associated with this cal in spec,
+    and is the cal scan working as expected?
+    """
+    history = find_result(workflow.__workflow_id__, state_path)
+
+    if history is None:
+        logger.debug(f'No history found for "{workflow.__workflow_id__}"')
+        result = Result()
+        result.in_spec = False
+        result.bad_data = False
+        return result
+
+    if history.bad_data:
+        logger.debug(
+            f'History found for "{workflow.__workflow_id__}", but bad data')
+        return history
+    if not history.in_spec:
+        logger.debug(
+            f'History found for "{workflow.__workflow_id__}", but out of spec')
+        return history
+
+    logger.debug(
+        f'History found for "{workflow.__workflow_id__}", but has expired')
+
+    if hasattr(workflow, 'check') and callable(workflow.check) and hasattr(
+            workflow, 'check_analyze') and callable(workflow.check_analyze):
+        logger.debug(f'Checking "{workflow}" with "check" method ...')
+        data = workflow.check()
+        result = Result()
+        result.data = data
+        save_result(workflow.__workflow_id__, result, state_path)
+
+        logger.debug(f'Checked "{workflow}" !')
+        result = call_analyzer(workflow, data, history, check=True, plot=plot)
+        if result.in_spec:
+            logger.debug(
+                f'"{workflow.__workflow_id__}": checked in spec, renewing result'
+            )
+            renew_result(workflow.__workflow_id__, state_path)
+        else:
+            logger.debug(
+                f'"{workflow.__workflow_id__}": checked out of spec, revoking result'
+            )
+            revoke_result(workflow.__workflow_id__, state_path)
+    else:
+        logger.debug(
+            f'Checking "{workflow.__workflow_id__}" with "calibrate" method ...'
+        )
+        data = workflow.calibrate()
+        result = Result()
+        result.data = data
+        save_result(workflow.__workflow_id__, result, state_path)
+
+        logger.debug(f'Calibrated "{workflow}" !')
+        result = call_analyzer(workflow, data, history, check=False, plot=plot)
+        save_result(workflow.__workflow_id__, result, state_path,
+                    get_head(workflow.__workflow_id__, state_path))
+
+    return result
+
+
+@functools.lru_cache(maxsize=128)
+def calibrate(workflow: WorkflowType, code_path: str | Path,
+              state_path: str | Path, plot: bool, session_id: str) -> Result:
+    history = find_result(workflow, state_path)
+
+    logger.debug(f'Calibrating "{workflow.__workflow_id__}" ...')
+    data = workflow.calibrate()
+    result = Result()
+    result.data = data
+    save_result(workflow.__workflow_id__, result, state_path)
+    logger.debug(f'Calibrated "{workflow.__workflow_id__}" !')
+    result = call_analyzer(workflow, data, history, check=False, plot=plot)
+    save_result(workflow.__workflow_id__, result, state_path,
+                get_head(workflow.__workflow_id__, state_path))
+    return result
+
+
+def diagnose(workflow: WorkflowType, code_path: str | Path,
+             state_path: str | Path, plot: bool, session_id: str):
+    '''
+    Returns: True if node or dependent recalibrated.
+    '''
+    logger.debug(f'diagnose "{workflow.__workflow_id__}"')
+    # check_data
+    result = check_data(workflow, code_path, state_path, plot, session_id)
+    # in spec case
+    if result.in_spec:
+        logger.debug(
+            f'"{workflow.__workflow_id__}": Checked! In spec, no need to diagnose'
+        )
+        return False
+    # bad data case
+    recalibrated = []
+    if result.bad_data:
+        logger.debug(
+            f'"{workflow.__workflow_id__}": Bad data, diagnosing dependents')
+        recalibrated = [
+            diagnose(n, code_path, state_path, plot, session_id)
+            for n in get_dependents(workflow, code_path)
+        ]
+    if not any(recalibrated):
+        if result.bad_data:
+            raise CalibrationFailedError(
+                f'"{workflow.__workflow_id__}": bad data but no dependents recalibrated.'
+            )
+        logger.debug(
+            f'"{workflow.__workflow_id__}": no dependents recalibrated.')
+    # calibrate
+    if any(recalibrated):
+        logger.debug(
+            f'recalibrate "{workflow.__workflow_id__}" because some dependents recalibrated.'
+        )
+    elif not result.in_spec and not result.bad_data:
+        logger.debug(
+            f'recalibrate "{workflow.__workflow_id__}" because out of spec.')
+    elif result.in_spec:
+        logger.error(
+            f'Never reach: recalibrate "{workflow.__workflow_id__}" because in spec.'
+        )
+    elif result.bad_data:
+        logger.error(
+            f'Never reach: recalibrate "{workflow.__workflow_id__}" because bad data.'
+        )
+    else:
+        logger.error(f'Never reach: recalibrate "{workflow.__workflow_id__}"')
+
+    result = calibrate(workflow, code_path, state_path, plot, session_id)
+    if result.bad_data or not result.in_spec:
+        raise CalibrationFailedError(
+            f'"{workflow.__workflow_id__}": All dependents passed, but calibration failed!'
+        )
+    transform.update_parameters(result)
+    return True
+
+
+def get_dependents(workflow: WorkflowType,
+                   code_path: str | Path) -> list[WorkflowType]:
+    return [load_workflow(n, code_path) for n in workflow.depends()[0]]
+
+
+@logger.catch(reraise=True)
+def maintain(workflow: WorkflowType,
+             code_path: str | Path,
+             state_path: str | Path,
+             session_id: str | None = None,
+             run: bool = False,
+             plot: bool = False):
+    if session_id is None:
+        session_id = uuid.uuid4().hex
+    logger.debug(f'run "{workflow.__workflow_id__}"'
+                 if run else f'maintain "{workflow.__workflow_id__}"')
+    # recursive maintain
+    for n in get_dependents(workflow, code_path):
+        logger.debug(
+            f'maintain "{n.__workflow_id__}" because it is depended by "{workflow.__workflow_id__}"'
+        )
+        maintain(n, code_path, state_path, session_id, run=False, plot=plot)
+    else:
+        logger.debug(
+            f'"{workflow.__workflow_id__}": All dependents maintained')
+    # check_state
+    if check_state(workflow, code_path, state_path) and not run:
+        logger.debug(
+            f'"{workflow.__workflow_id__}": In spec, no need to maintain')
+        return
+    # check_data
+    result = check_data(workflow, code_path, state_path, plot, session_id)
+    if result.in_spec:
+        if not run:
+            logger.debug(
+                f'"{workflow.__workflow_id__}": In spec, no need to maintain')
+            return
+    elif result.bad_data:
+        logger.debug(
+            f'"{workflow.__workflow_id__}": Bad data, diagnosing dependents')
+        for n in get_dependents(workflow, code_path):
+            logger.debug(
+                f'diagnose "{n.__workflow_id__}" because of "{workflow.__workflow_id__}" bad data'
+            )
+            diagnose(n, code_path, state_path, plot, session_id)
+        else:
+            logger.debug(
+                f'"{workflow.__workflow_id__}": All dependents diagnosed')
+    # calibrate
+    logger.debug(f'recalibrate "{workflow.__workflow_id__}"')
+    result = calibrate(workflow, code_path, state_path, plot, session_id)
+    if result.bad_data or not result.in_spec:
+        raise CalibrationFailedError(
+            f'"{workflow.__workflow_id__}": All dependents passed, but calibration failed!'
+        )
+    transform.update_parameters(result)
+    return
+
+
+@logger.catch(reraise=True)
+def run(workflow: WorkflowType,
+        code_path: str | Path,
+        state_path: str | Path,
+        plot: bool = False):
+    logger.debug(f'run "{workflow.__workflow_id__}" without dependences.')
+    result = calibrate(workflow, code_path, state_path, plot)
+    if result.bad_data or not result.in_spec:
+        raise CalibrationFailedError(
+            f'"{workflow.__workflow_id__}": All dependents passed, but calibration failed!'
+        )
+    transform.update_parameters(result)
+    return

qulab-2.4.11/qulab/version.py

@@ -0,0 +1 @@
+__version__ = "2.4.11"