apsbits 1.0.0rc2__py3-none-any.whl

apsbits/__init__.py

@@ -0,0 +1,18 @@
+# -*- coding: iso-8859-1 -*-
+
+"""Model Bluesky Data Acquisition Instrument."""
+
+from apsbits.utils.logging_setup import configure_logging
+
+configure_logging()
+
+__package__ = "apsbits"
+try:
+    from setuptools_scm import get_version
+
+    __version__ = get_version(root="..", relative_to=__file__)
+    del get_version
+except (LookupError, ModuleNotFoundError):
+    from importlib.metadata import version
+
+    __version__ = version(__package__)

apsbits/_version.py

@@ -0,0 +1,21 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '1.0.0rc2'
+__version_tuple__ = version_tuple = (1, 0, 0, 'rc2')

apsbits/core/__init__.py

@@ -0,0 +1,11 @@
+"""
+Utility support to start bluesky sessions.
+
+Also contains setup code that MUST run before other code in this directory.
+"""
+
+from apsbits.utils.helper_functions import debug_python
+from apsbits.utils.helper_functions import mpl_setup
+
+debug_python()
+mpl_setup()

apsbits/core/best_effort_init.py

@@ -0,0 +1,38 @@
+"""
+BestEffortCallback: simple real-time visualizations, provides ``bec``.
+======================================================================
+
+.. autosummary::
+    ~bec
+    ~peaks
+"""
+
+import logging
+
+from bluesky.callbacks.best_effort import BestEffortCallback
+
+from apsbits.utils.config_loaders import iconfig
+from apsbits.utils.helper_functions import running_in_queueserver
+
+logger = logging.getLogger(__name__)
+logger.bsdev(__file__)
+
+bec = BestEffortCallback()
+"""BestEffortCallback object, creates live tables and plots."""
+
+bec_config = iconfig.get("BEC", {})
+
+if not bec_config.get("BASELINE", True):
+    bec.disable_baseline()
+
+if not bec_config.get("HEADING", True):
+    bec.disable_heading()
+
+if not bec_config.get("PLOTS", True) or running_in_queueserver():
+    bec.disable_plots()
+
+if not bec_config.get("TABLE", True):
+    bec.disable_table()
+
+peaks = bec.peaks
+"""Dictionary with statistical analysis of LivePlots."""

apsbits/core/catalog_init.py

@@ -0,0 +1,29 @@
+"""
+Databroker catalog, provides ``cat``
+====================================
+
+.. autosummary::
+    ~cat
+"""
+
+import logging
+
+import databroker
+
+from apsbits.utils.config_loaders import iconfig
+
+logger = logging.getLogger(__name__)
+logger.bsdev(__file__)
+
+TEMPORARY_CATALOG_NAME = "temp"
+
+catalog_name = iconfig.get("DATABROKER_CATALOG", TEMPORARY_CATALOG_NAME)
+try:
+    _cat = databroker.catalog[catalog_name].v2
+except KeyError:
+    _cat = databroker.temp().v2
+
+cat = _cat
+"""Databroker catalog object, receives new data from ``RE``."""
+
+logger.info("Databroker catalog: %s", cat.name)

apsbits/core/run_engine_init.py

@@ -0,0 +1,71 @@
+"""
+Setup the Bluesky RunEngine, provides ``RE`` and ``sd``.
+========================================================
+
+.. autosummary::
+    ~RE
+    ~sd
+"""
+
+import logging
+
+import bluesky
+from bluesky.utils import ProgressBarManager
+
+from apsbits.core.best_effort_init import bec
+from apsbits.core.catalog_init import cat
+from apsbits.utils.config_loaders import iconfig
+from apsbits.utils.controls_setup import connect_scan_id_pv
+from apsbits.utils.controls_setup import set_control_layer
+from apsbits.utils.controls_setup import set_timeouts
+from apsbits.utils.metadata import MD_PATH
+from apsbits.utils.stored_dict import StoredDict
+
+logger = logging.getLogger(__name__)
+logger.bsdev(__file__)
+
+re_config = iconfig.get("RUN_ENGINE", {})
+
+RE = bluesky.RunEngine()
+"""The bluesky RunEngine object."""
+
+# Save/restore RE.md dictionary, in this precise order.
+if MD_PATH is not None:
+    handler_name = re_config.get("MD_STORAGE_HANDLER", "StoredDict")
+    logger.debug(
+        "Select %r to store 'RE.md' dictionary in %s.",
+        handler_name,
+        MD_PATH,
+    )
+    try:
+        if handler_name == "PersistentDict":
+            RE.md = bluesky.utils.PersistentDict(MD_PATH)
+        else:
+            RE.md = StoredDict(MD_PATH)
+    except Exception as error:
+        print(
+            "\n"
+            f"Could not create {handler_name} for RE metadata. Continuing"
+            f" without saving metadata to disk. {error=}\n"
+        )
+        logger.warning("%s('%s') error:%s", handler_name, MD_PATH, error)
+
+# RE.md.update(re_metadata(cat))  # programmatic metadata
+# RE.md.update(re_config.get("DEFAULT_METADATA", {}))
+
+sd = bluesky.SupplementalData()
+"""Baselines & monitors for ``RE``."""
+
+RE.subscribe(cat.v1.insert)
+RE.subscribe(bec)
+RE.preprocessors.append(sd)
+
+set_control_layer()
+set_timeouts()  # MUST happen before ANY EpicsSignalBase (or subclass) is created.
+
+connect_scan_id_pv(RE)  # if configured
+
+if re_config.get("USE_PROGRESS_BAR", True):
+    # Add a progress bar.
+    pbar_manager = ProgressBarManager()
+    RE.waiting_hook = pbar_manager

apsbits/demo_instrument/README.md

@@ -0,0 +1 @@
+## Demo Instrument

apsbits/demo_instrument/__init__.py

@@ -0,0 +1,3 @@
+"""
+Custom folder to store all beamline specific implementations
+"""

apsbits/demo_instrument/callbacks/__init__.py

@@ -0,0 +1 @@
+"""RunEngine callbacks, mostly."""

apsbits/demo_instrument/callbacks/nexus_data_file_writer.py

@@ -0,0 +1,60 @@
+"""
+
+NeXus Writer
+============
+
+Write scan(s) to a NeXus/HDF5 file.
+
+.. autosummary::
+    :nosignatures:
+
+    ~MyNXWriter
+    ~nxwriter
+"""
+
+import logging
+
+from apsbits.core.run_engine_init import RE
+from apsbits.utils.aps_functions import host_on_aps_subnet
+from apsbits.utils.config_loaders import iconfig
+
+logger = logging.getLogger(__name__)
+logger.bsdev(__file__)
+
+
+if host_on_aps_subnet():
+    from apstools.callbacks import NXWriterAPS as NXWriter
+else:
+    from apstools.callbacks import NXWriter
+
+
+class MyNXWriter(NXWriter):
+    """Patch to get sample title from metadata, if available."""
+
+    def get_sample_title(self):
+        """
+        Get the title from the metadata or modify the default.
+
+        default title: S{scan_id}-{plan_name}-{short_uid}
+        """
+        try:
+            title = self.metadata["title"]
+        except KeyError:
+            # title = super().get_sample_title()  # the default title
+            title = f"S{self.scan_id:05d}-{self.plan_name}-{self.uid[:7]}"
+        return title
+
+
+nxwriter = MyNXWriter()  # create the callback instance
+"""The NeXus file writer object."""
+
+if iconfig.get("NEXUS_DATA_FILES", {}).get("ENABLE", False):
+    RE.subscribe(nxwriter.receiver)  # write data to NeXus files
+
+nxwriter.file_extension = iconfig.get("NEXUS_DATA_FILES", {}).get(
+    "FILE_EXTENSION", "hdf"
+)
+print("\n\n\n")
+print(nxwriter.file_extension)
+warn_missing = iconfig.get("NEXUS_DATA_FILES", {}).get("WARN_MISSING", False)
+nxwriter.warn_on_missing_content = warn_missing

apsbits/demo_instrument/callbacks/spec_data_file_writer.py

@@ -0,0 +1,93 @@
+"""
+custom callbacks
+================
+
+.. autosummary::
+    :nosignatures:
+
+    ~newSpecFile
+    ~spec_comment
+    ~specwriter
+"""
+
+import datetime
+import logging
+import pathlib
+
+import apstools.callbacks
+import apstools.utils
+
+from apsbits.core.run_engine_init import RE
+from apsbits.utils.config_loaders import iconfig
+
+logger = logging.getLogger(__name__)
+logger.bsdev(__file__)
+
+
+file_extension = iconfig.get("SPEC_DATA_FILES", {}).get("FILE_EXTENSION", "dat")
+
+
+def spec_comment(comment, doc=None):
+    """Make it easy for user to add comments to the data file."""
+    apstools.callbacks.spec_comment(comment, doc, specwriter)
+
+
+def newSpecFile(title, scan_id=None, RE=None):
+    """
+    User choice of the SPEC file name.
+
+    Cleans up title, prepends month and day and appends file extension.
+    If ``RE`` is passed, then resets ``RE.md["scan_id"] = scan_id``.
+
+    If the SPEC file already exists, then ``scan_id`` is ignored and
+    ``RE.md["scan_id"]`` is set to the last scan number in the file.
+    """
+    kwargs = {}
+    if RE is not None:
+        kwargs["RE"] = RE
+
+    mmdd = str(datetime.datetime.now()).split()[0][5:].replace("-", "_")
+    clean = apstools.utils.cleanupText(title)
+    fname = pathlib.Path(f"{mmdd}_{clean}.{file_extension}")
+    if fname.exists():
+        logger.warning(f">>> file already exists: {fname} <<<")
+        handled = "appended"
+    else:
+        kwargs["scan_id"] = scan_id or 1
+        handled = "created"
+
+    specwriter.newfile(fname, **kwargs)
+
+    logger.info(f"SPEC file name : {specwriter.spec_filename}")
+    logger.info(f"File will be {handled} at end of next bluesky scan.")
+
+
+# write scans to SPEC data file
+try:
+    # apstools >=1.6.21
+    _specwriter = apstools.callbacks.SpecWriterCallback2()
+except AttributeError:
+    # apstools <1.6.21
+    _specwriter = apstools.callbacks.SpecWriterCallback()
+
+specwriter = _specwriter
+"""The SPEC file writer object."""
+
+# make the SPEC file in current working directory (assumes is writable)
+specwriter.newfile(specwriter.spec_filename)
+
+if iconfig.get("SPEC_DATA_FILES", {}).get("ENABLE", False):
+    RE.subscribe(specwriter.receiver)  # write data to SPEC files
+    logger.info("SPEC data file: %s", specwriter.spec_filename.resolve())
+
+try:
+    # feature new in apstools 1.6.14
+    from apstools.plans import label_stream_wrapper
+
+    def motor_start_preprocessor(plan):
+        """Record motor positions at start of each run."""
+        return label_stream_wrapper(plan, "motor", when="start")
+
+    RE.preprocessors.append(motor_start_preprocessor)
+except Exception:
+    logger.warning("Could load support to log motors positions.")

apsbits/demo_instrument/configs/__init__.py

@@ -0,0 +1 @@
+"""Configs required to set up user package"""

apsbits/demo_instrument/configs/devices.yml

@@ -0,0 +1,52 @@
+# Guarneri-style device YAML configuration
+
+apsbits.utils.sim_creator.predefined_device:
+- {creator: ophyd.sim.motor, name: sim_motor}
+- {creator: ophyd.sim.noisy_det, name: sim_det}
+
+apstools.devices.SimulatedApsPssShutterWithStatus:
+- name: shutter
+  labels: ["shutters"]
+
+# ophyd.Signal:
+# - name: test
+#   value: 50.7
+# - name: t2
+#   value: 2
+
+# apstools.synApps.Optics2Slit2D_HV:
+# - name: slit1
+#   prefix: ioc:Slit1
+#   labels: ["slits"]
+
+# hkl.SimulatedE4CV:
+# - name: sim4c
+#   prefix: ""
+#   labels: ["diffractometer"]
+
+# ophyd.scaler.ScalerCH:
+# - name: scaler1
+#   prefix: vme:scaler1
+#   labels: ["scalers", "detectors"]
+
+# ophyd.EpicsMotor:
+# - {name: m1, prefix: gp:m1, labels: ["motor"]}
+# - {name: m2, prefix: gp:m2, labels: ["motor"]}
+# - {name: m3, prefix: gp:m3, labels: ["motor"]}
+# - {name: m4, prefix: gp:m4, labels: ["motor"]}
+
+# apstools.devices.ad_creator:
+#   - name: adsimdet
+#     prefix: "ad:"
+#     labels: ["area_detector", "detectors"]
+#     plugins:
+#       - cam:
+#           class: apstools.devices.SimDetectorCam_V34
+#       - image
+#       - pva
+#       - hdf1:
+#           class: apstools.devices.AD_EpicsFileNameHDF5Plugin
+#           read_path_template: "/path/to/bluesky/tmp/"
+#           write_path_template: "/path/to/ioc/tmp/"
+#       - roi1
+#       - stats1

apsbits/demo_instrument/configs/devices_aps_only.yml

@@ -0,0 +1,6 @@
+# Guarneri-style device YAML configuration
+
+# Objects only available at APS
+
+apstools.devices.ApsMachineParametersDevice:
+  - name: aps

apsbits/demo_instrument/configs/iconfig.yml

@@ -0,0 +1,81 @@
+# Configuration for the Bluesky instrument package.
+
+# identify the version of this iconfig.yml file
+ICONFIG_VERSION: 2.0.0
+
+# Add additional configuration for use with your instrument.
+
+### The short name for the databroker catalog.
+DATABROKER_CATALOG: &databroker_catalog temp
+
+### RunEngine configuration
+RUN_ENGINE:
+    DEFAULT_METADATA:
+        beamline_id: demo_instrument
+        instrument_name: Most Glorious Scientific Instrument
+        proposal_id: commissioning
+        databroker_catalog: *databroker_catalog
+
+    ### EPICS PV to use for the `scan_id`.
+    ### Default: `RE.md["scan_id"]` (not using an EPICS PV)
+    # SCAN_ID_PV: "IOC:bluesky_scan_id"
+
+    ### Where to "autosave" the RE.md dictionary.
+    ### Defaults:
+    MD_STORAGE_HANDLER: StoredDict
+    MD_PATH: .re_md_dict.yml
+
+    ### The progress bar is nice to see,
+    ### except when it clutters the output in Jupyter notebooks.
+    ### Default: False
+    USE_PROGRESS_BAR: false
+
+# Command-line tools, such as %wa, %ct, ...
+USE_BLUESKY_MAGICS: true
+
+### Best Effort Callback Configurations
+### Defaults: all true
+### except no plots in queueserver
+BEC:
+    BASELINE: true
+    HEADING: true
+    PLOTS: false
+    TABLE: true
+
+### Support for known output file formats.
+### Uncomment to use.  If undefined, will not write that type of file.
+### Each callback should apply its configuration from here.
+NEXUS_DATA_FILES:
+    ENABLE: false
+    FILE_EXTENSION: hdf
+    WARN_MISSING_CONTENT: true
+
+SPEC_DATA_FILES:
+    ENABLE: true
+    FILE_EXTENSION: dat
+
+### APS Data Management
+### Use bash shell, deactivate all conda environments, source this file:
+DM_SETUP_FILE: "/home/dm/etc/dm.setup.sh"
+
+### Local OPHYD Device Control Yaml
+DEVICES_FILE: devices.yml
+APS_DEVICES_FILE: devices_aps_only.yml
+
+# ----------------------------------
+
+OPHYD:
+    ### Control layer for ophyd to communicate with EPICS.
+    ### Default: PyEpics
+    ### Choices: "PyEpics" or "caproto" # caproto is not yet supported
+    CONTROL_LAYER: PyEpics
+
+    ### default timeouts (seconds)
+    TIMEOUTS:
+        PV_READ: &TIMEOUT 5
+        PV_WRITE: *TIMEOUT
+        PV_CONNECTION: *TIMEOUT
+
+# Control detail of exception traces in IPython (console and notebook).
+# Options are: Minimal, Plain, Verbose
+XMODE_DEBUG_LEVEL: Plain

apsbits/demo_instrument/configs/logging.yml

@@ -0,0 +1,41 @@
+# Bluesky Session Logging Configuration
+
+console_logs:
+  date_format: "%a-%H:%M:%S"
+  log_format: "%(levelname)-.1s %(asctime)s.%(msecs)03d: %(message)s"
+  level: info
+  root_level: bsdev
+
+file_logs:
+  date_format: "%Y-%m-%d %H:%M:%S"
+  log_directory: .logs
+  log_filename_base: logging.log
+  log_format: "|\
+    %(asctime)s.%(msecs)03d|\
+    %(levelname)s|\
+    %(process)d|\
+    %(name)s|\
+    %(module)s|\
+    %(lineno)d|\
+    %(threadName)s| - \
+    %(message)s"
+  maxBytes: 1_000_000
+  backupCount: 9
+  level: info
+  rotate_on_startup: true
+
+ipython_logs:
+  log_directory: .logs
+  log_filename_base: ipython_log.py
+  log_mode: rotate
+  options: -o -t
+
+modules:
+  apstools: warning
+  bluesky-queueserver: warning
+  bluesky: warning
+  bluesky.RE: warning
+  caproto: warning
+  databroker: warning
+  instrument: bsdev
+  ophyd: warning

apsbits/demo_instrument/devices/__init__.py

@@ -0,0 +1 @@
+"""Ophyd-style devices."""

apsbits/demo_instrument/plans/__init__.py

@@ -0,0 +1,8 @@
+"""Bluesky plans."""
+
+from .dm_plans import dm_kickoff_workflow  # noqa: F401
+from .dm_plans import dm_list_processing_jobs  # noqa: F401
+from .dm_plans import dm_submit_workflow_job  # noqa: F401
+from .sim_plans import sim_count_plan  # noqa: F401
+from .sim_plans import sim_print_plan  # noqa: F401
+from .sim_plans import sim_rel_scan_plan  # noqa: F401

apsbits/demo_instrument/plans/dm_plans.py

@@ -0,0 +1,111 @@
+"""
+Plans in support of APS Data Management
+=======================================
+
+.. autosummary::
+
+    ~dm_kickoff_workflow
+    ~dm_list_processing_jobs
+    ~dm_submit_workflow_job
+"""
+
+import logging
+
+from apstools.devices import DM_WorkflowConnector
+from apstools.utils import dm_api_proc
+from apstools.utils import share_bluesky_metadata_with_dm
+from bluesky import plan_stubs as bps
+
+logger = logging.getLogger(__name__)
+logger.bsdev(__file__)
+
+
+def dm_kickoff_workflow(run, argsDict, timeout=None, wait=False):
+    """
+    Start a DM workflow for this bluesky run and share run's metadata with DM.
+
+    PARAMETERS:
+
+    run (*obj*): Bluesky run object (such as 'run = cat[uid]').
+
+    argsDict (*dict*): Dictionary of parameters needed by 'workflowName'.
+        At minimum, most workflows expect these keys: 'filePath' and
+        'experimentName'.  Consult the workflow for the expected
+        content of 'argsDict'.
+
+    timeout (*number*): When should bluesky stop reporting on this
+        DM workflow job (if it has not ended). Units are seconds.
+        Default is forever.
+
+    wait (*bool*): Should this plan stub wait for the job to end?
+        Default is 'False'.
+    """
+    dm_workflow = DM_WorkflowConnector(name="dm_workflow")
+
+    if timeout is None:
+        # Disable periodic reports, use a long time (s).
+        timeout = 999_999_999_999
+
+    yield from bps.mv(dm_workflow.concise_reporting, True)
+    yield from bps.mv(dm_workflow.reporting_period, timeout)
+
+    workflow_name = argsDict.pop["workflowName"]
+    yield from dm_workflow.run_as_plan(
+        workflow=workflow_name,
+        wait=wait,
+        timeout=timeout,
+        **argsDict,
+    )
+
+    # Upload bluesky run metadata to APS DM.
+    share_bluesky_metadata_with_dm(argsDict["experimentName"], workflow_name, run)
+
+    # Users requested the DM workflow job ID be printed to the console.
+    dm_workflow._update_processing_data()
+    job_id = dm_workflow.job_id.get()
+    job_stage = dm_workflow.stage_id.get()
+    job_status = dm_workflow.status.get()
+    print(f"DM workflow id: {job_id!r}  status: {job_status}  stage: {job_stage}")
+
+
+def dm_list_processing_jobs(exclude=None):
+    """
+    Show all the DM jobs with status not excluded.
+
+    Excluded status (default): 'done', 'failed'
+    """
+    yield from bps.null()  # make this a plan stub
+    api = dm_api_proc()
+    if exclude is None:
+        exclude = ("done", "failed")
+
+    for j in api.listProcessingJobs():
+        if j["status"] not in exclude:
+            print(
+                f"id={j['id']!r}"
+                f"  submitted={j.get('submissionTimestamp')}"
+                f"  status={j['status']!r}"
+            )
+
+
+def dm_submit_workflow_job(workflowName, argsDict):
+    """
+    Low-level plan stub to submit a job to a DM workflow.
+
+    It is recommended to use dm_kickoff_workflow() instead.
+    This plan does not share run metadata with DM.
+
+    PARAMETERS:
+
+    workflowName (*str*): Name of the DM workflow to be run.
+
+    argsDict (*dict*): Dictionary of parameters needed by 'workflowName'.
+        At minimum, most workflows expect these keys: 'filePath' and
+        'experimentName'.  Consult the workflow for the expected
+        content of 'argsDict'.
+    """
+    yield from bps.null()  # make this a plan stub
+    api = dm_api_proc()
+
+    job = api.startProcessingJob(api.username, workflowName, argsDict)
+    print(f"workflow={workflowName!r}  id={job['id']!r}")