ChessAnalysisPipeline 0.0.17.dev3__py3-none-any.whl

CHAP/edd/writer.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+"""EDD specific writers."""
+
+# Local modules
+from CHAP import Writer
+
+class StrainAnalysisUpdateWriter(Writer):
+    def write(self, data):
+        # Third party modules
+        from nexusformat.nexus import nxload
+
+        # Local modules
+        from CHAP.edd.processor import StrainAnalysisProcessor
+
+        points = self.unwrap_pipelinedata(data)[0]
+        nxroot = nxload(self.filename, mode='r+')
+        StrainAnalysisProcessor.add_points(nxroot, points, logger=self.logger)
+
+        return nxroot
+
+
+if __name__ == '__main__':
+    # Local modules
+    from CHAP.writer import main
+
+    main()

CHAP/foxden/__init__.py

@@ -0,0 +1,19 @@
+"""This subpackage contains `PipelineItems` to communicate with FOXDEN
+services.
+"""
+
+from CHAP.foxden.processor import (
+#    FoxdenMetadataProcessor,
+    FoxdenProvenanceProcessor,
+)
+from CHAP.foxden.reader import (
+    FoxdenDataDiscoveryReader,
+    FoxdenMetadataReader,
+    FoxdenProvenanceReader,
+    FoxdenSpecScansReader,
+)
+from CHAP.foxden.writer import (
+    FoxdenDoiWriter,
+    FoxdenMetadataWriter,
+    FoxdenProvenanceWriter,
+)

CHAP/foxden/models.py

@@ -0,0 +1,71 @@
+"""FOXDEN Pydantic model classes."""
+
+# System modules
+from typing import (
+#    Literal,
+    Optional,
+)
+
+# Third party modules
+from pydantic import (
+    conint,
+    constr,
+#    field_validator,
+)
+
+# Local modules
+from CHAP import CHAPBaseModel
+
+
+class FoxdenRequestConfig(CHAPBaseModel):
+    """FOXDEN HTTP request base configuration class.
+
+    :param url: URL of service.
+    :type url: str
+    :param did: FOXDEN dataset identifier (did).
+    :type did: string, optional
+    :param query: FOXDEN query.
+    :type query: string, optional
+    :param method: HTTP request method (not case sensitive),
+        defaults to `'POST'`.
+    :type method: Literal['DELETE', 'GET', 'POST', 'PUT'], optional
+#    :param scope: FOXDEN scope (not case sensitive).
+#    :type scope: Literal['read', 'write'], optional
+#    :param idx: Ask Valentin, currently it's ignored
+#    :type idx: int, optional
+    :param limit: Maximum number of returned records,
+        defaults to `10`.
+    :type limit: int, optional
+    :param verbose: Verbose output flag, defaults to `False`.
+    :type verbose: bool, optional
+    """
+    # Mimics golib.services.data.ServiceQuery
+    did: Optional[constr(
+        strict=True, strip_whitespace=True, to_lower=True)] = None
+    limit: Optional[conint(gt=0)] = 10
+    query: Optional[constr(
+        strict=True, strip_whitespace=True, to_lower=True)] = None
+#    method: Optional[Literal['DELETE', 'GET', 'POST', 'PUT']] = 'POST'
+#    scope: Optional[Literal['read', 'write']] = None
+#    sortkeys: Optional[
+#        conlist[item_type=constr(strict=True, strip_whitespace=True)]] = None
+#    sortorder: Optional[int] = None
+#    spec: Optional[map[string]any] ?
+#    sql: Optional[constr(strict=True, strip_whitespace=True)] = None
+#    idx: Optional[conint(ge=0)] = 0
+    url: constr(strict=True, strip_whitespace=True)
+    verbose: Optional[bool] = None
+
+#    @field_validator('method', mode='before')
+#    @classmethod
+#    def validate_method(cls, method):
+#        """Capitalize method."""
+#        return method.upper()
+
+#    @field_validator('scope', mode='before')
+#    @classmethod
+#    def validate_scope(cls, scope):
+#        """Enforce lowercase for scope."""
+#        if isinstance(scope, str):
+#            return scope.lower()
+#        return scope

CHAP/foxden/processor.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+"""
+File       : processor.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: Processor module for FOXDEN services
+"""
+
+# Local modules
+from CHAP.common.utils import (
+    osinfo,
+    environments,
+)
+from CHAP.processor import Processor
+
+
+#class FoxdenMetadataProcessor(Processor):
+#    """Processor to collect CHAP workflow metadata from a workflow
+#    NeXus output object.
+#    """
+#    def process(self, data):
+#        """Extract metadata from a workflow NeXus output object for
+#        submission to the FOXDEN Metadata service.
+#
+#        :param data: Input data.
+#        :type data: list[PipelineData]
+#        :return: CHAP workflow metadata record.
+#        :rtype: dict
+#        """
+#        # Third party modules
+#        from json import loads
+#        from nexusformat.nexus import (
+#            NXentry,
+#            NXroot,
+#        )
+#
+#        # Load and validate the workflow NeXus output object
+#        nxentry = self.get_data(data, remove=False)
+#        if isinstance(nxentry, NXroot):
+#            nxentry = nxentry[nxentry.default]
+#        if not isinstance(nxentry, NXentry):
+#            raise ValueError(f'Invalid input data type {type(nxentry)}')
+#
+#        # Get did and experiment type
+#        map_config = loads(str(nxentry.map_config))
+#        did = map_config['did']
+#        experiment_type = map_config['experiment_type']
+#
+#        # Extract metadata
+#        method = getattr(self, f'_get_metadata_{experiment_type.lower()}')
+#        metadata = method(nxentry)
+#
+#        if 'reconstructed_data' in metadata:
+#            did = f'{did}/{experiment_type.lower()}_reconstructed'
+#        else:
+#            did = f'{did}/{experiment_type.lower()}_reduced'
+#        return {'did': did, 'application': 'CHAP', 'metadata': metadata}
+#
+#    def _get_metadata_tomo(self, nxentry):
+#        metadata = {}
+#        if 'reduced_data' in nxentry:
+#            data = nxentry.reduced_data
+#            metadata.update({
+#                'reduced_data': {
+#                    'date': str(data.date),
+#                    'img_row_bounds': data.img_row_bounds.tolist(),
+#                }
+#            })
+#        if 'reconstructed_data' in nxentry:
+#            data = nxentry.reconstructed_data
+#            metadata.update({
+#                'reconstructed_data': {
+#                    'date': str(data.date),
+#                    'center_offsets': data.center_offsets.tolist(),
+#                    'center_rows': data.center_offsets.tolist(),
+#                    'center_stack_index': int(data.center_stack_index),
+#                    'x_bounds': data.x_bounds.tolist(),
+#                    'y_bounds': data.y_bounds.tolist(),
+#                }
+#            })
+#        if 'combined_data' in nxentry:
+#            data = nxentry.combined_data
+#            metadata.update({
+#                'combined_data': {
+#                    'date': str(data.date),
+#                }
+#            })
+#        return metadata
+
+
+class FoxdenProvenanceProcessor(Processor):
+    """Processor to collect CHAP workflow provenance data."""
+    def process(self, data):
+        """Extract provenance data from the pipeline data for
+        submission to the FOXDEN Provenance service.
+
+        :param data: Input data.
+        :type data: list[PipelineData]
+        :return: CHAP workflow provenance record.
+        :rtype: dict
+        """
+        # Load the provenance info
+        provenance = self.get_data(data, schema='provenance')
+
+        # Add system info to provenance data
+        did = provenance['did']
+        provenance.update({
+            'parent_did': did.rsplit('/', 1)[0],
+            'scripts': [
+                {'name': 'CHAP', 'parent_script': None, 'order_idx': 1}],
+            'site': 'Cornell',
+            'osinfo': osinfo(),
+            'environments': environments(),
+            'processing': 'CHAP pipeline',
+        })
+
+        return provenance
+
+
+if __name__ == '__main__':
+    # Local modules
+    from CHAP.processor import main
+
+    main()

CHAP/foxden/reader.py

@@ -0,0 +1,224 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+"""
+File       : reader.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: FOXDEN readers
+"""
+
+# System modules
+import json
+
+# Local modules
+from CHAP.pipeline import PipelineItem
+from CHAP.foxden.utils import HttpRequest
+
+
+class FoxdenDataDiscoveryReader(PipelineItem):
+    """Reader for the FOXDEN Data Discovery service."""
+    def read(self, config):
+        """Read records from the FOXDEN Data Discovery service based on
+        did or an arbitrary query.
+
+        :param config: FOXDEN HTTP request configuration.
+        :type config: CHAP.foxden.models.FoxdenRequestConfig
+        :return: Discovered data records.
+        :rtype: list
+        """
+        # Load and validate the FoxdenRequestConfig configuration
+        config = self.get_config(
+            config=config, schema='foxden.models.FoxdenRequestConfig')
+        self.logger.debug(f'config: {config}')
+
+        # Submit HTTP request and return response
+        rurl = f'{config.url}/search'
+        request = {'client': 'CHAP-FoxdenDataDiscoveryReader'}
+        if config.did is None:
+            if config.query is None:
+                query = '{}'
+            else:
+                query = config.query
+            request['service_query'] = {'query': query, 'limit': config.limit}
+        else:
+            if config.limit is not None:
+                self.logger.warning(
+                    f'Ignoring parameter "limit" ({config.limit}), '
+                    'when "did" is specified')
+            if config.query is not None:
+                self.logger.warning(
+                    f'Ignoring parameter "query" ({config.query}), '
+                    'when "did" is specified')
+            request['service_query'] = {'query': f'did:{config.did}'}
+        payload = json.dumps(request)
+        self.logger.info(f'method=POST url={rurl} payload={payload}')
+        response = HttpRequest(rurl, payload, method='POST', scope='read')
+        if config.verbose:
+            self.logger.info(
+                f'code={response.status_code} data={response.text}')
+        if response.status_code == 200:
+            result = json.loads(response.text)['results']['records']
+        else:
+            self.logger.warning(f'HTTP error code {response.status_code}')
+            result = []
+        self.logger.debug(f'Returning {len(result)} records')
+        return result
+
+
+class FoxdenMetadataReader(PipelineItem):
+    """Reader for the FOXDEN Metadata service."""
+    def read(self, config):
+        """Read records from the FOXDEN Metadata service based on did
+        or an arbitrary query.
+
+        :param config: FOXDEN HTTP request configuration.
+        :type config: CHAP.foxden.models.FoxdenRequestConfig
+        :return: Metadata records.
+        :rtype: list
+        """
+        # Load and validate the FoxdenRequestConfig configuration
+        config = self.get_config(
+            config=config, schema='foxden.models.FoxdenRequestConfig')
+        self.logger.debug(f'config: {config}')
+
+        # Submit HTTP request and return response
+        rurl = f'{config.url}/search'
+        request = {'client': 'CHAP-FoxdenMetadataReader'}
+        if config.did is None:
+            if config.query is None:
+                query = '{}'
+            else:
+                query = config.query
+            request['service_query'] = {'query': query}
+        else:
+            if config.query is not None:
+                self.logger.warning(
+                    f'Ignoring parameter "query" ({config.query}), '
+                    'when "did" is specified')
+            request['service_query'] = {'query': f'did:{config.did}'}
+        payload = json.dumps(request)
+        self.logger.info(f'method=POST url={rurl} payload={payload}')
+        response = HttpRequest(rurl, payload, method='POST', scope='read')
+        if config.verbose:
+            self.logger.info(
+                f'code={response.status_code} data={response.text}')
+        if response.status_code == 200:
+            result = json.loads(response.text)
+        else:
+            self.logger.warning(f'HTTP error code {response.status_code}')
+            result = []
+        self.logger.debug(f'Returning {len(result)} records')
+        return result
+
+
+class FoxdenProvenanceReader(PipelineItem):
+    """Reader for FOXDEN Provenance data from a specific FOXDEN
+    Provenance service.
+    """
+    def read(self, config):
+        """Read records from the FOXDEN Provenance service based on did
+        or an arbitrary query.
+
+        :param config: FOXDEN HTTP request configuration.
+        :type config: CHAP.foxden.models.FoxdenRequestConfig
+        :return: Provenance input and output file records.
+        :rtype: list
+        """
+        # Load and validate the FoxdenRequestConfig configuration
+        config = self.get_config(
+            config=config, schema='foxden.models.FoxdenRequestConfig')
+        self.logger.debug(f'config: {config}')
+
+        # Submit HTTP request and return response
+        rurl = f'{config.url}/files?did={config.did}'
+        request = {'client': 'CHAP-FoxdenProvenanceReader'}
+        if config.did is None:
+            if config.query is None:
+                query = '{}'
+            else:
+                query = config.query
+            request['service_query'] = {'query': query, 'limit': config.limit}
+        else:
+            if config.limit is not None:
+                self.logger.warning(
+                    f'Ignoring parameter "limit" ({config.limit}), '
+                    'when "did" is specified')
+            if config.query is not None:
+                self.logger.warning(
+                    f'Ignoring parameter "query" ({config.query}), '
+                    'when "did" is specified')
+            request['service_query'] = {'query': f'did:{config.did}'}
+        payload = json.dumps(request)
+        self.logger.info(f'method=GET url={rurl} payload={payload}')
+        response = HttpRequest(rurl, payload, method='GET', scope='read')
+        exit(f'\n\nresponse.text:\n{response.text}')
+        if config.verbose:
+            self.logger.info(
+                f'code={response.status_code} data={response.text}')
+        if response.status_code == 200:
+            result = [{'name': v['name'], 'file_type': v['file_type']}
+                      for v in json.loads(response.text)]
+        else:
+            self.logger.warning(f'HTTP error code {response.status_code}')
+            result = []
+        self.logger.debug(f'Returning {len(result)} records')
+        return result
+
+
+class FoxdenSpecScansReader(PipelineItem):
+    """Reader for FOXDEN SpecScans data from a specific FOXDEN
+    SpecScans service.
+    """
+    def read(
+            self, url, data, did='', query='', spec=None, method='POST', # 'GET',
+            verbose=False):
+# TODO FIX
+        """Read and return data from a specific FOXDEN SpecScans
+        service.
+
+        :param url: URL of service.
+        :type url: str
+        :param data: Input data.
+        :type data: list[PipelineData]
+        :param did: FOXDEN dataset identifier (did).
+        :type did: string, optional
+        :param query: FOXDEN query.
+        :type query: string, optional
+        :param spec: FOXDEN spec.
+        :type spec: dictionary, optional
+        :param method: HTTP method to use, `'POST'` for creation or
+            `'PUT'` for update, defaults to `'POST'`.
+        :type method: str, optional
+        :param verbose: Verbose output flag, defaults to `False`.
+        :type verbose: bool, optional
+        :return: Contents of the input data.
+        :rtype: object
+        """
+        self.logger.info(
+            f'Executing "process" with url={url} data={data} did={did}')
+        rurl = f'{url}/search'
+        request = {'client': 'CHAP-FoxdenSpecScansReader', 'service_query': {}}
+        if did:
+            request['service_query'].update({'spec': {'did': did}})
+        if query:
+            request['service_query'].update({'query': query})
+        if spec:
+            request['service_query'].update({'spec': spec})
+        payload = json.dumps(request)
+        if verbose:
+            self.logger.info(f'method={method} url={rurl} payload={payload}')
+        response = HttpRequest(rurl, payload, method=method)
+        if verbose:
+            self.logger.info(
+                f'code={response.status_code} data={response.text}')
+        if response.status_code == 200:
+            data = json.loads(response.text)
+        else:
+            data = []
+        return data
+
+
+if __name__ == '__main__':
+    # Local modules
+    from CHAP.reader import main
+
+    main()

CHAP/foxden/utils.py

@@ -0,0 +1,80 @@
+"""FOXDEN utils module."""
+
+
+def readFoxdenToken(scope):
+    """Obtain a FOXDEN token.
+
+    :param scope: FOXDEN scope: `'read'` or `'write'`.
+    :type scope: string
+    """
+    # System modules
+    import os
+
+    token = os.getenv(f'FOXDEN_{scope.upper()}_TOKEN')
+    if not token:
+        token_file = os.path.join(
+            os.getenv('HOME'), f'.foxden.{scope.lower()}.token')
+        if os.path.exists(token_file):
+            with open(token_file, 'r') as f:
+                token = f.read()
+    return token
+
+def HttpRequest(
+        url, payload, method='POST', headers=None, scope='read', timeout=10,
+        dry_run=False):
+    """Submit a HTTP request to a FOXDEN service
+
+    :param url: URL of service.
+    :type url: str
+    :param payload: HTTP request payload.
+    :type payload: str
+    :param method: HTTP method to use, defaults to `'POST'`.
+    :type method: str, optional
+    :param headers: HTTP headers to use.
+    :type headers: dictionary, optional
+    :param scope: FOXDEN scope: `'read'` or `'write'`,
+        defaults to `'read'`.
+    :type scope: string, optional
+    :param timeout: Timeout of HTTP request, defaults to `10`.
+    :type timeout: str, optional
+    :param dry_run: `dry_run` option to verify HTTP workflow,
+        defaults to `False`.
+    :type dry_run: bool, optional
+    :return: HTTP response.
+    :rtype: requests.models.Response
+    """
+    # Third party modules
+    from requests import (
+        get,
+        post,
+        put,
+        delete,
+    )
+
+    if headers is None:
+        headers = {}
+    if 'Content-Type' not in headers:
+        headers['Content-type'] = 'application/json'
+    if 'Accept' not in headers:
+        headers['Accept'] = 'application/json'
+    token = readFoxdenToken(scope)
+    if token:
+        headers['Authorization'] = f'Bearer {token}'
+    else:
+        raise RuntimeError(f'Unable to obtain token with scope {scope}')
+    if dry_run:
+        print('### HTTP reader call', url, headers, payload)
+        return []
+
+    # Make actual HTTP request to FOXDEN service
+    if method.lower() == 'get':
+        response = get(url, headers=headers, timeout=timeout)
+    elif method.lower() == 'post':
+        response = post(url, headers=headers, timeout=timeout, data=payload)
+    elif method.lower() == 'put':
+        response = put(url, headers=headers, timeout=timeout, data=payload)
+    elif method.lower() == 'delete':
+        response = delete(url, headers=headers, timeout=timeout, data=payload)
+    else:
+        raise ValueError(f'Unsupported method {method}')
+    return response