ChessAnalysisPipeline 0.0.17.dev3__py3-none-any.whl

CHAP/utils/parfile.py

@@ -0,0 +1,224 @@
+"""Utilities for interacting with scans using an SMB-style .par file
+as input.
+"""
+
+# System modules
+import csv
+import os
+
+# Third party modules
+import json
+
+class ParFile():
+    """Representation of a .par file.
+
+    :ivar par_file: Name of the .par file.
+    :type par_file: str
+    :ivar json_file: Name of the .json file containing the keys for
+        the column names of the .par file.
+    :type json_file: str
+    :ivar spec_file: Name of the SPEC data file associated with this
+        .par file.
+    :type spec_file: str
+    :ivar column_names: List of the names of each column in the par
+        file.
+    :type column_names: list[str]
+    :ivar data: A 2D array of the data in this .par file. 0th index:
+        row. 1st index: column
+    :type data: list[list]
+    """
+    def __init__(self, par_file, scan_numbers=None, scann_col_name='SCAN_N'):
+        # Local modules
+        from CHAP.utils.general import (
+            is_int_series,
+            string_to_list,
+        )
+
+        self.par_file = str(par_file)
+        self.json_file = self.par_file.replace('.par', '.json')
+        self.spec_file = os.path.join(
+            os.path.dirname(self.par_file), 'spec.log')
+
+        with open(self.json_file) as json_file:
+            columns = json.load(json_file)
+        num_column = len(columns)
+        self.column_names = [None] * num_column
+        for i, name in columns.items():
+            self.column_names[int(i)] = name
+
+        self.data = []
+        with open(self.par_file) as f:
+            reader = csv.reader(f, delimiter=' ')
+            for i, row in enumerate(reader):
+                if len(row) == 0:
+                    continue
+                if row[0].startswith('#'):
+                    continue
+                row_data = []
+                for value in row:
+                    try:
+                        value = int(value)
+                    except ValueError:
+                        try:
+                            value = float(value)
+                        except Exception:
+                            pass
+                    row_data.append(value)
+                if len(row_data) != num_column:
+                    raise ValueError(
+                        'Mismatch between the number of columns in the json '
+                        f'({num_column}) and line {i+1} of the par file '
+                        f'({len(row_data)})')
+                self.data.append(row_data)
+
+        self.scann_i = self.column_names.index(scann_col_name)
+        self.scan_numbers = [data[self.scann_i] for data in self.data]
+        if scan_numbers is not None:
+            if isinstance(scan_numbers, int):
+                scan_numbers = [scan_numbers]
+            elif isinstance(scan_numbers, str):
+                scan_numbers = string_to_list(scan_numbers)
+            if not is_int_series(scan_numbers, ge=0, log=False):
+                raise TypeError(
+                    f'Invalid scan_numbers parameter ({scan_numbers})')
+            self.scan_numbers = [
+                n for n in scan_numbers if n in self.scan_numbers]
+
+    def get_map(
+            self, experiment_type, station, par_dims, other_dims=None):
+        """Return a map configuration based on this par file.
+
+        :param experiment_type: Experiment type name for the map
+            that this .par file represents.
+        :type experiment_type:
+            Literal['SAXSWAXS', 'EDD', 'XRF', 'TOMO']
+        :param station: Station name at which the data were collected.
+        :type station: Literal['id1a3','id3a','id3b']
+        :param par_dims: List of dictionaries configuring the map's
+            independent dimensions.
+        :type par_dims: list[dict[str, str]]
+        :param other_dims: List of other dimensions to include in
+            the returned MapConfig's independednt_dimensions. Use this
+            if each scans in thhis par ile captured more than one
+            frame of data. Defaults to `None`
+        :type other_dims: list[dict[str,str]], optional
+        :return: The map configuration.
+        :rtype: CHAP.common.models.map.MapConfig
+        """
+        # Third party modeuls
+        # pylint: disable=import-error
+        from chess_scanparsers import SMBScanParser
+        # pylint: enable=import-error
+
+        # Local modules
+        from CHAP.common.models.map import MapConfig
+
+        scanparser = SMBScanParser(self.spec_file, 1)
+        good_scans = self.good_scan_numbers()
+        if other_dims is None:
+            other_dims = []
+        map_config = {
+            'title': scanparser.scan_name,
+            'station': station, #scanparser.station,
+            'experiment_type': experiment_type,
+            'sample': {'name': scanparser.scan_name},
+            'spec_scans': [
+                {'spec_file': self.spec_file,
+                 'scan_numbers': good_scans}],
+            'independent_dimensions': [
+                {'label': dim['label'],
+                 'units': dim['units'],
+                 'name': dim['name'],
+                 'data_type': 'smb_par'}
+                for dim in par_dims] + other_dims
+        }
+        return MapConfig(**map_config)
+
+    def good_scan_numbers(self, good_col_name='1/0'):
+        """Return the numbers of scans marked with a "1" in the
+        indicated "good" column of the .par file.
+        
+        :param good_col_name: The name of the "good" column of the par
+            file, defaults to "1/0"
+        :type good_col_name: str, optional
+        :raises ValueError: If this .par file does not have a column
+            with the same name as `good_col_name`.
+        :return: "good" scan numbers.
+        :rtype: list[int]
+        """
+        good_col_i = self.column_names.index(good_col_name)
+        return [self.scan_numbers[i] for i in range(len(self.scan_numbers))
+                if self.data[i][good_col_i] == 1]
+
+    def get_values(self, column, scan_numbers=None):
+        """Return values from a single column of the par file.
+
+        :param column: The string name OR index of the column to return
+            values for.
+        :type column: str or int
+        :param scan_numbers: List of specific scan numbers to return
+            values in the given column for (instead of the default
+            behavior: return the entire column of values).
+        :type scan_numbers: list[int], optional
+        :raise:
+            ValueError: Unavailable column name.
+            TypeError: Illegal column name type.
+        :return: A list of values from a single column in the par file.
+        :rtype: list[object]
+        """
+        if isinstance(column, str):
+            if column in self.column_names:
+                column_idx = self.column_names.index(column)
+#            elif column in ('dataset_id', 'scan_type'):
+#                column_idx = None
+            else:
+                raise ValueError(f'Unavailable column name: {column} not in '
+                                 f'{self.column_names}')
+        elif isinstance(column, int):
+            column_idx = column
+        else:
+            raise TypeError(f'column must be a str or int, not {type(column)}')
+
+        if column_idx is None:
+            column_data = [None]*len(self.data)
+        else:
+            column_data = [
+                self.data[i][column_idx] for i in range(len(self.data))]
+        if scan_numbers is not None:
+            column_data = [column_data[self.scan_numbers.index(scan_n)] \
+                           for scan_n in scan_numbers]
+        return column_data
+
+    def map_values(self, map_config, values):
+        """Return a reshaped array of the 1D list `values` so that it
+        matches up with the coordinates of `map_config`.
+
+        :param map_config: The map configuration according to which
+            values will be reshaped.
+        :type map_config: MapConfig
+        :param values: A 1D list of values to reshape.
+        :type values: list or np.ndarray
+        :return: Reshaped array of values.
+        :rtype: np.ndarray
+        """
+        # Third party modules
+        import numpy as np
+
+        good_scans = self.good_scan_numbers()
+        if len(values) != len(good_scans):
+            raise ValueError('number of values provided ({len(values)}) does '
+                             'not match the number of good scans in '
+                             f'{self.par_file} ({len(good_scans)})')
+        n_map_points = np.prod(map_config.shape)
+        if len(values) != n_map_points:
+            raise ValueError(
+                f'Cannot reshape {len(values)} values into an array of shape '
+                f'{map_config.shape}')
+
+        map_values = np.empty(map_config.shape)
+        for map_index in np.ndindex(map_config.shape):
+            _, scan_number, _ = \
+                map_config.get_scan_step_index(map_index)
+            value_index = good_scans.index(scan_number)
+            map_values[map_index] = values[value_index]
+        return map_values

CHAP/writer.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python
+"""
+File       : writer.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: generic Writer module
+
+Define a generic `Writer` object.
+"""
+
+# System modules
+import argparse
+import logging
+import os
+from sys import modules
+from typing import Optional
+
+# Third party modules
+from pydantic import model_validator
+
+# Local modules
+from CHAP.pipeline import PipelineItem
+
+
+def validate_writer_model(writer):
+    writer.filename = os.path.normpath(os.path.realpath(
+        os.path.join(writer.outputdir, writer.filename)))
+    if (not writer.force_overwrite
+            and os.path.isfile(writer.filename)):
+        raise ValueError(
+            'Writing to an existing file without overwrite permission. '
+            f'permission. Remove {writer.filename} or set '
+            '"force_overwrite" in pipeline configuration for '
+            f'{writer.name}')
+    return writer
+
+
+class Writer(PipelineItem):
+    """Generic file writer.
+
+    The job of any `Writer` in a `Pipeline` is to receive input
+    returned by a previous `PipelineItem`, write its data to a
+    particular file format, then return the same data unaltered so it
+    can be used by a successive `PipelineItem`.
+
+    :ivar filename: Name of file to write to.
+    :type filename: str
+    :ivar force_overwrite: Flag to allow data in `filename` to be
+        overwritten if it already exists, defaults to `False`.
+    :type force_overwrite: bool, optional
+    :ivar remove: Flag to remove the dictionary from `data`,
+        defaults to `False`.
+    :type remove: bool, optional
+    """
+    filename: str
+    force_overwrite: Optional[bool] = False
+    remove: Optional[bool] = False
+
+    _validate_filename = model_validator(mode="after")(
+        validate_writer_model)
+
+    def write(self, data):
+        """Write the last `CHAP.pipeline.PipelineData` item in `data`
+        as text to a file.
+
+        :param data: Input data.
+        :type data: list[CHAP.pipeline.PipelineData]
+        :return: Contents of the input data.
+        :rtype: list[PipelineData]
+        """
+        ddata = self.unwrap_pipelinedata(data)[-1]
+        if os.path.isfile(self.filename) and not self.force_overwrite:
+            raise FileExistsError(f'{self.filename} already exists')
+        with open(self.filename, 'w') as f:
+            f.write(ddata)
+        if self.remove:
+            data.pop()
+        self.status = 'written' # Right now does nothing yet, but could
+                                # add a sort of modification flag later
+        return data
+
+
+class OptionParser():
+    """User based option parser."""
+    def __init__(self):
+        self.parser = argparse.ArgumentParser(prog='PROG')
+        self.parser.add_argument(
+            '--data', action='store',
+            dest='data', default='', help='Input data')
+        self.parser.add_argument(
+            '--filename', action='store',
+            dest='filename', default='', help='Output file')
+        self.parser.add_argument(
+            '--writer', action='store',
+            dest='writer', default='Writer', help='Writer class name')
+        self.parser.add_argument(
+            '--log-level', choices=logging._nameToLevel.keys(),
+            dest='log_level', default='INFO', help='logging level')
+
+
+def main(opt_parser=OptionParser):
+    """Main function."""
+    optmgr = opt_parser()
+    opts = optmgr.parser.parse_args()
+    cls_name = opts.writer
+    try:
+        writer_cls = getattr(modules[__name__], cls_name)
+    except AttributeError:
+        print(f'Unsupported writer {cls_name}')
+        raise
+
+    writer = writer_cls()
+    writer.logger.setLevel(getattr(logging, opts.log_level))
+    log_handler = logging.StreamHandler()
+    log_handler.setFormatter(logging.Formatter(
+        '{name:20}: {message}', style='{'))
+    writer.logger.addHandler(log_handler)
+    data = writer.write(opts.data, opts.filename)
+    print(f'Writer {writer} writes to {opts.filename}, data {data}')
+
+
+if __name__ == '__main__':
+    main()

MLaaS/ktrain.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+#pylint: disable=
+"""
+File       : ktrain.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: Keras based ML network to train over MNIST dataset
+"""
+
+# system modules
+import os
+import sys
+import json
+import gzip
+import pickle
+import argparse
+
+# third-party modules
+import numpy as np
+import tensorflow as tf
+from tensorflow import keras
+from tensorflow.keras import layers
+from tensorflow.keras import backend as K
+from tensorflow.python.tools import saved_model_utils
+
+
+def modelGraph(model_dir):
+    """
+    Provide input/output names used by TF Graph along with graph itself
+    The code is based on TF saved_model_cli.py script.
+    """
+    input_names = []
+    output_names = []
+    tag_sets = saved_model_utils.get_saved_model_tag_sets(model_dir)
+    for tag_set in sorted(tag_sets):
+        print('%r' % ', '.join(sorted(tag_set)))
+        meta_graph_def = saved_model_utils.get_meta_graph_def(model_dir, tag_set[0])
+        for key in meta_graph_def.signature_def.keys():
+            meta = meta_graph_def.signature_def[key]
+            if hasattr(meta, 'inputs') and hasattr(meta, 'outputs'):
+                inputs = meta.inputs
+                outputs = meta.outputs
+                input_signatures = list(meta.inputs.values())
+                input_names = [signature.name for signature in input_signatures]
+                if len(input_names) > 0:
+                    output_signatures = list(meta.outputs.values())
+                    output_names = [signature.name for signature in output_signatures]
+    return input_names, output_names, meta_graph_def
+
+def readData(fin, num_classes):
+    """
+    Helper function to read MNIST data and provide it to
+    upstream code, e.g. to the training layer
+    """
+    # Load the data and split it between train and test sets
+#     (x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()
+    f = gzip.open(fin, 'rb')
+    if sys.version_info < (3,):
+        mnist_data = pickle.load(f)
+    else:
+        mnist_data = pickle.load(f, encoding='bytes')
+    f.close()
+    (x_train, y_train), (x_test, y_test) = mnist_data
+
+    # Scale images to the [0, 1] range
+    x_train = x_train.astype("float32") / 255
+    x_test = x_test.astype("float32") / 255
+    # Make sure images have shape (28, 28, 1)
+    x_train = np.expand_dims(x_train, -1)
+    x_test = np.expand_dims(x_test, -1)
+    print("x_train shape:", x_train.shape)
+    print(x_train.shape[0], "train samples")
+    print(x_test.shape[0], "test samples")
+
+
+    # convert class vectors to binary class matrices
+    y_train = keras.utils.to_categorical(y_train, num_classes)
+    y_test = keras.utils.to_categorical(y_test, num_classes)
+    return x_train, y_train, x_test, y_test
+
+
+def train(fin, fout=None, model_name=None, epochs=1, batch_size=128, h5=False):
+    """
+    train function for MNIST
+    """
+    # Model / data parameters
+    num_classes = 10
+    input_shape = (28, 28, 1)
+
+    # create ML model
+    model = keras.Sequential(
+        [
+            keras.Input(shape=input_shape),
+            layers.Conv2D(32, kernel_size=(3, 3), activation="relu"),
+            layers.MaxPooling2D(pool_size=(2, 2)),
+            layers.Conv2D(64, kernel_size=(3, 3), activation="relu"),
+            layers.MaxPooling2D(pool_size=(2, 2)),
+            layers.Flatten(),
+            layers.Dropout(0.5),
+            layers.Dense(num_classes, activation="softmax"),
+        ]
+    )
+
+    model.summary()
+    print("model input", model.input, type(model.input), model.input.__dict__)
+    print("model output", model.output, type(model.output), model.output.__dict__)
+    model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"])
+
+    # train model
+    x_train, y_train, x_test, y_test = readData(fin, num_classes)
+    model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_split=0.1)
+
+    # evaluate trained model
+    score = model.evaluate(x_test, y_test, verbose=0)
+    print("Test loss:", score[0])
+    print("Test accuracy:", score[1])
+    print("save model to", fout)
+    writer(fout, model_name, model, input_shape, h5)
+
+def writer(fout, model_name, model, input_shape, h5=False):
+    """
+    Writer provide write function for given model
+    """
+    if not fout:
+        return
+    model.save(fout)
+    if h5:
+        model.save('{}/{}'.format(fout, h5), save_format='h5')
+    pbModel = '{}/saved_model.pb'.format(fout)
+    pbtxtModel = '{}/saved_model.pbtxt'.format(fout)
+    convert(pbModel, pbtxtModel)
+
+    # get meta-data information about our ML model
+    input_names, output_names, model_graph = modelGraph(model_name)
+    print("### input", input_names)
+    print("### output", output_names)
+    # ML uses (28,28,1) shape, i.e. 28x28 black-white images
+    # if we'll use color images we'll use shape (28, 28, 3)
+    img_channels = input_shape[2]  # last item represent number of colors
+    meta = {'name': model_name,
+            'model': 'saved_model.pb',
+            'labels': 'labels.txt',
+            'img_channels': img_channels,
+            'input_name': input_names[0].split(':')[0],
+            'output_name': output_names[0].split(':')[0],
+            'input_node': model.input.name,
+            'output_node': model.output.name
+    }
+    with open(fout+'/params.json', 'w') as ostream:
+        ostream.write(json.dumps(meta))
+    with open(fout+'/labels.txt', 'w') as ostream:
+        for i in range(0, 10):
+            ostream.write(str(i)+'\n')
+    with open(fout + '/model.graph', 'wb') as ostream:
+        ostream.write(model_graph.SerializeToString())
+
+def convert(fin, fout):
+    """
+    convert input model.pb into output model.pbtxt
+    Based on internet search:
+    - https://www.tensorflow.org/guide/saved_model
+    - https://www.programcreek.com/python/example/123317/tensorflow.core.protobuf.saved_model_pb2.SavedModel
+    """
+    import google.protobuf
+    from tensorflow.core.protobuf import saved_model_pb2
+    import tensorflow as tf
+
+    saved_model = saved_model_pb2.SavedModel()
+
+    with open(fin, 'rb') as f:
+        saved_model.ParseFromString(f.read())
+        
+    with open(fout, 'w') as f:
+        f.write(google.protobuf.text_format.MessageToString(saved_model))
+
+
+class OptionParser():
+    def __init__(self):
+        "User based option parser"
+        self.parser = argparse.ArgumentParser(prog='PROG')
+        self.parser.add_argument("--fin", action="store",
+            dest="fin", default="", help="Input MNIST file")
+        self.parser.add_argument("--fout", action="store",
+            dest="fout", default="", help="Output models area")
+        self.parser.add_argument("--model", action="store",
+            dest="model", default="mnist", help="model name")
+        self.parser.add_argument("--epochs", action="store",
+            dest="epochs", default=1, help="number of epochs to use in ML training")
+        self.parser.add_argument("--batch_size", action="store",
+            dest="batch_size", default=128, help="batch size to use in training")
+        self.parser.add_argument("--h5", action="store",
+            dest="h5", default="mnist", help="h5 model file name")
+
+def main():
+    "Main function"
+    optmgr  = OptionParser()
+    opts = optmgr.parser.parse_args()
+    train(opts.fin, opts.fout,
+          model_name=opts.model,
+          epochs=opts.epochs,
+          batch_size=opts.batch_size,
+          h5=opts.h5)
+
+if __name__ == '__main__':
+    main()

MLaaS/mnist_img.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+#pylint: disable=
+"""
+File       : mnist_img.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: 
+"""
+
+import json
+import gzip
+import argparse
+# from itertools import chain
+
+import numpy as np
+import matplotlib.pyplot as plt
+
+
+def readImage(fname, fout, num_images=5, imgId=2):
+    """
+    Helper function to read MNIST image
+    """
+    image_size = 28
+    with gzip.open(fname, 'r') as fstream:
+        fstream.read(16)
+        buf = fstream.read(image_size * image_size * num_images)
+        data = np.frombuffer(buf, dtype=np.uint8).astype(np.float32)
+        data = data.reshape(num_images, image_size, image_size, 1)
+        image = np.asarray(data[imgId]).squeeze()
+        plt.imsave(fout, image)
+        print("read:", fname, "wrote:", fout, "image:", type(image), "shape:", image.shape)
+
+def img2json(image):
+    """
+    Convert given image to JSON data format used by TFaaS
+    """
+    # values = [int(i) for i in list(chain.from_iterable(image))]
+    # values = image.tolist()
+    values = []
+    for row in image.tolist():
+        row = [int(i) for i in row]
+        vals = [[i] for i in row]
+        values.append(vals)
+    # final values should be an array of elements, e.g. single image representation
+    values = [values]
+    keys = [str(i) for i in range(0, 10)]
+    meta = {
+        'keys': keys,
+        'values': values,
+        'model': 'mnist'
+    }
+    with open('img.json', 'w') as ostream:
+        ostream.write(json.dumps(meta))
+
+
+class OptionParser():
+    def __init__(self):
+        "User based option parser"
+        fname = "train-images-idx3-ubyte.gz"
+        self.parser = argparse.ArgumentParser(prog='PROG')
+        self.parser.add_argument("--fin", action="store",
+            dest="fin", default=fname, help=f"Input MNIST file, default {fname}")
+        self.parser.add_argument("--fout", action="store",
+            dest="fout", default="img.png", help="Output image fila name, default img.png")
+        self.parser.add_argument("--nimages", action="store",
+            dest="nimages", default=5, help="number of images to read, default 5")
+        self.parser.add_argument("--imgid", action="store",
+            dest="imgid", default=2, help="image index to use from nimages, default 2 (number 4)")
+
+def main():
+    """
+    main function to produce image file from mnist dataset.
+    MNIST dataset can be downloaded from
+    curl -O http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz
+    """
+    optmgr  = OptionParser()
+    opts = optmgr.parser.parse_args()
+    num_images = int(opts.nimages)
+    imgId = int(opts.imgid)
+    img = readImage(opts.fin, opts.fout, num_images, imgId)
+
+if __name__ == '__main__':
+    main()