data_pipeline_api 0.2.0__py3-none-any.whl

data_pipeline_api/file_api.py

@@ -0,0 +1,181 @@
+from io import IOBase
+from datetime import datetime
+from pathlib import Path
+from typing import Union
+from functools import wraps
+from hashlib import sha1
+import yaml
+
+from data_pipeline_api.metadata import Metadata, MetadataKey
+from data_pipeline_api.metadata_store import MetadataStore
+from data_pipeline_api.overrides import Overrides
+
+
+class FileAPI:
+    def __init__(
+        self, config_filename: Union[Path, str],
+    ):
+        self._open_timestamp = datetime.now()
+        self._accesses = []
+        config_filename = Path(config_filename)
+
+        try:
+            with open(config_filename) as config_file:
+                self._config = yaml.safe_load(config_file)
+                if "run_id" in self._config:
+                    self.run_id = self._config["run_id"]
+                else:
+                    # Compute a run_id.
+                    with open(config_filename, "rb") as file:
+                        m = sha1(file.read())
+                    m.update(str(self._open_timestamp).encode())
+                    self.run_id = m.hexdigest()
+                self.data_directory = Path(self._config.get("data_directory", "."))
+                self.fail_on_hash_mismatch = self._config.get(
+                    "fail_on_hash_mismatch", True
+                )
+                self._read_overrides = Overrides(
+                    (override.get("where", {}), override.get("use", {}))
+                    for override in self._config.get("read", ())
+                )
+                self._write_overrides = Overrides(
+                    (override.get("where", {}), override.get("use", {}))
+                    for override in self._config.get("write", ())
+                )
+                access_log = self._config.get("access_log", "access-{run_id}.yaml")
+                if access_log is False:
+                    self.access_log_path = None
+                else:
+                    self.access_log_path = Path(access_log.format(run_id=self.run_id))
+                    if not self.access_log_path.is_absolute():
+                        self.access_log_path = (
+                            config_filename.parent / self.access_log_path
+                        )
+
+        except Exception as exception:
+            raise ValueError("could not parse config file") from exception
+
+        if self.data_directory.is_absolute():
+            self.normalised_data_directory = self.data_directory
+        else:
+            self.normalised_data_directory = (
+                config_filename.parent / self.data_directory
+            )
+
+        metadata_filename = self.normalised_data_directory / "metadata.yaml"
+        if metadata_filename.exists():
+            with open(metadata_filename) as metadata_store_file:
+                metadata_store = yaml.safe_load(metadata_store_file)
+        else:
+            metadata_store = {}
+        self._metadata_store = MetadataStore(metadata_store)
+
+    def _record_access(
+        self,
+        access_type: str,
+        call_metadata: Metadata,
+        access_metadata: Metadata,
+        path: Path,
+        verify_hash: bool,
+    ):
+        access_metadata = access_metadata.copy()
+        with open(path, "rb") as file:
+            access_metadata[MetadataKey.calculated_hash] = sha1(file.read()).hexdigest()
+        if verify_hash:
+            if (
+                access_metadata[MetadataKey.calculated_hash]
+                != access_metadata[MetadataKey.verified_hash]
+            ):
+                raise ValueError(
+                    (
+                        "calculated hash {calculated_hash} != "
+                        "verified hash {verified_hash}"
+                    ).format(**access_metadata)
+                )
+        self._accesses.append(
+            {
+                "type": access_type,
+                "timestamp": datetime.now(),
+                "call_metadata": call_metadata,
+                "access_metadata": access_metadata,
+            }
+        )
+
+    def open_for_read(self, **metadata) -> IOBase:
+        """Return a file open for reading corresponding to the given metadata.
+
+        The file contents are hashed, and a record is made of the read.
+        """
+        read_metadata = metadata.copy()
+        self._read_overrides.apply(read_metadata)
+        read_metadata = self._metadata_store.find(read_metadata) or read_metadata
+        try:
+            filename = Path(read_metadata[MetadataKey.filename])
+        except KeyError:
+            raise KeyError(f"could not find {MetadataKey.filename} in {read_metadata}")
+        path = self.normalised_data_directory / filename
+        self._record_access(
+            "read",
+            metadata,
+            read_metadata,
+            path,
+            verify_hash=self.fail_on_hash_mismatch,
+        )
+        return open(path, mode="rb")
+
+    def open_for_write(self, **metadata) -> IOBase:
+        """Return a file open for update corresponding to the given metadata.
+
+        When the file is closed the file contents are hashed, and a record is made of
+        the write.
+        """
+        write_metadata = metadata.copy()
+        self._write_overrides.apply(write_metadata)
+        if MetadataKey.filename not in write_metadata:
+            write_metadata[MetadataKey.filename] = str(
+                Path(write_metadata[MetadataKey.data_product])
+                / "{}.{}".format(self.run_id, write_metadata[MetadataKey.extension],)
+            )
+
+        path = self.normalised_data_directory / write_metadata[MetadataKey.filename]
+        if path.exists():
+            file = open(path, mode="r+b")
+        else:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            file = open(path, mode="w+b")
+        # Wrap the file close method with something to record the file access.
+        close_file = file.close
+
+        @wraps(close_file)
+        def close():
+            file.flush()
+            self._record_access("write", metadata, write_metadata, path, False)
+            return close_file()
+
+        file.close = close
+        return file
+
+    def close(self):
+        """Close the session and write the access log.
+        """
+        if self.access_log_path:
+            with open(self.access_log_path, "w") as output_file:
+                yaml.dump(
+                    {
+                        "data_directory": str(self.data_directory),
+                        "open_timestamp": self._open_timestamp,
+                        "close_timestamp": datetime.now(),
+                        "run_id": self.run_id,
+                        "config": self._config,
+                        "io": self._accesses,
+                    },
+                    output_file,
+                    sort_keys=False,
+                )
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if exc_type is None:
+            self.close()

data_pipeline_api/metadata.py

@@ -0,0 +1,21 @@
+from typing import Mapping, Optional
+from pathlib import Path
+
+Metadata = Mapping[str, str]
+
+class MetadataKey:
+    filename = "filename"
+    data_product = "data_product"
+    extension = "extension"
+    run_id = "run_id"
+    version = "version"
+    verified_hash = "verified_hash"
+    calculated_hash = "calculated_hash"
+
+
+def is_superset(metadataA: Metadata, metadataB: Metadata) -> bool:
+    """Return True if metadataA is a superset of metadataB.
+    """
+    return all(
+        key in metadataA and metadataA[key] == value for key, value in metadataB.items()
+    )

data_pipeline_api/metadata_store.py

@@ -0,0 +1,37 @@
+from typing import Sequence, Optional, NamedTuple
+from operator import attrgetter
+from semver import VersionInfo
+from data_pipeline_api.metadata import Metadata, MetadataKey, is_superset
+
+
+class MetadataRecord(NamedTuple):
+    metadata: Metadata
+    version: Optional[VersionInfo]
+
+
+class MetadataStore:
+    def __init__(self, metadata_sequence: Sequence[Metadata]):
+        try:
+            self._metadata_records = tuple(
+                MetadataRecord(
+                    metadata=metadata,
+                    version=VersionInfo.parse(metadata[MetadataKey.version])
+                    if MetadataKey.version in metadata
+                    else None,
+                )
+                for metadata in metadata_sequence
+            )
+        except Exception as exception:
+            raise ValueError("invalid metadata") from exception
+
+    def find(self, metadata: Metadata) -> Optional[Metadata]:
+        try:
+            return max(
+                filter(
+                    lambda record: is_superset(record.metadata, metadata),
+                    self._metadata_records,
+                ),
+                key=attrgetter("version"),
+            ).metadata
+        except ValueError:
+            return None

data_pipeline_api/overrides.py

@@ -0,0 +1,21 @@
+from typing import Iterable, NamedTuple, Sequence, Tuple
+from data_pipeline_api.metadata import Metadata, is_superset
+
+
+class Override(NamedTuple):
+    where: Metadata
+    use: Metadata
+
+
+class Overrides:
+    def __init__(self, overrides: Sequence[Tuple[Metadata, Metadata]]):
+        self._overrides = tuple(Override(*override) for override in overrides)
+
+    def find(self, metadata: Metadata) -> Iterable[Override]:
+        return filter(
+            lambda override: is_superset(metadata, override.where), self._overrides,
+        )
+
+    def apply(self, metadata: Metadata):
+        for override in self.find(metadata):
+            metadata.update(override.use)

data_pipeline_api/simple_network_sim_api.py

@@ -0,0 +1,13 @@
+from io import TextIOWrapper
+from pathlib import Path
+from pandas import read_csv, DataFrame
+from data_pipeline_api.file_api import FileAPI
+
+class SimpleNetworkSimAPI(FileAPI):
+    def read_table(self, data_product: str) -> DataFrame:
+        with TextIOWrapper(self.open_for_read(data_product=data_product, extension="csv")) as csv_file:
+            return read_csv(csv_file)
+
+    def write_table(self, data_product: str, value: DataFrame):
+        with TextIOWrapper(self.open_for_write(data_product=data_product, extension="csv")) as csv_file:
+            value.to_csv(csv_file, index=False)

data_pipeline_api/standard_api.py

@@ -0,0 +1,136 @@
+from io import TextIOWrapper
+from pathlib import Path
+from contextlib import contextmanager
+import numpy as np
+import pandas as pd
+from data_pipeline_api.file_api import FileAPI
+from data_pipeline_api.file_formats import parameter_file, object_file
+
+
+class StandardAPI(FileAPI):
+    # ==================================================================================
+    # Parameter files
+    # ==================================================================================
+
+    @contextmanager
+    def open_parameter_file_for_read(self, data_product: str, component: str):
+        with TextIOWrapper(
+            self.open_for_read(data_product=data_product, component=component)
+        ) as parameter_file:
+            yield parameter_file
+
+    @contextmanager
+    def open_parameter_file_for_write(self, data_product: str, component: str):
+        with TextIOWrapper(
+            self.open_for_write(
+                data_product=data_product, component=component, extension="toml"
+            )
+        ) as parameter_file:
+            yield parameter_file
+
+    # ----------------------------------------------------------------------------------
+    # Estimate
+    # ----------------------------------------------------------------------------------
+
+    def read_estimate(
+        self, data_product: str, component: str
+    ) -> parameter_file.Estimate:
+        with self.open_parameter_file_for_read(data_product, component) as file:
+            parameter_type = parameter_file.read_type(file, component)
+            if parameter_type is parameter_file.Type.POINT_ESTIMATE:
+                return parameter_file.read_estimate(file, component)
+            elif parameter_type is parameter_file.Type.DISTRIBUTION:
+                return parameter_file.read_distribution(file, component).mean()
+            elif parameter_type is parameter_file.Type.SAMPLES:
+                return parameter_file.read_samples(file, component).mean()
+            else:
+                raise ValueError(f"unrecognised type {parameter_type}")
+
+    def write_estimate(
+        self, data_product: str, component: str, estimate: parameter_file.Estimate
+    ):
+        with self.open_parameter_file_for_write(data_product, component) as file:
+            parameter_file.write_estimate(file, component, estimate)
+
+    # ----------------------------------------------------------------------------------
+    # Distribution
+    # ----------------------------------------------------------------------------------
+
+    def read_distribution(
+        self, data_product: str, component: str
+    ) -> parameter_file.Distribution:
+        with self.open_parameter_file_for_read(data_product, component) as file:
+            parameter_type = parameter_file.read_type(file, component)
+            if parameter_type is parameter_file.Type.POINT_ESTIMATE:
+                raise ValueError("point-estimate cannot be read as a distribution")
+            elif parameter_type is parameter_file.Type.DISTRIBUTION:
+                return parameter_file.read_distribution(file, component)
+            elif parameter_type is parameter_file.Type.SAMPLES:
+                raise ValueError("samples cannot be read as a distribution")
+            else:
+                raise ValueError(f"unrecognised type {parameter_type}")
+
+    def write_distribution(
+        self,
+        data_product: str,
+        component: str,
+        distribution: parameter_file.Distribution,
+    ):
+        with self.open_parameter_file_for_write(data_product, component) as file:
+            parameter_file.write_distribution(file, component, distribution)
+
+    # ----------------------------------------------------------------------------------
+    # Samples
+    # ----------------------------------------------------------------------------------
+
+    def read_sample(self, data_product: str, component: str) -> float:
+        with self.open_parameter_file_for_read(data_product, component) as file:
+            parameter_type = parameter_file.read_type(file, component)
+            if parameter_type is parameter_file.Type.POINT_ESTIMATE:
+                return parameter_file.read_estimate(file, component)
+            elif parameter_type is parameter_file.Type.DISTRIBUTION:
+                return parameter_file.read_distribution(file, component).rvs()
+            elif parameter_type is parameter_file.Type.SAMPLES:
+                return np.random.choice(parameter_file.read_samples(file, component))
+            else:
+                raise ValueError(f"unrecognised type {parameter_type}")
+
+    def write_samples(
+        self, data_product: str, component: str, samples: parameter_file.Samples
+    ):
+        with self.open_parameter_file_for_write(data_product, component) as file:
+            parameter_file.write_samples(file, component, samples)
+
+    # ==================================================================================
+    # Object (hdf5) files
+    # ==================================================================================
+
+    @contextmanager
+    def open_object_file_for_read(self, data_product: str, component: str):
+        with self.open_for_read(
+            data_product=data_product, component=component
+        ) as object_file:
+            yield object_file
+
+    @contextmanager
+    def open_object_file_for_write(self, data_product: str, component: str):
+        with self.open_for_write(
+            data_product=data_product, component=component, extension="h5"
+        ) as object_file:
+            yield object_file
+
+    def read_table(self, data_product: str, component: str) -> object_file.Table:
+        with self.open_object_file_for_read(data_product, component) as file:
+            return object_file.read_table(file, component)
+
+    def write_table(self, data_product: str, component: str, table: object_file.Table):
+        with self.open_object_file_for_write(data_product, component) as file:
+            object_file.write_table(file, component, table)
+
+    def read_array(self, data_product: str, component: str) -> object_file.Array:
+        with self.open_object_file_for_read(data_product, component) as file:
+            return object_file.read_array(file, component)
+
+    def write_array(self, data_product: str, component: str, array: object_file.Array):
+        with self.open_object_file_for_write(data_product, component) as file:
+            object_file.write_array(file, component, array)

data_pipeline_api-0.2.0.dist-info/LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2020, SCRC
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data_pipeline_api-0.2.0.dist-info/METADATA

@@ -0,0 +1,21 @@
+Metadata-Version: 2.1
+Name: data-pipeline-api
+Version: 0.2.0
+Summary: Interacts with SCRC data pipeline
+Home-page: UNKNOWN
+Author: SCRC
+Author-email: scrc@glasgow.ac.uk
+License: UNKNOWN
+Platform: UNKNOWN
+Requires-Dist: networkx (>=2.4)
+Requires-Dist: matplotlib (>=3.1.3)
+Requires-Dist: pandas (>=1.0.3)
+Requires-Dist: toml (>=0.9.4)
+Requires-Dist: h5py (>=2.9.0)
+Requires-Dist: scipy (>=1.4.1)
+Requires-Dist: pyyaml (>=5.3.1)
+Requires-Dist: semver (>=2.9.0)
+
+UNKNOWN
+
+

data_pipeline_api-0.2.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+data_pipeline_api/file_api.py,sha256=dwFovDj-KhVa4jMHjn5FV_lhTLExAMZ_8cqqpdtZ_3s,6860
+data_pipeline_api/metadata.py,sha256=k6Wb1UAouI_IGtB-IGTmMAIlW6B0HRbUFSIljP9wrL4,570
+data_pipeline_api/metadata_store.py,sha256=DXRO2sscGsFpbSVioibjh0cM73qpuoh4mr4v_rfz7RU,1222
+data_pipeline_api/overrides.py,sha256=GXd6SMWfN4mKLUflhJMuTf3w4xx7d7eJqZ2PQD8yyCM,666
+data_pipeline_api/simple_network_sim_api.py,sha256=iKFzxLWTSc0dLIhHfu11UX-osyxO1RO4Kqq8cgP39Co,595
+data_pipeline_api/standard_api.py,sha256=0CEgrn8t4CtY1hsy5gAk4as2UIvVr-QMCYTs7Fc-x58,6414
+data_pipeline_api-0.2.0.dist-info/LICENSE,sha256=ok0qSVR1GQvE2voRomNyZVHlq2aoO_Bo6HCZGCZUeZM,1512
+data_pipeline_api-0.2.0.dist-info/METADATA,sha256=qWB2CiLgppz2V21fR9tbt5zX_rhPDrTrCw_Y6aRY2HM,470
+data_pipeline_api-0.2.0.dist-info/WHEEL,sha256=g4nMs7d-Xl9-xC9XovUrsDHGXt-FT0E17Yqo92DEfvY,92
+data_pipeline_api-0.2.0.dist-info/top_level.txt,sha256=f2wi2a5FngfSqjkTh5YGJg8ty1JsV-GExu6qZOeqlac,18
+data_pipeline_api-0.2.0.dist-info/RECORD,,

data_pipeline_api-0.2.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.34.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

data_pipeline_api-0.2.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+data_pipeline_api