data_pipeline_api 0.6.0__tar.gz → 0.7.0__tar.gz

{data_pipeline_api-0.6.0 → data_pipeline_api-0.7.0}/.travis.yml

@@ -58,5 +58,7 @@ matrix:
       jdk: openjdk11
       language: java
       script: "./gradlew build"
-      after_success: "./gradlew publish"
+      after_success: 
+        - bash <(curl -s https://codecov.io/bash)
+        - ./gradlew publish
 

{data_pipeline_api-0.6.0 → data_pipeline_api-0.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: data_pipeline_api
-Version: 0.6.0
+Version: 0.7.0
 Summary: Interacts with SCRC data pipeline
 Home-page: UNKNOWN
 Author: SCRC

{data_pipeline_api-0.6.0 → data_pipeline_api-0.7.0}/README.md

@@ -53,6 +53,10 @@ cd data_pipeline_api
 git tag --sort=v:refname
 ```
 
+## Data Registry Interactions
+
+See [registry README](data_pipeline_api/registry/README.md).
+
 ## Releasing a new version
 
 New versions can be released from the master branch at any time. Whenever a git

data_pipeline_api-0.7.0/data_pipeline_api/data_processing_api.py

@@ -0,0 +1,14 @@
+from io import IOBase
+from data_pipeline_api.standard_api import StandardAPI
+
+
+class DataProcessingAPI(StandardAPI):
+    """The DataProcessingAPI extends the StandardAPI with access to external objects.
+    """
+
+    def read_external_object(self, doi_or_unique_name: str, component: str) -> IOBase:
+        """Return an open file handle to read the given external object.
+        """
+        return self.file_api.open_for_read(
+            doi_or_unique_name=doi_or_unique_name, component=component
+        )

data_pipeline_api-0.7.0/data_pipeline_api/file_api.py

@@ -0,0 +1,318 @@
+from io import IOBase
+from uuid import uuid4
+from datetime import datetime
+from pathlib import Path
+from typing import Union, Optional, Any, Iterable, Dict, List
+from dataclasses import dataclass
+from hashlib import sha1
+from logging import getLogger, WARNING, DEBUG
+import yaml
+
+from data_pipeline_api.metadata import Metadata, MetadataKey, log_format_metadata
+from data_pipeline_api.metadata_store import MetadataStore
+from data_pipeline_api.overrides import Overrides
+
+logger = getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class FileAccess:
+    """Represents a generic file access.
+    """
+
+    timestamp: datetime
+    call_metadata: Metadata
+    access_metadata: Metadata
+    path: Path
+
+    def to_access_log_record(
+        self, hash_cache: Optional[Dict[Path, str]] = None
+    ) -> Dict[str, Any]:
+        if hash_cache is None:
+            hash_cache = {}
+        access_metadata = self.access_metadata.copy()
+        if MetadataKey.calculated_hash not in access_metadata:
+            absolute_path = self.path.resolve()
+            try:
+                access_metadata[MetadataKey.calculated_hash] = hash_cache[absolute_path]
+            except KeyError:
+                access_metadata[MetadataKey.calculated_hash] = hash_cache[
+                    absolute_path
+                ] = FileAPI.calculate_hash(absolute_path)
+        return {
+            "timestamp": self.timestamp,
+            "call_metadata": self.call_metadata,
+            "access_metadata": access_metadata,
+        }
+
+
+class ReadAccess(FileAccess):
+    """Represents a file read.
+    """
+
+    def to_access_log_record(
+        self, hash_cache: Optional[Dict[Path, str]] = None
+    ) -> Dict[str, Any]:
+        return dict(type="read", **super().to_access_log_record(hash_cache))
+
+
+@dataclass(frozen=True)
+class WriteAccess(FileAccess):
+    """Represents a file write.
+    """
+
+    file_handle: IOBase
+
+    def to_access_log_record(
+        self, hash_cache: Optional[Dict[Path, str]] = None
+    ) -> Dict[str, Any]:
+        if not self.file_handle.closed:
+            logger.warning(
+                "the file handle to write to %s is still open, attempting to flush",
+                self.path,
+            )
+            self.file_handle.flush()
+        return dict(type="write", **super().to_access_log_record(hash_cache))
+
+
+class FileAPI:
+    RESERVED_RUN_METADATA_KEYS = {
+        "run_id",
+        "open_timestamp",
+        "close_timestamp",
+        "data_directory",
+    }
+
+    @staticmethod
+    def normalise_path(root: Path, path: Path) -> Path:
+        """Normalise a path by prepending a known root if it is not absolute.
+        """
+        return path if path.is_absolute() else root / path
+
+    @staticmethod
+    def calculate_hash(filename: Path, extra_bytes: Optional[bytes] = None) -> str:
+        """Calculate the SHA1 hash of a file, optionally appending extra bytes.
+        """
+        with open(filename, "rb") as file:
+            message = sha1(file.read())
+        if extra_bytes:
+            message.update(extra_bytes)
+        hexdigest = message.hexdigest()
+        logger.debug(
+            "calculated hash of %s as %s", filename, hexdigest,
+        )
+        return hexdigest
+
+    @staticmethod
+    def construct_overrides(overrides: Iterable[Dict[str, Dict[str, Any]]]):
+        """Construct an Overrides given the structure found in the FileAPI config.
+        """
+        return Overrides(
+            (override.get("where", {}), override.get("use", {}))
+            for override in overrides
+        )
+
+    @staticmethod
+    def construct_metadata_store(metadata_filename: Path) -> MetadataStore:
+        """Construct a MetadataStore corresponding to a file.
+        """
+        metadata_store = None
+        if metadata_filename.exists():
+            logger.debug("loading metadata from %s", metadata_filename)
+            with open(metadata_filename) as metadata_store_file:
+                metadata_store = yaml.safe_load(metadata_store_file)
+        else:
+            logger.warning("could not find metadata.yaml")
+        return MetadataStore(metadata_store)
+
+    def __init__(self, config_filename: Optional[Union[Path, str]] = None):
+        """The FileAPI class provides tracked interaction with the filesystem.
+
+        Files are located through a metadata lookup system based on two files: a
+        metadata database file, which lists all known files and their associated
+        metadata, and a configuration file, which provides mechanisms for influencing
+        the metadata lookup process.
+        """
+        self._accesses: List[FileAccess] = []
+        self._run_metadata = {}
+
+        self._open_timestamp = datetime.now()
+        self._run_metadata["open_timestamp"] = self._open_timestamp
+        logger.debug("open_timestamp = %s", self._open_timestamp)
+
+        if config_filename is None:
+            self._config_filename = None
+            self._config = {}
+            self._root = Path()
+        else:
+            self._config_filename = Path(config_filename)
+            with open(self._config_filename) as config_file:
+                self._config = yaml.safe_load(config_file)
+            self._root = self._config_filename.parent
+        logger.debug("config_filename = %s", self._config_filename)
+        logger.debug("root = %s", self._root.resolve())
+
+        self._run_id = self._config.get("run_id", uuid4().hex)
+        self._run_metadata["run_id"] = self._run_id
+        logger.debug("run_id = %s", self._run_id)
+
+        self._fail_on_hash_mismatch = self._config.get("fail_on_hash_mismatch", True)
+        logger.log(
+            DEBUG if self._fail_on_hash_mismatch else WARNING,
+            "fail_on_hash_mismatch = %s",
+            self._fail_on_hash_mismatch,
+        )
+
+        data_directory = self._config.get("data_directory", self._root)
+        self._unnormalised_data_directory = Path(data_directory)
+        self._run_metadata["data_directory"] = str(self._unnormalised_data_directory)
+        self._data_directory = FileAPI.normalise_path(
+            self._root, self._unnormalised_data_directory
+        )
+        logger.debug("data_directory = %s", self._data_directory)
+
+        access_log = self._config.get("access_log", "access-{run_id}.yaml")
+        if access_log is False:
+            self._access_log_path = None
+            logger.warning("disabled access log")
+        else:
+            self._access_log_path = FileAPI.normalise_path(
+                self._root, Path(access_log.format(run_id=self._run_id)),
+            )
+        logger.debug("access_log_path = %s", self._access_log_path)
+
+        self._run_metadata.update(self._config.get("metadata", {}))
+        self._read_overrides = FileAPI.construct_overrides(self._config.get("read", ()))
+        self._write_overrides = FileAPI.construct_overrides(
+            self._config.get("write", ())
+        )
+
+        self.load_metadata_store()
+
+    def load_metadata_store(self):
+        metadata_store_filename = self._data_directory / "metadata.yaml"
+        logger.debug("loading metadata store from %s", metadata_store_filename)
+        self._metadata_store = FileAPI.construct_metadata_store(metadata_store_filename)
+
+    def get_read_metadata(self, metadata: Metadata) -> Metadata:
+        read_metadata = metadata.copy()
+        self._read_overrides.apply(read_metadata)
+        return self._metadata_store.find(read_metadata) or read_metadata
+
+    def open_for_read(self, **call_metadata) -> IOBase:
+        """Return a file open for reading corresponding to the given metadata.
+
+        The file contents are hashed, and a record is made of the read.
+        """
+        logger.debug("starting open_for_read(%s)", log_format_metadata(call_metadata))
+        read_metadata = self.get_read_metadata(call_metadata)
+        path = self._data_directory / read_metadata[MetadataKey.filename]
+        read_metadata[MetadataKey.calculated_hash] = FileAPI.calculate_hash(path)
+
+        if self._fail_on_hash_mismatch:
+            if (
+                read_metadata[MetadataKey.calculated_hash]
+                == read_metadata[MetadataKey.verified_hash]
+            ):
+                logger.debug("verified hash")
+            raise ValueError(
+                (
+                    "calculated hash {calculated_hash} != "
+                    "verified hash {verified_hash}"
+                ).format(**read_metadata)
+            )
+
+        logger.debug("open('%s', mode='rb')", path)
+        file = open(path, mode="rb")
+        self._accesses.append(
+            ReadAccess(
+                timestamp=datetime.now(),
+                call_metadata=call_metadata,
+                access_metadata=read_metadata,
+                path=path,
+            )
+        )
+        logger.info("recorded read(%s)", log_format_metadata(call_metadata))
+        return file
+
+    def get_write_metadata(self, metadata: Metadata) -> Metadata:
+        write_metadata = metadata.copy()
+        self._write_overrides.apply(write_metadata)
+        if MetadataKey.filename not in write_metadata:
+            write_metadata[MetadataKey.filename] = str(
+                Path(write_metadata[MetadataKey.data_product])
+                / "{}.{}".format(self._run_id, write_metadata[MetadataKey.extension])
+            )
+            logger.debug("generated filename %s", write_metadata[MetadataKey.filename])
+        return write_metadata
+
+    def open_for_write(self, **call_metadata) -> IOBase:
+        """Return a file open for update corresponding to the given metadata.
+
+        When the file is closed the file contents are hashed, and a record is made of
+        the write.
+        """
+        logger.debug("starting open_for_write(%s)", log_format_metadata(call_metadata))
+        write_metadata = self.get_write_metadata(call_metadata)
+
+        path = self._data_directory / write_metadata[MetadataKey.filename]
+        if path.exists():
+            mode = "r+b"
+        else:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            mode = "w+b"
+        logger.debug("open('%s', mode='%s')", path, mode)
+        file = open(path, mode=mode)
+        self._accesses.append(
+            WriteAccess(
+                timestamp=datetime.now(),
+                call_metadata=call_metadata,
+                access_metadata=write_metadata,
+                path=path,
+                file_handle=file,
+            )
+        )
+        logger.info("recorded write(%s)", log_format_metadata(call_metadata))
+        return file
+
+    def set_run_metadata(self, key: str, value: Any):
+        """Set run-level metadata.
+        """
+        if key in FileAPI.RESERVED_RUN_METADATA_KEYS:
+            raise ValueError(f"{key} is reserved")
+        self._run_metadata[key] = value
+
+    def get_run_metadata(self, key: str) -> Any:
+        """Get run-level metadata.
+        """
+        return self._run_metadata[key]
+
+    def _generate_access_log(self) -> Dict[str, Any]:
+        calculated_path_hashes = {}
+        return {
+            "run_metadata": dict(close_timestamp=datetime.now(), **self._run_metadata),
+            "config": self._config,
+            "io": [
+                access.to_access_log_record(calculated_path_hashes)
+                for access in self._accesses
+            ],
+        }
+
+    def close(self):
+        """Close the session and write the access log.
+        """
+        if self._access_log_path:
+            with open(self._access_log_path, "w") as output_file:
+                yaml.dump(
+                    self._generate_access_log(), output_file, sort_keys=False,
+                )
+            logger.info("wrote access log")
+        else:
+            logger.warning("did not write access log")
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if exc_type is None:
+            self.close()

{data_pipeline_api-0.6.0 → data_pipeline_api-0.7.0}/data_pipeline_api/file_formats/object_file.py

@@ -77,16 +77,15 @@ def write_table(file: IOBase, component: str, table: Table):
         index=False,
         column_dtypes={
             column: (
-                table[column].values.astype(np.string_).dtype
-                if dtype == "O"
-                else dtype
+                table[column].values.astype(np.string_).dtype if dtype == "O" else dtype
             )
             for column, dtype in table.dtypes.items()
         },
     )
-    get_write_group(file, component).require_dataset(
-        "table", shape=records.shape, dtype=records.dtype, data=records
-    )
+    group = get_write_group(file, component)
+    for dataset in group:
+        del group[dataset]
+    group.create_dataset("table", data=records, track_times=False)
 
 
 DIMENSION_PREFIX = "Dimension_"
@@ -172,42 +171,50 @@ def read_array(file: IOBase, component: str) -> Array:
 def write_array(file: IOBase, component: str, array: Array):
     # TODO : More validation on the inputs?
     group = get_write_group(file, component)
-    group.require_dataset(
-        "array", shape=array.data.shape, dtype=array.data.dtype, data=array.data
-    )
+    for dataset in group:
+        del group[dataset]
+    group.create_dataset("array", data=array.data, track_times=False)
     if array.dimensions is not None:
         for i, dimension in enumerate(array.dimensions, start=1):
             if dimension.title is not None:
-                group.require_dataset(
+                group.create_dataset(
                     f"Dimension_{i}_title",
                     dtype=h5py.string_dtype(),
                     shape=(),
                     data=dimension.title,
+                    track_times=False,
                 )
             if dimension.names is not None:
                 encoded_names = np.char.encode(dimension.names)
-                group.require_dataset(
+                group.create_dataset(
                     f"Dimension_{i}_names",
                     dtype=h5py.string_dtype(),
                     shape=encoded_names.shape,
                     data=encoded_names,
+                    track_times=False,
                 )
             if dimension.values is not None:
                 values = np.array(dimension.values)
-                group.require_dataset(
+                group.create_dataset(
                     f"Dimension_{i}_values",
                     dtype=values.dtype,
                     shape=values.shape,
                     data=values,
+                    track_times=False,
                 )
             if dimension.units is not None:
-                group.require_dataset(
+                group.create_dataset(
                     f"Dimension_{i}_units",
                     dtype=h5py.string_dtype(),
                     shape=(),
                     data=dimension.units,
+                    track_times=False,
                 )
     if array.units is not None:
-        group.require_dataset(
-            "units", dtype=h5py.string_dtype(), shape=(), data=array.units,
+        group.create_dataset(
+            "units",
+            dtype=h5py.string_dtype(),
+            shape=(),
+            data=array.units,
+            track_times=False,
         )

{data_pipeline_api-0.6.0 → data_pipeline_api-0.7.0}/data_pipeline_api/file_formats/parameter_file.py

@@ -79,7 +79,8 @@ def read_distribution(file: TextIOBase, component: str) -> Distribution:
         raise ValueError(f"{parameter['type']} != 'distribution'")
 
 
-def write_distribution(file: TextIOBase, component: str, distribution: Distribution):
+def encode_distribution(distribution: Distribution) -> Dict[str, Any]:
+    """Encode distribution into a serialisable format."""
     shape, loc, scale = distribution.dist._parse_args(
         *distribution.args, **distribution.kwds
     )
@@ -93,8 +94,13 @@ def write_distribution(file: TextIOBase, component: str, distribution: Distribut
         parameter["shape"] = shape[0]
     if scale:
         parameter["scale"] = scale
+    return parameter
+
+
+def write_distribution(file: TextIOBase, component: str, distribution: Distribution):
+    """Write distribution to file under component."""
     write_parameter(
-        file, component, parameter,
+        file, component, encode_distribution(distribution),
     )
 
 

data_pipeline_api-0.7.0/data_pipeline_api/git_info.py

@@ -0,0 +1,56 @@
+from typing import NamedTuple, Optional
+from pathlib import Path
+try:
+    from git import Repo, InvalidGitRepositoryError  # type: ignore
+    git_installed = True
+except ImportError:
+    Repo = None
+    InvalidGitRepositoryError = None
+    git_installed = False
+
+
+class RepoInfo(NamedTuple):
+    """
+    The info needed by the data pipeline API
+    """
+
+    git_sha: str
+    uri: str
+    is_dirty: bool
+
+
+def get_repo(path: Optional[Path] = None) -> Repo:
+    """Attempt to get a repo from path, or the working directory.
+    """
+    if Repo is None:
+        raise ImportError("Could not import git")
+    else:
+        return Repo(path=path, search_parent_directories=True)
+
+
+def get_repo_info(path: Optional[Path] = None, default_repo: str = "") -> RepoInfo:
+    """Get the git sha and uri for path, or the working directory.
+
+    :path:         A path to use to start looking for the root of the repo.
+    :default_repo: A default repo url to use if we cannot obtain one.
+    :return:       A RepoInfo object. If not inside a git repo, is_dirty will be True, git_sha empty and uri will be a
+                   default value
+    """
+    try:
+        repo = get_repo(path)
+        return RepoInfo(
+            git_sha=repo.head.commit.hexsha,
+            uri=next(repo.remote("origin").urls),
+            is_dirty=repo.is_dirty(),
+        )
+    except (ImportError, InvalidGitRepositoryError):
+        return RepoInfo(git_sha="", uri=default_repo, is_dirty=True)
+
+
+def get_path_relative_to_repo(path: Path) -> Path:
+    """Get path relative to the repo which contains it, or raise an exception.
+    """
+    result = get_repo(path).git.execute(['git', 'ls-files', '--full-name', str(path)])
+    if result:
+        return Path(result)
+    raise ValueError(f"Could not get {path} relative to repo")

{data_pipeline_api-0.6.0 → data_pipeline_api-0.7.0}/data_pipeline_api/metadata.py

@@ -19,6 +19,7 @@ class MetadataKey:
     verified_hash = "verified_hash"
     calculated_hash = "calculated_hash"
     accessibility = "accessibility"
+    doi_or_unique_name = "doi_or_unique_name"
 
 
 def value_matches(value_a: Any, value_b: Any) -> bool:

{data_pipeline_api-0.6.0 → data_pipeline_api-0.7.0}/data_pipeline_api/metadata_store.py

@@ -20,24 +20,27 @@ class MetadataRecord(NamedTuple):
 
 
 class MetadataStore:
-    def __init__(self, metadata_sequence: Sequence[Metadata]):
+    def __init__(self, metadata_sequence: Optional[Sequence[Metadata]] = None):
         """The MetadataStore class provides a simple metadata lookup mechanism.
 
         A MetadataStore is initialised with a Sequence of Metadata objects, which can
         then be searched by matching against Metadata fragments.
         """
-        try:
-            self._metadata_records = tuple(
-                MetadataRecord(
-                    metadata=metadata,
-                    version=VersionInfo.parse(metadata[MetadataKey.version])
-                    if MetadataKey.version in metadata
-                    else None,
+        if metadata_sequence is None:
+            self._metadata_records = ()
+        else:
+            try:
+                self._metadata_records = tuple(
+                    MetadataRecord(
+                        metadata=metadata,
+                        version=VersionInfo.parse(metadata[MetadataKey.version])
+                        if MetadataKey.version in metadata
+                        else None,
+                    )
+                    for metadata in metadata_sequence
                 )
-                for metadata in metadata_sequence
-            )
-        except Exception as exception:
-            raise ValueError("invalid metadata") from exception
+            except Exception as exception:
+                raise ValueError("invalid metadata") from exception
 
     def find(self, metadata: Metadata) -> Optional[Metadata]:
         try: