data_pipeline_api 0.7.1__tar.gz → 0.7.3__tar.gz

{data_pipeline_api-0.7.1 → data_pipeline_api-0.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: data_pipeline_api
-Version: 0.7.1
+Version: 0.7.3
 Summary: Interacts with SCRC data pipeline
 Home-page: UNKNOWN
 Author: SCRC

{data_pipeline_api-0.7.1 → data_pipeline_api-0.7.3}/README.md

@@ -36,7 +36,7 @@ This is most useful for testing and development.
 A packaged version can be installed via [Anaconda Cloud](https://anaconda.org/scottishcovidresponse/data_pipeline_api) for use in other applications:
 
 ```bash
-conda install -c ScottishCovidResponse data-pipeline-api
+conda install -c scottishcovidresponse data-pipeline-api
 ```
 
 ### Pip

data_pipeline_api-0.7.3/data_pipeline_api/data_processing_api.py

@@ -0,0 +1,18 @@
+from io import IOBase
+from typing import Optional
+from data_pipeline_api.standard_api import StandardAPI
+
+
+class DataProcessingAPI(StandardAPI):
+    """The DataProcessingAPI extends the StandardAPI with access to external objects.
+    """
+
+    def read_external_object(
+        self, doi_or_unique_name: str, title: str, component: Optional[str] = None,
+    ) -> IOBase:
+        """Return an open file handle to read the given external object.
+        """
+        kwds = dict(doi_or_unique_name=doi_or_unique_name, title=title)
+        if component is not None:
+            kwds["component"] = component
+        return self.file_api.open_for_read(**kwds)

{data_pipeline_api-0.7.1 → data_pipeline_api-0.7.3}/data_pipeline_api/database_file_api.py

@@ -18,8 +18,6 @@ class DatabaseFileAPI(FileAPI):
     RUN_METADATA_NEEDED_FOR_UPLOAD = {
         RunMetadata.data_directory,
         RunMetadata.run_id,
-        RunMetadata.model_version,
-        RunMetadata.model_name,
         RunMetadata.open_timestamp,
         RunMetadata.git_sha,
         RunMetadata.git_repo,

{data_pipeline_api-0.7.1 → data_pipeline_api-0.7.3}/data_pipeline_api/file_api.py

@@ -202,7 +202,10 @@ class FileAPI:
         self._run_metadata["run_id"] = self._run_id
         self._run_metadata["data_directory"] = str(self._unnormalised_data_directory)
         self._run_metadata["open_timestamp"] = self._open_timestamp
-        self._run_metadata.update(self._config.get("run_metadata", {}))
+        run_metadata = self._config.get("run_metadata", {})
+        if any(key in run_metadata for key in FileAPI.RESERVED_RUN_METADATA_KEYS):
+            raise ValueError(f"reserved key {key} is set in run_metadata")
+        self._run_metadata.update(run_metadata)
 
         self._read_overrides = FileAPI.construct_overrides(self._config.get("read", ()))
         self._write_overrides = FileAPI.construct_overrides(
@@ -237,12 +240,13 @@ class FileAPI:
                 == read_metadata[MetadataKey.verified_hash]
             ):
                 logger.debug("verified hash")
-            raise ValueError(
-                (
-                    "calculated hash {calculated_hash} != "
-                    "verified hash {verified_hash}"
-                ).format(**read_metadata)
-            )
+            else:
+                raise ValueError(
+                    (
+                        "calculated hash {calculated_hash} != "
+                        "verified hash {verified_hash}"
+                    ).format(**read_metadata)
+                )
 
         logger.debug("open('%s', mode='rb')", path)
         file = open(path, mode="rb")

data_pipeline_api-0.7.3/data_pipeline_api/file_formats/parameter_file.py

@@ -0,0 +1,188 @@
+from io import TextIOBase
+from enum import Enum
+from numbers import Real
+from typing import Union, Dict, Any, Tuple
+import toml
+import numpy as np
+from scipy import stats
+
+# ======================================================================================
+# Common
+# ======================================================================================
+
+ParameterComponent = Dict[str, Any]
+Estimate = Real
+Distribution = Union[stats.rv_discrete, stats.rv_continuous]
+Samples = np.ndarray
+
+
+class ParameterType(Enum):
+    POINT_ESTIMATE = "point-estimate"
+    DISTRIBUTION = "distribution"
+    SAMPLES = "samples"
+
+
+def read_parameter(file: TextIOBase, component: str) -> ParameterComponent:
+    file.seek(0)
+    return toml.load(file)[component]
+
+
+def write_parameter(file: TextIOBase, component: str, parameter: ParameterComponent):
+    parameter_data = toml.load(file)
+    parameter_data[component] = parameter
+    file.seek(0)
+    file.truncate()
+    toml.dump(parameter_data, file)
+
+
+def read_type(file: TextIOBase, component: str) -> ParameterType:
+    parameter = read_parameter(file, component)
+    return ParameterType(parameter["type"])
+
+
+# ======================================================================================
+# Estimate
+# ======================================================================================
+
+
+def read_estimate(file: TextIOBase, component: str) -> Estimate:
+    parameter = read_parameter(file, component)
+    if ParameterType(parameter["type"]) is ParameterType.POINT_ESTIMATE:
+        # TODO : validate
+        return parameter["value"]
+    else:
+        raise ValueError(f"{parameter['type']} != 'point-estimate'")
+
+
+def write_estimate(file: TextIOBase, component: str, estimate: Estimate):
+    write_parameter(
+        file, component, {"type": "point-estimate", "value": float(estimate)}
+    )
+
+
+# ======================================================================================
+# Distribution
+# ======================================================================================
+
+
+class Categorical(stats._multivariate.multinomial_frozen):
+    """A scipy-compatible categorical distribution, built on top of a multinomial.
+    """
+
+    def __init__(self, categories, p):
+        super().__init__(n=1, p=p)
+        self.categories = np.array(categories)
+
+    def rvs(self, size=1, random_state=None):
+        return self.categories[
+            super().rvs(size=size, random_state=random_state).argmax(axis=-1)
+        ]
+
+
+def distribution_parameters(
+    distribution: Distribution,
+) -> Tuple[Tuple[float, ...], float, float]:
+    """Extract the parameters from a scipy rv_frozen object, in the form
+    ((shape, ...), loc, scale).
+    """
+    return distribution.dist._parse_args(*distribution.args, **distribution.kwds)
+
+
+# Mapping between scipy and standard names.
+distribution_name_mapping = {
+    "gamma": "gamma",
+    "norm": "normal",
+    "uniform": "uniform",
+    "poisson": "poisson",
+    "expon": "exponential",
+    "beta": "beta",
+    "binom": "binomial",
+}
+
+# Functions to encode (scipy) distribution parameters.
+distribution_parameter_encoders = {
+    "gamma": lambda shapes, loc, scale: dict(k=shapes[0], theta=scale),
+    "normal": lambda shapes, loc, scale: dict(mu=loc, sigma=scale),
+    "uniform": lambda shapes, loc, scale: dict(a=loc, b=loc + scale),
+    "poisson": lambda shapes, loc, scale: {"lambda": shapes[0]},
+    "exponential": lambda shapes, loc, scale: {"lambda": 1 / scale},
+    "beta": lambda shapes, loc, scale: dict(alpha=shapes[0], beta=shapes[1]),
+    "binomial": lambda shapes, loc, scale: dict(n=shapes[0], p=shapes[1]),
+}
+
+
+def encode_distribution(distribution: Distribution) -> Dict[str, Any]:
+    """Encode distribution into a serialisable format."""
+    if isinstance(distribution, Categorical):
+        name = "categorical"
+        encoded_parameters = {
+            "bins": [str(c) for c in distribution.categories],
+            "weights": list(distribution.p),
+        }
+    elif isinstance(distribution, stats._multivariate.multinomial_frozen):
+        name = "multinomial"
+        encoded_parameters = {
+            "n": distribution.n,
+            "p": list(distribution.p),
+        }
+    elif isinstance(distribution, stats.distributions.rv_frozen):
+        name = distribution_name_mapping[distribution.dist.name]
+        encoded_parameters = distribution_parameter_encoders[name](
+            *distribution_parameters(distribution)
+        )
+    else:
+        raise ValueError(f"Do not have a codec for {distribution}")
+    return dict(type="distribution", distribution=name, **encoded_parameters)
+
+
+# Functions to decode serialised distributions.
+distribution_decoders = {
+    "categorical": lambda data: Categorical(data["bins"], data["weights"]),
+    "gamma": lambda data: stats.gamma(data["k"], scale=data["theta"]),
+    "normal": lambda data: stats.norm(data["mu"], data["sigma"]),
+    "uniform": lambda data: stats.uniform(data["a"], data["b"] - data["a"]),
+    "poisson": lambda data: stats.poisson(data["lambda"]),
+    "exponential": lambda data: stats.expon(scale=1 / data["lambda"]),
+    "beta": lambda data: stats.beta(data["alpha"], data["beta"]),
+    "binomial": lambda data: stats.binom(data["n"], data["p"]),
+    "multinomial": lambda data: stats.multinomial(data["n"], data["p"]),
+}
+
+
+def decode_distribution(encoded_distribution: Dict[str, Any]) -> Distribution:
+    """Decode distribution from serialised format.
+    """
+    return distribution_decoders[encoded_distribution["distribution"]](
+        encoded_distribution
+    )
+
+
+def read_distribution(file: TextIOBase, component: str) -> Distribution:
+    parameter = read_parameter(file, component)
+    if ParameterType(parameter["type"]) is ParameterType.DISTRIBUTION:
+        return decode_distribution(parameter)
+    raise ValueError(f"{parameter['type']} != 'distribution'")
+
+
+def write_distribution(file: TextIOBase, component: str, distribution: Distribution):
+    """Write distribution to file under component."""
+    write_parameter(
+        file, component, encode_distribution(distribution),
+    )
+
+
+# ======================================================================================
+# Samples
+# ======================================================================================
+
+
+def read_samples(file: TextIOBase, component: str) -> Samples:
+    parameter = read_parameter(file, component)
+    if ParameterType(parameter["type"]) is ParameterType.SAMPLES:
+        return np.array(parameter["samples"])
+    else:
+        raise ValueError(f"{parameter['type']} != 'samples'")
+
+
+def write_samples(file: TextIOBase, component: str, samples: Samples):
+    write_parameter(file, component, {"type": "samples", "samples": samples.tolist()})

{data_pipeline_api-0.7.1 → data_pipeline_api-0.7.3}/data_pipeline_api/metadata.py

@@ -4,7 +4,6 @@ from typing import Mapping, Any
 Metadata = Mapping[str, Any]
 
 
-
 class MetadataKey:
     """Metadata key constants.
     """
@@ -20,6 +19,7 @@ class MetadataKey:
     calculated_hash = "calculated_hash"
     accessibility = "accessibility"
     doi_or_unique_name = "doi_or_unique_name"
+    title = "title"
 
 
 def value_matches(value_a: Any, value_b: Any) -> bool:

{data_pipeline_api-0.7.1 → data_pipeline_api-0.7.3}/data_pipeline_api/registry/access_upload.py

@@ -189,7 +189,6 @@ def _add_data_product_output_posts(
     path: str,
     data_product_name: str,
     namespace_str: str,
-    model_version_str: str,
     run_id: str,
     component_name: str,
     calculated_hash: str,
@@ -203,7 +202,6 @@ def _add_data_product_output_posts(
     :param path: StorageLocation path
     :param data_product_name: Name of the output data product
     :param namespace_str: namespace that the data product is a member of
-    :param model_version_str: version number of the model
     :param run_id: id of the run
     :param component_name: name of the output component
     :param calculated_hash: calculated hash of the output data product file
@@ -229,7 +227,7 @@ def _add_data_product_output_posts(
         DataRegistryTarget.data_product,
         {
             DataRegistryField.name: data_product_name,
-            DataRegistryField.version: f"{model_version_str}+{run_id}",
+            DataRegistryField.version: f"{run_id}",
             DataRegistryField.namespace: namespace,
             DataRegistryField.object: obj,
         },
@@ -327,21 +325,21 @@ def to_github_uri(input_uri: str, git_sha: str = "master") -> str:
 
 def _add_code_repo(
     posts: List[YamlDict],
-    model_name: str,
-    model_version: str,
     model_git_sha: str,
     model_uri: str,
+    model_name: Optional[str],
+    model_version: Optional[str],
 ) -> YamlDict:
     """
-    Creates a code_repo_release and storage_location for the model and adds them to the list of objects to post to the
-    data registry
+    Creates a code_repo and storage_location, and optionally a code_repo_release for the
+    model and adds them to the list of objects to post to the data registry
 
     :param posts: List of posts to the data registry, will be modified
-    :param model_name: name of the model
-    :param model_version: version of the model
     :param model_git_sha: git sha of the model
     :param model_uri: uri that the model is located at
-    :return: the object reference to the code repo release
+    :param model_name: name of the model
+    :param model_version: version of the model
+    :return: the object reference to the code repo
     """
     storage_root_uri = to_github_uri(model_uri, model_git_sha)
 
@@ -368,22 +366,24 @@ def _add_code_repo(
     code_repo_obj = _create_target_data_dict(
         DataRegistryTarget.object,
         {
-            DataRegistryField.description: f"{model_name}+{model_version}",
+            DataRegistryField.description: f"{storage_root_uri}",
             DataRegistryField.storage_location: code_repo_location,
         },
     )
-    code_repo_release = _create_target_data_dict(
-        DataRegistryTarget.code_repo_release,
-        {
-            DataRegistryField.name: model_name,
-            DataRegistryField.version: model_version,
-            DataRegistryField.website: model_uri,
-            DataRegistryField.object: code_repo_obj,
-        },
-    )
     posts.extend(
-        [code_repo_storage, code_repo_location, code_repo_obj, code_repo_release]
+        [code_repo_storage, code_repo_location, code_repo_obj]
     )
+    if model_name is not None and model_version is not None:
+        code_repo_release = _create_target_data_dict(
+            DataRegistryTarget.code_repo_release,
+            {
+                DataRegistryField.name: model_name,
+                DataRegistryField.version: model_version,
+                DataRegistryField.website: model_uri,
+                DataRegistryField.object: code_repo_obj,
+            },
+        )
+        posts.append(code_repo_release)
 
     return code_repo_obj
 
@@ -482,8 +482,8 @@ def upload_model_run(
         data_directory = config_filename.parent / data_directory
     run_id = run_metadata[RunMetadata.run_id]
     namespace = run_metadata.get(RunMetadata.default_output_namespace)
-    model_version_str = run_metadata[RunMetadata.model_version]
-    model_name = run_metadata[RunMetadata.model_name]
+    model_version_str = run_metadata.get(RunMetadata.model_version)
+    model_name = run_metadata.get(RunMetadata.model_name)
     open_timestamp = run_metadata[RunMetadata.open_timestamp]
     model_git_sha = run_metadata[RunMetadata.git_sha]
     model_uri = run_metadata[RunMetadata.git_repo]
@@ -502,7 +502,7 @@ def upload_model_run(
         posts, remote_uri_override, accessibility, data_registry_url, token
     )
     code_repo = _add_code_repo(
-        posts, model_name, model_version_str, model_git_sha, model_uri
+        posts, model_git_sha, model_uri, model_name, model_version_str
     )
 
     for event in config["io"]:
@@ -540,7 +540,6 @@ def upload_model_run(
                     path,
                     data_product_name,
                     event_namespace,
-                    model_version_str,
                     run_id,
                     component,
                     access_calculated_hash,

{data_pipeline_api-0.7.1 → data_pipeline_api-0.7.3}/data_pipeline_api/registry/download.py

@@ -6,6 +6,7 @@ from typing import Dict, List, Union, Optional, Any
 import click
 import yaml
 
+from data_pipeline_api.metadata import MetadataKey
 from data_pipeline_api.registry.common import (
     configure_cli_logging,
     DATA_REGISTRY_ACCESS_TOKEN,
@@ -61,7 +62,10 @@ def _parse_read_config(
 
 
 def download_from_configs(
-    run_metadata: Dict[str, Any], read_configs: ReadConfigs, token: str, root_dir: Optional[Union[Path, str]] = None
+    run_metadata: Dict[str, Any],
+    read_configs: ReadConfigs,
+    token: str,
+    root_dir: Optional[Union[Path, str]] = None,
 ) -> None:
     """
     Iterates through the config read blocks and downloads the relevant data for each block
@@ -85,7 +89,10 @@ def download_from_configs(
             config = read_config["where"].copy()
             config.update(read_config.get("use", {}))
             downloader.add_external_object(
-                config["doi_or_unique_name"], config.get("version")
+                config[MetadataKey.doi_or_unique_name],
+                config.get(MetadataKey.title),
+                config.get(MetadataKey.component),
+                config.get(MetadataKey.version),
             )
         else:
             parsed_config = _parse_read_config(
@@ -113,6 +120,8 @@ def download_from_config_file(config_filename: Union[Path, str], token: str) ->
     with open(config_filename, "r") as cf:
         config = yaml.safe_load(cf)
     run_metadata = config.get("run_metadata", {})
+    # Copy the top level data_directory into the run_metadata.
+    run_metadata[RunMetadata.data_directory] = config["data_directory"]
     read_configs = config.get("read")
     if not read_configs:
         raise ValueError("No read config specified in configuration file")

{data_pipeline_api-0.7.1 → data_pipeline_api-0.7.3}/data_pipeline_api/registry/downloader.py

@@ -72,13 +72,15 @@ class Downloader:
             }
         )
 
-    def add_external_object(self, doi_or_unique_name: str, version: Optional[str] = None) -> None:
+    def add_external_object(self, doi_or_unique_name: str, title: Optional[str] = None, component: Optional[str] = None, version: Optional[str] = None) -> None:
         """
         Registers an external object to be downloaded
         """
         self._external_objects.append(
             {
                 (DataRegistryTarget.external_object, DataRegistryField.doi_or_unique_name): doi_or_unique_name,
+                (DataRegistryTarget.external_object, DataRegistryField.title): title,
+                (DataRegistryTarget.object_component, DataRegistryField.name): component,
                 (DataRegistryTarget.external_object, DataRegistryField.version): version,
             }
         )
@@ -137,7 +139,7 @@ class Downloader:
         resolved = []
         for block in input_blocks:
             obj = None
-            component = None if external else block.get((DataRegistryTarget.object_component, DataRegistryField.name))
+            component = block.get((DataRegistryTarget.object_component, DataRegistryField.name))
             target = DataRegistryTarget.external_object if external else DataRegistryTarget.data_product
             object_ref = block[target, DataRegistryField.object]
             if component:
@@ -176,11 +178,22 @@ class Downloader:
                     next(iter(sort_by_semver(v, key=(DataRegistryTarget.data_product, DataRegistryField.version))))
                 )
         else:
-            versioned_blocks = input_blocks
+            grouped = defaultdict(list)
+            for block in input_blocks:
+                grouped[
+                    block[DataRegistryTarget.external_object, DataRegistryField.doi_or_unique_name],
+                    block[DataRegistryTarget.external_object, DataRegistryField.title],
+                    block[DataRegistryTarget.object_component, DataRegistryField.name],
+                ].append(block)
+            versioned_blocks = []
+            for k, v in grouped.items():
+                versioned_blocks.append(
+                    next(iter(sort_by_semver(v, key=(DataRegistryTarget.external_object, DataRegistryField.version))))
+                )
 
         resolved = []
         for block in versioned_blocks:
-            cname = None if external else block.get((DataRegistryTarget.object_component, DataRegistryField.name))
+            cname = block.get((DataRegistryTarget.object_component, DataRegistryField.name))
             components = block[DataRegistryTarget.object, DataRegistryField.components]
             for component_url in components:
                 cblock = block.copy()
@@ -203,11 +216,10 @@ class Downloader:
             for k, v in storage_location.items():
                 cblock[DataRegistryTarget.storage_location, k] = v
             target = DataRegistryTarget.external_object if external else DataRegistryTarget.data_product
-            name_field = DataRegistryField.doi_or_unique_name if external else DataRegistryField.name
-            name = Path(cblock[target, name_field])
-            version = cblock[target, DataRegistryField.version]
+            name_fields = (DataRegistryField.doi_or_unique_name, DataRegistryField.title, DataRegistryField.version) if external else (DataRegistryField.name, DataRegistryField.version)
+            name = Path("/".join(filter(None, (cblock.get((target, name_field)) for name_field in name_fields))))
             output_filename = (
-                name / version / Path(cblock[DataRegistryTarget.storage_location, DataRegistryField.path]).name
+                name / Path(cblock[DataRegistryTarget.storage_location, DataRegistryField.path]).name
             )
             cblock[OUTPUT_FILENAME] = output_filename.as_posix()
             cblock[FULL_OUTPUT_FILENAME] = (self._data_directory / output_filename).as_posix()
@@ -238,17 +250,22 @@ class Downloader:
             version = block.get((DataRegistryTarget.external_object, DataRegistryField.version))
             if version is not None:
                 query_data[DataRegistryField.version] = version
+            title = block.get((DataRegistryTarget.external_object, DataRegistryField.title))
+            if title is not None:
+                query_data[DataRegistryField.title] = title
             external_objects = get_data(
                 query_data, DataRegistryTarget.external_object, self._data_registry_url, self._token, exact=False
             )
             if external_objects:
                 external_objects = sort_by_semver(external_objects)
-                grouped_external_objects = {}
-                for external_object in external_objects:
-                    external_object_name = external_object[DataRegistryField.doi_or_unique_name]
-                    if external_object_name not in grouped_external_objects:
-                        grouped_external_objects[external_object_name] = external_object
-                external_objects = list(grouped_external_objects.values())
+                if block.get((DataRegistryTarget.object_component, DataRegistryField.name)) is None:
+                    grouped_external_objects = {}
+                    for external_object in external_objects:
+                        external_object_name = external_object[DataRegistryField.doi_or_unique_name]
+                        external_object_title = external_object[DataRegistryField.title]
+                        if (external_object_name, external_object_title) not in grouped_external_objects:
+                            grouped_external_objects[external_object_name, external_object_title] = external_object
+                    external_objects = list(grouped_external_objects.values())
                 for external_object in external_objects:
                     cblock = block.copy()
                     for k, v in external_object.items():
@@ -280,6 +297,9 @@ class Downloader:
                 MetadataKey.doi_or_unique_name: block[
                     DataRegistryTarget.external_object, DataRegistryField.doi_or_unique_name
                 ],
+                MetadataKey.title: block[
+                    DataRegistryTarget.external_object, DataRegistryField.title
+                ],
                 MetadataKey.accessibility: block[DataRegistryTarget.storage_root, DataRegistryField.accessibility],
                 MetadataKey.version: block[DataRegistryTarget.external_object, DataRegistryField.version],
                 MetadataKey.verified_hash: block[DataRegistryTarget.storage_location, DataRegistryField.hash],

{data_pipeline_api-0.7.1 → data_pipeline_api-0.7.3}/data_pipeline_api/standard_api.py

@@ -5,7 +5,7 @@ from typing import Union, NamedTuple, Optional, Sequence, Type
 from data_pipeline_api.file_api import FileAPI, RunMetadata
 from data_pipeline_api.metadata import Metadata
 from data_pipeline_api.file_formats.parameter_file import (
-    Type as ParameterType,
+    ParameterType,
     Estimate,
     Distribution,
     Samples,

{data_pipeline_api-0.7.1 → data_pipeline_api-0.7.3}/data_pipeline_api.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: data-pipeline-api
-Version: 0.7.1
+Version: 0.7.3
 Summary: Interacts with SCRC data pipeline
 Home-page: UNKNOWN
 Author: SCRC

{data_pipeline_api-0.7.1 → data_pipeline_api-0.7.3}/data_pipeline_api.egg-info/SOURCES.txt

@@ -97,6 +97,7 @@ java_api/standard_api/src/main/java/uk/ramp/distribution/MinMax.java
 java_api/standard_api/src/main/java/uk/ramp/distribution/MinMaxDeserializer.java
 java_api/standard_api/src/main/java/uk/ramp/distribution/MinMaxSerializer.java
 java_api/standard_api/src/main/java/uk/ramp/estimate/Estimate.java
+java_api/standard_api/src/main/java/uk/ramp/mapper/DataPipelineMapper.java
 java_api/standard_api/src/main/java/uk/ramp/objects/NumericalArray.java
 java_api/standard_api/src/main/java/uk/ramp/objects/NumericalArrayImpl.java
 java_api/standard_api/src/main/java/uk/ramp/objects/StandardArrayDataReader.java
@@ -110,6 +111,8 @@ java_api/standard_api/src/main/java/uk/ramp/parameters/ParameterDataReader.java
 java_api/standard_api/src/main/java/uk/ramp/parameters/ParameterDataReaderImpl.java
 java_api/standard_api/src/main/java/uk/ramp/parameters/ParameterDataWriter.java
 java_api/standard_api/src/main/java/uk/ramp/parameters/ParameterDataWriterImpl.java
+java_api/standard_api/src/main/java/uk/ramp/parameters/RandomGeneratorDeserializer.java
+java_api/standard_api/src/main/java/uk/ramp/parameters/RandomGeneratorSerializer.java
 java_api/standard_api/src/main/java/uk/ramp/parameters/ReadComponent.java
 java_api/standard_api/src/main/java/uk/ramp/parameters/WriteComponent.java
 java_api/standard_api/src/main/java/uk/ramp/samples/Sampler.java

{data_pipeline_api-0.7.1 → data_pipeline_api-0.7.3}/java_api/file_api/src/main/java/uk/ramp/config/Config.java

@@ -16,6 +16,7 @@ import uk.ramp.metadata.ImmutableMetadataItem;
 @JsonSerialize
 @JsonDeserialize
 public interface Config {
+
   @JsonProperty("data_directory")
   Optional<String> internalDataDirectory();
 

{data_pipeline_api-0.7.1 → data_pipeline_api-0.7.3}/java_api/file_api/src/main/java/uk/ramp/metadata/MetadataItem.java

@@ -65,6 +65,8 @@ public interface MetadataItem {
 
   Optional<String> description();
 
+  Optional<String> accessibility();
+
   default boolean isSuperSetOf(MetadataItem key) {
     return List.<Function<MetadataItem, Optional<String>>>of(
             MetadataItem::internalFilename,
@@ -78,7 +80,8 @@ public interface MetadataItem {
             MetadataItem::source,
             metadataItem -> issuesInComparableFormat(metadataItem.issues().orElse(List.of())),
             MetadataItem::namespace,
-            MetadataItem::description)
+            MetadataItem::description,
+            MetadataItem::accessibility)
         .stream()
         .map(func -> func.andThen(s -> s.orElse("")))
         .allMatch(func -> keyIsEitherNotPresentOrEqual(func.apply(key), func.apply(this)));
@@ -158,6 +161,10 @@ public interface MetadataItem {
     Function<MetadataItem, Boolean> isDataDirectoryPresent =
         meta -> meta.dataDirectory().isPresent();
     Function<MetadataItem, Boolean> isIssuesPresent = meta -> meta.issues().isPresent();
+    Function<MetadataItem, Boolean> isNamespacePresent = meta -> meta.namespace().isPresent();
+    Function<MetadataItem, Boolean> isDescriptionPresent = meta -> meta.description().isPresent();
+    Function<MetadataItem, Boolean> isAccessibilityPresent =
+        meta -> meta.accessibility().isPresent();
 
     if (shouldOverride.apply(isFilenamePresent)) {
       newMetadataItem =
@@ -205,14 +212,18 @@ public interface MetadataItem {
       newMetadataItem = newMetadataItem.withIssues(metadataOverride.issues().get());
     }
 
-    if (metadataOverride.namespace().isPresent()) {
+    if (shouldOverride.apply(isNamespacePresent)) {
       newMetadataItem = newMetadataItem.withNamespace(metadataOverride.namespace().get());
     }
 
-    if (metadataOverride.description().isPresent()) {
+    if (shouldOverride.apply(isDescriptionPresent)) {
       newMetadataItem = newMetadataItem.withDescription(metadataOverride.description().get());
     }
 
+    if (shouldOverride.apply(isAccessibilityPresent)) {
+      newMetadataItem = newMetadataItem.withAccessibility(metadataOverride.accessibility().get());
+    }
+
     return newMetadataItem;
   }
 }

{data_pipeline_api-0.7.1 → data_pipeline_api-0.7.3}/java_api/file_api/src/main/java/uk/ramp/yaml/BaseYamlWriter.java

@@ -1,5 +1,6 @@
 package uk.ramp.yaml;
 
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator.Feature;
@@ -19,7 +20,8 @@ public class BaseYamlWriter implements YamlWriter {
             .disable(Feature.WRITE_DOC_START_MARKER)
             .registerModule(new Jdk8Module())
             .registerModule(new JavaTimeModule())
-            .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
+            .configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false)
+            .setSerializationInclusion(Include.NON_DEFAULT);
   }
 
   @Override