data_pipeline_api 0.7.0__tar.gz → 0.7.2__tar.gz

{data_pipeline_api-0.7.0 → data_pipeline_api-0.7.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.0
 Name: data_pipeline_api
-Version: 0.7.0
+Version: 0.7.2
 Summary: Interacts with SCRC data pipeline
 Home-page: UNKNOWN
 Author: SCRC

data_pipeline_api-0.7.2/data_pipeline_api/database_file_api.py

@@ -0,0 +1,67 @@
+from logging import getLogger
+from io import IOBase
+import __main__
+from data_pipeline_api.file_api import FileAPI, RunMetadata
+from data_pipeline_api.registry.download import download_from_configs
+from data_pipeline_api.registry.access_upload import upload_model_run
+from data_pipeline_api.registry.utils import get_access_token, get_remote_options
+
+logger = getLogger(__name__)
+
+
+class DatabaseFileAPI(FileAPI):
+    """The DatabaseFileAPI extends the FileAPI and provides direct access to the
+    database, but still mediated by the file system.
+    """
+
+    RUN_METADATA_NEEDED_FOR_DOWNLOAD = {RunMetadata.data_registry_url}
+    RUN_METADATA_NEEDED_FOR_UPLOAD = {
+        RunMetadata.data_directory,
+        RunMetadata.run_id,
+        RunMetadata.model_version,
+        RunMetadata.model_name,
+        RunMetadata.open_timestamp,
+        RunMetadata.git_sha,
+        RunMetadata.git_repo,
+        RunMetadata.remote_uri,
+        RunMetadata.description,
+    }
+
+    def _has_run_metadata(self, keys):
+        missing_keys = {key for key in keys if key not in self._run_metadata}
+        if missing_keys:
+            logger.warning("missing %s", missing_keys)
+            return False
+        return True
+
+    def open_for_read(self, **call_metadata) -> IOBase:
+        """Attempt a read, and if it fails, attempt a download and try the read again.
+        """
+        try:
+            return super().open_for_read(**call_metadata)
+        except (KeyError, FileNotFoundError):
+            if self._has_run_metadata(DatabaseFileAPI.RUN_METADATA_NEEDED_FOR_DOWNLOAD):
+                download_from_configs(
+                    self._run_metadata,
+                    [{"where": self.get_read_metadata(call_metadata)}],
+                    get_access_token(),
+                    self._root,
+                )
+                self.load_metadata_store()
+                return super().open_for_read(**call_metadata)
+            raise
+
+    def close(self):
+        """Close as normal, then attempt to upload the results to the database.
+        """
+        super().close()
+        if self._has_run_metadata(DatabaseFileAPI.RUN_METADATA_NEEDED_FOR_UPLOAD):
+            upload_model_run(
+                config_filename=self._access_log_path,
+                model_config_filename=self._config_filename,
+                submission_script_filename=self._run_metadata.get(
+                    RunMetadata.submission_script, __main__.__file__
+                ),
+                remote_options=get_remote_options(),
+                token=get_access_token(),
+            )

{data_pipeline_api-0.7.0 → data_pipeline_api-0.7.2}/data_pipeline_api/file_api.py

@@ -75,12 +75,31 @@ class WriteAccess(FileAccess):
         return dict(type="write", **super().to_access_log_record(hash_cache))
 
 
+class RunMetadata:
+    run_id = "run_id"
+    open_timestamp = "open_timestamp"
+    close_timestamp = "close_timestamp"
+    data_directory = "data_directory"
+
+    git_repo = "git_repo"
+    git_sha = "git_sha"
+    default_input_namespace = "default_input_namespace"
+    default_output_namespace = "default_output_namespace"
+    data_registry_url = "data_registry_url"
+    remote_uri = "remote_uri"
+    remote_uri_override = "remote_uri_override"
+    model_version = "model_version"
+    model_name = "model_name"
+    description = "description"
+    submission_script = "submission_script"
+
+
 class FileAPI:
     RESERVED_RUN_METADATA_KEYS = {
-        "run_id",
-        "open_timestamp",
-        "close_timestamp",
-        "data_directory",
+        RunMetadata.run_id,
+        RunMetadata.open_timestamp,
+        RunMetadata.close_timestamp,
+        RunMetadata.data_directory,
     }
 
     @staticmethod
@@ -137,7 +156,6 @@ class FileAPI:
         self._run_metadata = {}
 
         self._open_timestamp = datetime.now()
-        self._run_metadata["open_timestamp"] = self._open_timestamp
         logger.debug("open_timestamp = %s", self._open_timestamp)
 
         if config_filename is None:
@@ -153,7 +171,6 @@ class FileAPI:
         logger.debug("root = %s", self._root.resolve())
 
         self._run_id = self._config.get("run_id", uuid4().hex)
-        self._run_metadata["run_id"] = self._run_id
         logger.debug("run_id = %s", self._run_id)
 
         self._fail_on_hash_mismatch = self._config.get("fail_on_hash_mismatch", True)
@@ -165,7 +182,7 @@ class FileAPI:
 
         data_directory = self._config.get("data_directory", self._root)
         self._unnormalised_data_directory = Path(data_directory)
-        self._run_metadata["data_directory"] = str(self._unnormalised_data_directory)
+
         self._data_directory = FileAPI.normalise_path(
             self._root, self._unnormalised_data_directory
         )
@@ -181,7 +198,12 @@ class FileAPI:
             )
         logger.debug("access_log_path = %s", self._access_log_path)
 
-        self._run_metadata.update(self._config.get("metadata", {}))
+        # Carefully set up the run metadata, preferring overrides from config.
+        self._run_metadata["run_id"] = self._run_id
+        self._run_metadata["data_directory"] = str(self._unnormalised_data_directory)
+        self._run_metadata["open_timestamp"] = self._open_timestamp
+        self._run_metadata.update(self._config.get("run_metadata", {}))
+
         self._read_overrides = FileAPI.construct_overrides(self._config.get("read", ()))
         self._write_overrides = FileAPI.construct_overrides(
             self._config.get("write", ())
@@ -215,12 +237,13 @@ class FileAPI:
                 == read_metadata[MetadataKey.verified_hash]
             ):
                 logger.debug("verified hash")
-            raise ValueError(
-                (
-                    "calculated hash {calculated_hash} != "
-                    "verified hash {verified_hash}"
-                ).format(**read_metadata)
-            )
+            else:
+                raise ValueError(
+                    (
+                        "calculated hash {calculated_hash} != "
+                        "verified hash {verified_hash}"
+                    ).format(**read_metadata)
+                )
 
         logger.debug("open('%s', mode='rb')", path)
         file = open(path, mode="rb")
@@ -276,14 +299,14 @@ class FileAPI:
         return file
 
     def set_run_metadata(self, key: str, value: Any):
-        """Set run-level metadata.
+        """Set the value for a run-level metadata key.
         """
         if key in FileAPI.RESERVED_RUN_METADATA_KEYS:
             raise ValueError(f"{key} is reserved")
         self._run_metadata[key] = value
 
     def get_run_metadata(self, key: str) -> Any:
-        """Get run-level metadata.
+        """Get the value for a run-level metadata key.
         """
         return self._run_metadata[key]